Initial commit.
commit
fad5d15987
GPG Key ID:
1573F6D8EFE4D0CF
|
|
@ -0,0 +1,52 @@
|
|||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||
*.o
|
||||
*.a
|
||||
*.so
|
||||
|
||||
# Folders
|
||||
_obj
|
||||
_test
|
||||
|
||||
# Architecture specific extensions/prefixes
|
||||
*.[568vq]
|
||||
[568vq].out
|
||||
|
||||
*.cgo1.go
|
||||
*.cgo2.c
|
||||
_cgo_defun.c
|
||||
_cgo_gotypes.go
|
||||
_cgo_export.*
|
||||
|
||||
_testmain.go
|
||||
|
||||
# Built binaries
|
||||
|
||||
smtplogparser
|
||||
*.exe
|
||||
*.test
|
||||
|
||||
# Windows image file caches
|
||||
Thumbs.db
|
||||
ehthumbs.db
|
||||
|
||||
# Folder config file
|
||||
Desktop.ini
|
||||
|
||||
# Recycle Bin used on file shares
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# Windows Installer files
|
||||
*.cab
|
||||
*.msi
|
||||
*.msm
|
||||
*.msp
|
||||
|
||||
[._]*.s[a-w][a-z]
|
||||
[._]s[a-w][a-z]
|
||||
*.un~
|
||||
Session.vim
|
||||
.netrwhist
|
||||
*~
|
||||
|
||||
# Log files
|
||||
*.log
|
||||
|
|
@ -0,0 +1,312 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"log"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
AnonymizedText = "**ANONYMIZED**"
|
||||
)
|
||||
|
||||
// Syntax of a message:
|
||||
// DD-MM-YYYY hh:mm:ss LogLevel Service [\[ID\]] LogMessage[ (MessageID)][.]
|
||||
|
||||
// Example failing mail:
|
||||
// 19-03-2018 03:54:07 3 Fuzzy-Filter (40935FC192D) phase 1 (ex) 325ms result: Major=spam Minor=normal Fallback=spam Virus=
|
||||
|
||||
var (
|
||||
// 05-03-2018 15:36:14 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Message delivered to: <person@kunde.de>
|
||||
rxSuccessfulMessage = regexp.MustCompile(`^Message\s+delivered\s+to:\s+<(.+)>\s*$`)
|
||||
|
||||
rxMapMessageID = regexp.MustCompile(`^MessageID: .+\s*$`)
|
||||
rxSubject = regexp.MustCompile(`^Testing: (.+)\s*$`)
|
||||
rxSenderAddress = regexp.MustCompile(`(?i)^Recieve: MAIL\s+FROM:\s*<(.+)>\s*`)
|
||||
rxRecipientAddress = regexp.MustCompile(`(?i)^Recieve: RCPT\s+TO:\s*<(.+)>\s*`)
|
||||
|
||||
/*
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on MAIL FROM: <MaxMustermann@daimler.com> = 250 2.1.0 Sender OK
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on RCPT TO: <person@kunde.de> = 250 2.1.5 Recipient OK
|
||||
*/
|
||||
rxSendMailFrom = regexp.MustCompile(`^Receive on MAIL\s+FROM:\s*<(.+)>\s+`)
|
||||
rxSendRcptTo = regexp.MustCompile(`^Receive on RCPT\s+TO:\s*<(.+)>\s+`)
|
||||
|
||||
maxMessageAge = 30 * time.Second
|
||||
)
|
||||
|
||||
type Log struct {
|
||||
// Maps sessions by SMTP server session ID
|
||||
smtpServerSessions map[string]*SMTPServerSession
|
||||
CSVWriter *csv.Writer
|
||||
AnonymizeSubject bool
|
||||
}
|
||||
|
||||
func (this *Log) flushSingle(sessionId string) (err error) {
|
||||
session := this.smtpServerSessions[sessionId]
|
||||
delete(this.smtpServerSessions, sessionId)
|
||||
if this.CSVWriter != nil {
|
||||
if err := this.CSVWriter.Write(session.csv()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Log) flush(timeout time.Duration, timestamp time.Time) (err error) {
|
||||
outerLoop:
|
||||
for {
|
||||
for id, session := range this.smtpServerSessions {
|
||||
if timestamp.Sub(session.Timestamp) >= timeout {
|
||||
err = this.flushSingle(id)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
continue outerLoop
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Log) Parse(r io.Reader) (err error) {
|
||||
this.smtpServerSessions = map[string]*SMTPServerSession{}
|
||||
|
||||
textreader := bufio.NewScanner(r)
|
||||
textreader.Scan()
|
||||
thisLine := textreader.Text()
|
||||
nextLine := ""
|
||||
var session *SMTPServerSession
|
||||
|
||||
if this.CSVWriter != nil {
|
||||
if err = this.CSVWriter.Write(sessionCSVHeader); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
hasNextLineAvailable := textreader.Scan()
|
||||
|
||||
if hasNextLineAvailable {
|
||||
nextLine = textreader.Text()
|
||||
|
||||
// Try parsing first two fields of the upcoming line. If it doesn't
|
||||
// work, assume leakage of previous line content.
|
||||
fields := strings.Fields(nextLine)
|
||||
if len(fields) < 2 {
|
||||
// this is a continuation (too few fields)
|
||||
thisLine = strings.TrimSpace(thisLine) + nextLine
|
||||
continue
|
||||
}
|
||||
_, err = time.Parse("02-01-2006 15:04:05", strings.Join(fields[0:2], " "))
|
||||
if err != nil {
|
||||
// this is a continuation (first two fields don't follow usual line syntax)
|
||||
err = nil
|
||||
thisLine = strings.TrimSpace(thisLine) + nextLine
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if len(strings.TrimSpace(thisLine)) > 0 {
|
||||
fields := strings.Fields(thisLine)
|
||||
if len(fields) < 2 {
|
||||
log.Print("WARNING: Found a line with less than 2 fields to work with. Skipping:", thisLine)
|
||||
} else {
|
||||
|
||||
timestamp, err := time.Parse("02-01-2006 15:04:05", strings.Join(fields[0:2], " "))
|
||||
if err != nil {
|
||||
// first two fields don't follow usual line syntax
|
||||
return err
|
||||
}
|
||||
|
||||
serviceName := fields[3]
|
||||
sessionId := ""
|
||||
messageId := ""
|
||||
logMessage := ""
|
||||
logMessageStartIndex := 4
|
||||
logMessageEndIndex := len(fields)
|
||||
|
||||
// Is the session ID stored in the message?
|
||||
// For this, fields[4] needs to start with [ and needs to contain an 8-char long ID.
|
||||
if strings.HasPrefix(fields[4], "[") &&
|
||||
strings.HasSuffix(fields[4], "]") &&
|
||||
len(fields[4]) == 1+8+1 {
|
||||
// Yup, there's an ID here!
|
||||
sessionId = fields[4][1:9]
|
||||
logMessageStartIndex = 5
|
||||
|
||||
}
|
||||
|
||||
// Check if there is a message ID at the end of the log message
|
||||
messageIdPart := strings.TrimRight(fields[len(fields)-1], ".")
|
||||
if strings.HasPrefix(messageIdPart, "(") &&
|
||||
strings.HasSuffix(messageIdPart, ")") &&
|
||||
len(messageIdPart) == 1+11+1 {
|
||||
// Yup, there's a message ID!
|
||||
messageId = messageIdPart[1:12]
|
||||
logMessageEndIndex--
|
||||
} else if strings.HasPrefix(fields[logMessageStartIndex], "(") &&
|
||||
strings.HasSuffix(fields[logMessageStartIndex], ")") &&
|
||||
len(fields[logMessageStartIndex]) == 1+11+1 {
|
||||
// Message ID but in fields[logMessageStartIndex], so shift start of log message
|
||||
messageId = fields[logMessageStartIndex][1:12]
|
||||
logMessageStartIndex++
|
||||
}
|
||||
|
||||
logMessage = strings.Join(fields[logMessageStartIndex:logMessageEndIndex], " ")
|
||||
|
||||
//log.Println(serviceName, "-", sessionId, "-", messageId, "-", logMessage)
|
||||
|
||||
switch serviceName {
|
||||
case "SMTPClient":
|
||||
// Could contain success message for delivery
|
||||
//log.Println(">> parsing for delivery success")
|
||||
if rxSuccessfulMessage.MatchString(logMessage) {
|
||||
//log.Println(">> got delivery success!")
|
||||
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||
if session == nil {
|
||||
log.Println("WARNING: Found a delivery success message for a message we don't know:", logMessage)
|
||||
break
|
||||
}
|
||||
session.IsSent = true
|
||||
// immediately unregister
|
||||
if err = this.flushSingle(session.SMTPServerSessionID); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Could contain sender
|
||||
subm := rxSendMailFrom.FindStringSubmatch(logMessage)
|
||||
if subm != nil {
|
||||
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||
if session == nil {
|
||||
log.Println("WARNING: Found a log line about the sender for a passed message we don't know:", logMessage)
|
||||
break
|
||||
}
|
||||
session.SenderAddress = subm[1]
|
||||
break
|
||||
}
|
||||
|
||||
// Could contain recipient
|
||||
subm = rxSendRcptTo.FindStringSubmatch(logMessage)
|
||||
if subm != nil {
|
||||
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||
if session == nil {
|
||||
log.Println("WARNING: Found a log line about the recipient for a passed message we don't know:", logMessage)
|
||||
break
|
||||
}
|
||||
session.RecipientAddress = subm[1]
|
||||
break
|
||||
}
|
||||
|
||||
case "SMTPServer":
|
||||
if len(sessionId) > 0 {
|
||||
session = this.createOrFetchSMTPServerSession(sessionId, timestamp)
|
||||
}
|
||||
|
||||
// Could contain recipient
|
||||
subm := rxRecipientAddress.FindStringSubmatch(logMessage)
|
||||
//log.Println(">> parsing for recipient")
|
||||
if subm != nil {
|
||||
//log.Println(">> got recipient")
|
||||
// We got submatches, this has a recipient!
|
||||
session.RecipientAddress = subm[1]
|
||||
break
|
||||
}
|
||||
// Could contain sender
|
||||
subm = rxSenderAddress.FindStringSubmatch(logMessage)
|
||||
//log.Println(">> parsing for sender")
|
||||
if subm != nil {
|
||||
//log.Println(">> got sender")
|
||||
// We got submatches, this has a sender!
|
||||
session.SenderAddress = subm[1]
|
||||
break
|
||||
}
|
||||
// Could have metadata about message ID
|
||||
//log.Println(">> parsing for message ID map")
|
||||
if len(messageId) > 0 {
|
||||
//log.Println(">> got message ID map 1")
|
||||
subm = rxMapMessageID.FindStringSubmatch(logMessage)
|
||||
if subm != nil {
|
||||
//log.Println(">> got message ID map 2")
|
||||
// We got submatches, this lets us map session ID to message ID!
|
||||
session.MessageID = messageId
|
||||
//log.Println(messageId, "->", sessionId)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
case "SBL-Filter", "SWL-Filter":
|
||||
// Could contain the subject
|
||||
//log.Println(">> parsing for subject")
|
||||
subm := rxSubject.FindStringSubmatch(logMessage)
|
||||
if subm != nil {
|
||||
//log.Println(">> got subject")
|
||||
// We got submatches, this has a subject!
|
||||
subject := subm[1]
|
||||
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||
if session == nil {
|
||||
log.Println("WARNING: Found a subject line for a message we don't know:", logMessage)
|
||||
break
|
||||
}
|
||||
if this.AnonymizeSubject {
|
||||
session.Subject = AnonymizedText
|
||||
} else {
|
||||
session.Subject = subject
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.flush(maxMessageAge, timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
if !hasNextLineAvailable {
|
||||
break
|
||||
}
|
||||
|
||||
thisLine = nextLine
|
||||
}
|
||||
|
||||
this.flush(0, time.Now())
|
||||
|
||||
if this.CSVWriter != nil {
|
||||
this.CSVWriter.Flush()
|
||||
}
|
||||
|
||||
/*if err = textreader.Err(); err != nil {
|
||||
return
|
||||
}*/
|
||||
err = textreader.Err()
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Log) createOrFetchSMTPServerSession(sessionId string, timestamp time.Time) (session *SMTPServerSession) {
|
||||
if len(sessionId) <= 0 {
|
||||
panic("Empty session ID")
|
||||
}
|
||||
session, ok := this.smtpServerSessions[sessionId]
|
||||
if !ok {
|
||||
session = &SMTPServerSession{
|
||||
SMTPServerSessionID: sessionId,
|
||||
Timestamp: timestamp,
|
||||
}
|
||||
this.smtpServerSessions[sessionId] = session
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (this *Log) FindSMTPServerSessionByMessageId(messageId string, timestamp time.Time) (session *SMTPServerSession) {
|
||||
for _, currentSession := range this.smtpServerSessions {
|
||||
if currentSession.IsMatchByMessageId(messageId, timestamp) {
|
||||
return currentSession
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
//"errors"
|
||||
"encoding/csv"
|
||||
"io"
|
||||
)
|
||||
|
||||
func TransformLog(r io.Reader, w io.Writer) (err error) {
|
||||
l := new(Log)
|
||||
l.CSVWriter = csv.NewWriter(w)
|
||||
err = l.Parse(r)
|
||||
return
|
||||
}
|
||||
|
|
@ -0,0 +1,176 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"io/ioutil"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
testMessageFailed = `19-03-2018 03:54:05 2 SMTPServer [F1AF6651] New connection from 80.249.239.48
|
||||
19-03-2018 03:54:05 3 SMTPServer Testing 80.249.239.48 on bl.spamcop.net
|
||||
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 220 mail.kunde.de SMTP server ready
|
||||
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Recieve: EHLO site.azauto.com.ua
|
||||
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250-OK
|
||||
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250-STARTTLS
|
||||
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250 SIZE 104857600
|
||||
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Ehlo Greeting from: [80.249.239.48] - site.azauto.com.ua
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: MAIL FROM:<info@byphil.comt>
|
||||
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Mail from: info@byphil.comt
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 OK smtp ready for info@byphil.comt
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: RCPT TO:<abteilung@kunde.de>
|
||||
19-03-2018 03:54:06 3 SMTPServer RVC-Filter testing: abteilung@kunde.de
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 OK smtp ready for abteilung@kunde.de
|
||||
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Mail to: <abteilung@kunde.de> accepted
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: DATA
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 354 Send message. End with CRLF.CRLF
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] MessageID: <00D6549C_3B615A6F_reddoxx@mail.kunde.de> (40935FC192D).
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] TransactionID: F84F9E0625A025C54F7DBD19AC13DEBF4C2736FB (40935FC192D).
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Saving message ... (40935FC192D)
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 message queued (40935FC192D)
|
||||
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Message queued (40935FC192D)
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: QUIT
|
||||
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 221 closing connection
|
||||
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Client disconnected.
|
||||
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Disconnected from 80.249.239.48
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Thread started. Validating message (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Load message ... (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Load message finished. (40935FC192D)
|
||||
19-03-2018 03:54:07 2 MailSealer [EB5FAD39] Message has no secure components. (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Starting validation of Message (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Using Profile: (1) Default-Filterprofile for <abteilung@kunde.de>
|
||||
19-03-2018 03:54:07 3 DWL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||
19-03-2018 03:54:07 3 AWL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||
19-03-2018 03:54:07 3 AWL-Filter info@byphil.comt is *NOT* on sender address whitelist. 0ms (40935FC192D)
|
||||
19-03-2018 03:54:07 3 SWL-Filter Testing:
|
||||
Finanzdienstleiter (40935FC192D)
|
||||
19-03-2018 03:54:07 3 RBL-Filter Testing 80.249.239.48 on bl.spamcop.net (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Advanced-RBL-Filter Testing (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Fuzzy-Filter Testing phase 1 (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Fuzzy-Filter (40935FC192D) phase 1 (ex) 325ms result: Major=spam Minor=normal Fallback=spam Virus=
|
||||
19-03-2018 03:54:07 2 Fuzzy-Filter (40935FC192D) blocked by phase 1
|
||||
19-03-2018 03:54:07 2 Validator [EB5FAD39] Filter: 'Fuzzy-Filter' will prevent archiving.
|
||||
19-03-2018 03:54:07 3 DBL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||
19-03-2018 03:54:07 3 ABL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||
19-03-2018 03:54:07 3 SBL-Filter Testing: Finanzdienstleiter (40935FC192D)
|
||||
19-03-2018 03:54:07 3 VirusScanner No Virus found in message (40935FC192D)
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] - abteilung@kunde.de validated. Result: 30
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Thread terminated.
|
||||
19-03-2018 03:54:07 3 Validator [EB5FAD39] Validation of Message (40935FC192D) finished.`
|
||||
|
||||
testMessageSuccess = `05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: RCPT TO:<person@kunde.de>
|
||||
05-03-2018 15:36:13 3 SMTPServer RVC-Filter testing: person@kunde.de
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 250 OK smtp ready for person@kunde.de
|
||||
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Mail to: <person@kunde.de> accepted
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: DATA
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 354 Send message. End with CRLF.CRLF
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] MessageID: <754a924065cc4c15ace114d9196e16ef@DE35S004EXC61.wp.corpintra.net> (4DBAB2C02A3).
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] TransactionID: 82CDA2B813955EB42DCDB70BDAED57F2A6C53C29 (4DBAB2C02A3).
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Saving message ... (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 250 message queued (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Message queued (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: QUIT
|
||||
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 221 closing connection
|
||||
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Client disconnected.
|
||||
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Disconnected from 141.113.102.113
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Thread started. Validating message (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Load message ... (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Load message finished. (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 2 MailSealer [E8F016EA] Message has no secure components. (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Starting validation of Message (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Using Profile: (1) Default-Filterprofile for <person@kunde.de>
|
||||
05-03-2018 15:36:13 3 DWL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 AWL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 AWL-Filter MaxMustermann@daimler.com is *NOT* on sender address whitelist. 0ms (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 SWL-Filter Testing: AW: Urlaubsplanung (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 RBL-Filter Testing 141.113.102.113 on bl.spamcop.net (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Advanced-RBL-Filter Testing (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Fuzzy-Filter Testing phase 1 (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Fuzzy-Filter (4DBAB2C02A3) phase 1 (ex) 161ms result: Major=clean Minor=normal Fallback=clean Virus=
|
||||
05-03-2018 15:36:13 3 Fuzzy-Filter Testing phase 2 (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Fuzzy-Filter Testing 1d28528ffe972a11b3789fac292200b2 (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 DBL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 ABL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 SBL-Filter Testing: AW: Urlaubsplanung (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 VirusScanner No Virus found in message (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] - person@kunde.de validated. Result: 0
|
||||
05-03-2018 15:36:13 3 Validator Wait for MailDepot lock ...
|
||||
05-03-2018 15:36:13 3 Validator MailDepot locked.
|
||||
05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 start ... - (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 2 Archive No policy matched. Message will be archived.
|
||||
05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 before critical section ... - (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 2 Archive Archiving for MailDepot 2.0 - (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Archive Add MailDepot 2.0 spooler entry: 82CDA2B813955EB42DCDB70BDAED57F2A6C53C29.rdxmt0
|
||||
05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 finished. - (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 3 Validator MailDepot unlocked.
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Thread terminated.
|
||||
05-03-2018 15:36:13 3 Validator [E8F016EA] Validation of Message (4DBAB2C02A3) finished.
|
||||
05-03-2018 15:36:13 3 SMTPClient [4041136960] Query queue
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] Thread started. Sending message (4DBAB2C02A3)
|
||||
05-03-2018 15:36:13 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Sending message for: <MaxMustermann@daimler.com>
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) connecting to: 10.1.0.18:25
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on HELO: 250 XSHADOW
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on MAIL FROM: <MaxMustermann@daimler.com> = 250 2.1.0 Sender OK
|
||||
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on RCPT TO: <person@kunde.de> = 250 2.1.5 Recipient OK
|
||||
05-03-2018 15:36:14 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Data result: 250 2.6.0 <754a924065cc4c15ace114d9196e16ef@DE35S004EXC61.wp.corpintra.net> [InternalId=1161195] Queued mail for delivery
|
||||
05-03-2018 15:36:14 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Message delivered to: <person@kunde.de>
|
||||
05-03-2018 15:36:14 3 SMTPClient [E8F00BA5] Thread terminated.`
|
||||
)
|
||||
|
||||
func Test_Message_Successful(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
transformer := new(Log)
|
||||
transformer.CSVWriter = csv.NewWriter(buf)
|
||||
err := transformer.Parse(strings.NewReader(testMessageSuccess))
|
||||
|
||||
require.NotEqual(t, buf.Len(), 0)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, strings.Join(sessionCSVHeader, ",")+
|
||||
"\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3",
|
||||
strings.TrimSpace(buf.String()))
|
||||
}
|
||||
|
||||
func Test_Message_DupeSessionId(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
transformer := new(Log)
|
||||
transformer.CSVWriter = csv.NewWriter(buf)
|
||||
err := transformer.Parse(strings.NewReader(testMessageSuccess + "\n" + testMessageSuccess))
|
||||
|
||||
require.NotEqual(t, buf.Len(), 0)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, strings.Join(sessionCSVHeader, ",")+
|
||||
"\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3"+
|
||||
"\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3",
|
||||
strings.TrimSpace(buf.String()))
|
||||
}
|
||||
|
||||
func Test_Message_Failed(t *testing.T) {
|
||||
buf := new(bytes.Buffer)
|
||||
|
||||
transformer := new(Log)
|
||||
transformer.CSVWriter = csv.NewWriter(buf)
|
||||
err := transformer.Parse(strings.NewReader(testMessageFailed))
|
||||
|
||||
require.NotEqual(t, buf.Len(), 0)
|
||||
require.Nil(t, err)
|
||||
require.Equal(t, strings.Join(sessionCSVHeader, ",")+
|
||||
"\n2018-03-19,03:54:05,info@byphil.comt,abteilung@kunde.de,Finanzdienstleiter,false,F1AF6651,40935FC192D",
|
||||
strings.TrimSpace(buf.String()))
|
||||
}
|
||||
|
||||
func Benchmark_Messages(b *testing.B) {
|
||||
output := ioutil.Discard
|
||||
|
||||
transformer := new(Log)
|
||||
transformer.CSVWriter = csv.NewWriter(output)
|
||||
|
||||
for n := 0; n < b.N; n++ {
|
||||
transformer.Parse(strings.NewReader(testMessageSuccess))
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
package internal
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
sessionCSVHeader = []string{
|
||||
"Date",
|
||||
"Time",
|
||||
"From",
|
||||
"To",
|
||||
"Subject",
|
||||
"Passed",
|
||||
"SMTPServer session ID",
|
||||
"Message short ID",
|
||||
}
|
||||
)
|
||||
|
||||
type SMTPServerSession struct {
|
||||
SMTPServerSessionID string
|
||||
MessageID string
|
||||
RecipientAddress string
|
||||
SenderAddress string
|
||||
Subject string
|
||||
IsSent bool
|
||||
Timestamp time.Time
|
||||
}
|
||||
|
||||
func (sss *SMTPServerSession) csv() []string {
|
||||
return []string{
|
||||
sss.Timestamp.Format("2006-01-02"),
|
||||
sss.Timestamp.Format("15:04:05"),
|
||||
sss.SenderAddress,
|
||||
sss.RecipientAddress,
|
||||
sss.Subject,
|
||||
fmt.Sprintf("%v", sss.IsSent),
|
||||
sss.SMTPServerSessionID,
|
||||
sss.MessageID,
|
||||
}
|
||||
}
|
||||
|
||||
func (sss *SMTPServerSession) IsMatchByMessageId(messageId string, timestamp time.Time) (result bool) {
|
||||
result = strings.EqualFold(messageId, sss.MessageID) &&
|
||||
timestamp.Sub(sss.Timestamp) < maxMessageAge
|
||||
return
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"encoding/csv"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/alecthomas/kingpin"
|
||||
|
||||
"local/smtpparse/internal"
|
||||
)
|
||||
|
||||
var (
|
||||
flagInputFiles = kingpin.Flag("input-file", "Input files. Can be glob patterns like `*.log` and `-` for standard input pipe.").Short('i').Required().Strings()
|
||||
flagOutputFile = kingpin.Flag("output-file", "Output file. Can be `-` for standard output pipe.").Default("-").Short('o').String()
|
||||
flagRecipients = kingpin.Flag("recipients", "List of recipients to find entries for. Defaults to everyone.").Short('r').Strings()
|
||||
flagAnonymize = kingpin.Flag("anonymize", "Anonymizes mail subjects.").Bool()
|
||||
)
|
||||
|
||||
func main() {
|
||||
kingpin.Parse()
|
||||
|
||||
// Open input file
|
||||
readers := []io.Reader{}
|
||||
for _, globPattern := range *flagInputFiles {
|
||||
// allow stdin
|
||||
if globPattern == "-" {
|
||||
readers = append(readers, os.Stdin, strings.NewReader("\n"))
|
||||
continue
|
||||
}
|
||||
|
||||
// check for files of pattern
|
||||
matches, err := filepath.Glob(globPattern)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
for _, match := range matches {
|
||||
f, err := os.Open(match)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
readers = append(readers, f, strings.NewReader("\n"))
|
||||
}
|
||||
}
|
||||
r := io.MultiReader(readers...)
|
||||
|
||||
// Create output
|
||||
var outputWriter io.Writer
|
||||
if *flagOutputFile == "-" {
|
||||
outputWriter = os.Stdout
|
||||
} else {
|
||||
f, err := os.Create(*flagOutputFile)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
outputWriter = f
|
||||
}
|
||||
|
||||
// Parse
|
||||
transformer := new(internal.Log)
|
||||
transformer.AnonymizeSubject = *flagAnonymize
|
||||
transformer.CSVWriter = csv.NewWriter(outputWriter)
|
||||
err := transformer.Parse(r)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue