From fad5d159874f97937f84b8051c316e44d4e69c77 Mon Sep 17 00:00:00 2001 From: Carl Kittelberger Date: Fri, 23 Mar 2018 00:35:29 +0100 Subject: [PATCH] Initial commit. --- .gitignore | 52 +++++++ internal/log.go | 312 ++++++++++++++++++++++++++++++++++++++ internal/parse.go | 14 ++ internal/parse_test.go | 176 +++++++++++++++++++++ internal/serversession.go | 50 ++++++ main.go | 75 +++++++++ 6 files changed, 679 insertions(+) create mode 100644 .gitignore create mode 100644 internal/log.go create mode 100644 internal/parse.go create mode 100644 internal/parse_test.go create mode 100644 internal/serversession.go create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fc2512e --- /dev/null +++ b/.gitignore @@ -0,0 +1,52 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +# Built binaries + +smtplogparser +*.exe +*.test + +# Windows image file caches +Thumbs.db +ehthumbs.db + +# Folder config file +Desktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msm +*.msp + +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist +*~ + +# Log files +*.log diff --git a/internal/log.go b/internal/log.go new file mode 100644 index 0000000..c2877ca --- /dev/null +++ b/internal/log.go @@ -0,0 +1,312 @@ +package internal + +import ( + "bufio" + "encoding/csv" + "io" + "log" + "regexp" + "strings" + "time" +) + +const ( + AnonymizedText = "**ANONYMIZED**" +) + +// Syntax of a message: +// DD-MM-YYYY hh:mm:ss LogLevel Service [\[ID\]] LogMessage[ (MessageID)][.] + +// Example failing mail: +// 19-03-2018 03:54:07 3 Fuzzy-Filter (40935FC192D) phase 1 (ex) 325ms result: Major=spam Minor=normal Fallback=spam Virus= + +var ( + // 05-03-2018 15:36:14 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Message delivered to: + rxSuccessfulMessage = regexp.MustCompile(`^Message\s+delivered\s+to:\s+<(.+)>\s*$`) + + rxMapMessageID = regexp.MustCompile(`^MessageID: .+\s*$`) + rxSubject = regexp.MustCompile(`^Testing: (.+)\s*$`) + rxSenderAddress = regexp.MustCompile(`(?i)^Recieve: MAIL\s+FROM:\s*<(.+)>\s*`) + rxRecipientAddress = regexp.MustCompile(`(?i)^Recieve: RCPT\s+TO:\s*<(.+)>\s*`) + + /* + 05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on MAIL FROM: = 250 2.1.0 Sender OK + 05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on RCPT TO: = 250 2.1.5 Recipient OK + */ + rxSendMailFrom = regexp.MustCompile(`^Receive on MAIL\s+FROM:\s*<(.+)>\s+`) + rxSendRcptTo = regexp.MustCompile(`^Receive on RCPT\s+TO:\s*<(.+)>\s+`) + + maxMessageAge = 30 * time.Second +) + +type Log struct { + // Maps sessions by SMTP server session ID + smtpServerSessions map[string]*SMTPServerSession + CSVWriter *csv.Writer + AnonymizeSubject bool +} + +func (this *Log) flushSingle(sessionId string) (err error) { + session := this.smtpServerSessions[sessionId] + delete(this.smtpServerSessions, sessionId) + if this.CSVWriter != nil { + if err := this.CSVWriter.Write(session.csv()); err != nil { + return err + } + } + return +} + +func (this *Log) flush(timeout time.Duration, timestamp time.Time) (err error) { +outerLoop: + for { + for id, session := range this.smtpServerSessions { + if timestamp.Sub(session.Timestamp) >= timeout { + err = this.flushSingle(id) + if err != nil { + return + } + continue outerLoop + } + } + break + } + return +} + +func (this *Log) Parse(r io.Reader) (err error) { + this.smtpServerSessions = map[string]*SMTPServerSession{} + + textreader := bufio.NewScanner(r) + textreader.Scan() + thisLine := textreader.Text() + nextLine := "" + var session *SMTPServerSession + + if this.CSVWriter != nil { + if err = this.CSVWriter.Write(sessionCSVHeader); err != nil { + return + } + } + + for { + hasNextLineAvailable := textreader.Scan() + + if hasNextLineAvailable { + nextLine = textreader.Text() + + // Try parsing first two fields of the upcoming line. If it doesn't + // work, assume leakage of previous line content. + fields := strings.Fields(nextLine) + if len(fields) < 2 { + // this is a continuation (too few fields) + thisLine = strings.TrimSpace(thisLine) + nextLine + continue + } + _, err = time.Parse("02-01-2006 15:04:05", strings.Join(fields[0:2], " ")) + if err != nil { + // this is a continuation (first two fields don't follow usual line syntax) + err = nil + thisLine = strings.TrimSpace(thisLine) + nextLine + continue + } + } + + if len(strings.TrimSpace(thisLine)) > 0 { + fields := strings.Fields(thisLine) + if len(fields) < 2 { + log.Print("WARNING: Found a line with less than 2 fields to work with. Skipping:", thisLine) + } else { + + timestamp, err := time.Parse("02-01-2006 15:04:05", strings.Join(fields[0:2], " ")) + if err != nil { + // first two fields don't follow usual line syntax + return err + } + + serviceName := fields[3] + sessionId := "" + messageId := "" + logMessage := "" + logMessageStartIndex := 4 + logMessageEndIndex := len(fields) + + // Is the session ID stored in the message? + // For this, fields[4] needs to start with [ and needs to contain an 8-char long ID. + if strings.HasPrefix(fields[4], "[") && + strings.HasSuffix(fields[4], "]") && + len(fields[4]) == 1+8+1 { + // Yup, there's an ID here! + sessionId = fields[4][1:9] + logMessageStartIndex = 5 + + } + + // Check if there is a message ID at the end of the log message + messageIdPart := strings.TrimRight(fields[len(fields)-1], ".") + if strings.HasPrefix(messageIdPart, "(") && + strings.HasSuffix(messageIdPart, ")") && + len(messageIdPart) == 1+11+1 { + // Yup, there's a message ID! + messageId = messageIdPart[1:12] + logMessageEndIndex-- + } else if strings.HasPrefix(fields[logMessageStartIndex], "(") && + strings.HasSuffix(fields[logMessageStartIndex], ")") && + len(fields[logMessageStartIndex]) == 1+11+1 { + // Message ID but in fields[logMessageStartIndex], so shift start of log message + messageId = fields[logMessageStartIndex][1:12] + logMessageStartIndex++ + } + + logMessage = strings.Join(fields[logMessageStartIndex:logMessageEndIndex], " ") + + //log.Println(serviceName, "-", sessionId, "-", messageId, "-", logMessage) + + switch serviceName { + case "SMTPClient": + // Could contain success message for delivery + //log.Println(">> parsing for delivery success") + if rxSuccessfulMessage.MatchString(logMessage) { + //log.Println(">> got delivery success!") + session = this.FindSMTPServerSessionByMessageId(messageId, timestamp) + if session == nil { + log.Println("WARNING: Found a delivery success message for a message we don't know:", logMessage) + break + } + session.IsSent = true + // immediately unregister + if err = this.flushSingle(session.SMTPServerSessionID); err != nil { + return err + } + } + + // Could contain sender + subm := rxSendMailFrom.FindStringSubmatch(logMessage) + if subm != nil { + session = this.FindSMTPServerSessionByMessageId(messageId, timestamp) + if session == nil { + log.Println("WARNING: Found a log line about the sender for a passed message we don't know:", logMessage) + break + } + session.SenderAddress = subm[1] + break + } + + // Could contain recipient + subm = rxSendRcptTo.FindStringSubmatch(logMessage) + if subm != nil { + session = this.FindSMTPServerSessionByMessageId(messageId, timestamp) + if session == nil { + log.Println("WARNING: Found a log line about the recipient for a passed message we don't know:", logMessage) + break + } + session.RecipientAddress = subm[1] + break + } + + case "SMTPServer": + if len(sessionId) > 0 { + session = this.createOrFetchSMTPServerSession(sessionId, timestamp) + } + + // Could contain recipient + subm := rxRecipientAddress.FindStringSubmatch(logMessage) + //log.Println(">> parsing for recipient") + if subm != nil { + //log.Println(">> got recipient") + // We got submatches, this has a recipient! + session.RecipientAddress = subm[1] + break + } + // Could contain sender + subm = rxSenderAddress.FindStringSubmatch(logMessage) + //log.Println(">> parsing for sender") + if subm != nil { + //log.Println(">> got sender") + // We got submatches, this has a sender! + session.SenderAddress = subm[1] + break + } + // Could have metadata about message ID + //log.Println(">> parsing for message ID map") + if len(messageId) > 0 { + //log.Println(">> got message ID map 1") + subm = rxMapMessageID.FindStringSubmatch(logMessage) + if subm != nil { + //log.Println(">> got message ID map 2") + // We got submatches, this lets us map session ID to message ID! + session.MessageID = messageId + //log.Println(messageId, "->", sessionId) + break + } + } + + case "SBL-Filter", "SWL-Filter": + // Could contain the subject + //log.Println(">> parsing for subject") + subm := rxSubject.FindStringSubmatch(logMessage) + if subm != nil { + //log.Println(">> got subject") + // We got submatches, this has a subject! + subject := subm[1] + session = this.FindSMTPServerSessionByMessageId(messageId, timestamp) + if session == nil { + log.Println("WARNING: Found a subject line for a message we don't know:", logMessage) + break + } + if this.AnonymizeSubject { + session.Subject = AnonymizedText + } else { + session.Subject = subject + } + } + } + + this.flush(maxMessageAge, timestamp) + } + } + + if !hasNextLineAvailable { + break + } + + thisLine = nextLine + } + + this.flush(0, time.Now()) + + if this.CSVWriter != nil { + this.CSVWriter.Flush() + } + + /*if err = textreader.Err(); err != nil { + return + }*/ + err = textreader.Err() + + return +} + +func (this *Log) createOrFetchSMTPServerSession(sessionId string, timestamp time.Time) (session *SMTPServerSession) { + if len(sessionId) <= 0 { + panic("Empty session ID") + } + session, ok := this.smtpServerSessions[sessionId] + if !ok { + session = &SMTPServerSession{ + SMTPServerSessionID: sessionId, + Timestamp: timestamp, + } + this.smtpServerSessions[sessionId] = session + } + return +} + +func (this *Log) FindSMTPServerSessionByMessageId(messageId string, timestamp time.Time) (session *SMTPServerSession) { + for _, currentSession := range this.smtpServerSessions { + if currentSession.IsMatchByMessageId(messageId, timestamp) { + return currentSession + } + } + return nil +} diff --git a/internal/parse.go b/internal/parse.go new file mode 100644 index 0000000..bc183e7 --- /dev/null +++ b/internal/parse.go @@ -0,0 +1,14 @@ +package internal + +import ( + //"errors" + "encoding/csv" + "io" +) + +func TransformLog(r io.Reader, w io.Writer) (err error) { + l := new(Log) + l.CSVWriter = csv.NewWriter(w) + err = l.Parse(r) + return +} diff --git a/internal/parse_test.go b/internal/parse_test.go new file mode 100644 index 0000000..6d31124 --- /dev/null +++ b/internal/parse_test.go @@ -0,0 +1,176 @@ +package internal + +import ( + "bytes" + "encoding/csv" + "io/ioutil" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +const ( + testMessageFailed = `19-03-2018 03:54:05 2 SMTPServer [F1AF6651] New connection from 80.249.239.48 +19-03-2018 03:54:05 3 SMTPServer Testing 80.249.239.48 on bl.spamcop.net +19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 220 mail.kunde.de SMTP server ready +19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Recieve: EHLO site.azauto.com.ua +19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250-OK +19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250-STARTTLS +19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250 SIZE 104857600 +19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Ehlo Greeting from: [80.249.239.48] - site.azauto.com.ua +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: MAIL FROM: +19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Mail from: info@byphil.comt +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 OK smtp ready for info@byphil.comt +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: RCPT TO: +19-03-2018 03:54:06 3 SMTPServer RVC-Filter testing: abteilung@kunde.de +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 OK smtp ready for abteilung@kunde.de +19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Mail to: accepted +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: DATA +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 354 Send message. End with CRLF.CRLF +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] MessageID: <00D6549C_3B615A6F_reddoxx@mail.kunde.de> (40935FC192D). +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] TransactionID: F84F9E0625A025C54F7DBD19AC13DEBF4C2736FB (40935FC192D). +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Saving message ... (40935FC192D) +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 message queued (40935FC192D) +19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Message queued (40935FC192D) +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: QUIT +19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 221 closing connection +19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Client disconnected. +19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Disconnected from 80.249.239.48 +19-03-2018 03:54:07 3 Validator [EB5FAD39] Thread started. Validating message (40935FC192D) +19-03-2018 03:54:07 3 Validator [EB5FAD39] Load message ... (40935FC192D) +19-03-2018 03:54:07 3 Validator [EB5FAD39] Load message finished. (40935FC192D) +19-03-2018 03:54:07 2 MailSealer [EB5FAD39] Message has no secure components. (40935FC192D) +19-03-2018 03:54:07 3 Validator [EB5FAD39] Starting validation of Message (40935FC192D) +19-03-2018 03:54:07 3 Validator [EB5FAD39] Using Profile: (1) Default-Filterprofile for +19-03-2018 03:54:07 3 DWL-Filter Testing (envelope): info@byphil.comt (40935FC192D) +19-03-2018 03:54:07 3 AWL-Filter Testing (envelope): info@byphil.comt (40935FC192D) +19-03-2018 03:54:07 3 AWL-Filter info@byphil.comt is *NOT* on sender address whitelist. 0ms (40935FC192D) +19-03-2018 03:54:07 3 SWL-Filter Testing: + Finanzdienstleiter (40935FC192D) +19-03-2018 03:54:07 3 RBL-Filter Testing 80.249.239.48 on bl.spamcop.net (40935FC192D) +19-03-2018 03:54:07 3 Advanced-RBL-Filter Testing (40935FC192D) +19-03-2018 03:54:07 3 Fuzzy-Filter Testing phase 1 (40935FC192D) +19-03-2018 03:54:07 3 Fuzzy-Filter (40935FC192D) phase 1 (ex) 325ms result: Major=spam Minor=normal Fallback=spam Virus= +19-03-2018 03:54:07 2 Fuzzy-Filter (40935FC192D) blocked by phase 1 +19-03-2018 03:54:07 2 Validator [EB5FAD39] Filter: 'Fuzzy-Filter' will prevent archiving. +19-03-2018 03:54:07 3 DBL-Filter Testing (envelope): info@byphil.comt (40935FC192D) +19-03-2018 03:54:07 3 ABL-Filter Testing (envelope): info@byphil.comt (40935FC192D) +19-03-2018 03:54:07 3 SBL-Filter Testing: Finanzdienstleiter (40935FC192D) +19-03-2018 03:54:07 3 VirusScanner No Virus found in message (40935FC192D) +19-03-2018 03:54:07 3 Validator [EB5FAD39] - abteilung@kunde.de validated. Result: 30 +19-03-2018 03:54:07 3 Validator [EB5FAD39] Thread terminated. +19-03-2018 03:54:07 3 Validator [EB5FAD39] Validation of Message (40935FC192D) finished.` + + testMessageSuccess = `05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: RCPT TO: +05-03-2018 15:36:13 3 SMTPServer RVC-Filter testing: person@kunde.de +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 250 OK smtp ready for person@kunde.de +05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Mail to: accepted +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: DATA +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 354 Send message. End with CRLF.CRLF +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] MessageID: <754a924065cc4c15ace114d9196e16ef@DE35S004EXC61.wp.corpintra.net> (4DBAB2C02A3). +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] TransactionID: 82CDA2B813955EB42DCDB70BDAED57F2A6C53C29 (4DBAB2C02A3). +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Saving message ... (4DBAB2C02A3) +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 250 message queued (4DBAB2C02A3) +05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Message queued (4DBAB2C02A3) +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: QUIT +05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 221 closing connection +05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Client disconnected. +05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Disconnected from 141.113.102.113 +05-03-2018 15:36:13 3 Validator [E8F016EA] Thread started. Validating message (4DBAB2C02A3) +05-03-2018 15:36:13 3 Validator [E8F016EA] Load message ... (4DBAB2C02A3) +05-03-2018 15:36:13 3 Validator [E8F016EA] Load message finished. (4DBAB2C02A3) +05-03-2018 15:36:13 2 MailSealer [E8F016EA] Message has no secure components. (4DBAB2C02A3) +05-03-2018 15:36:13 3 Validator [E8F016EA] Starting validation of Message (4DBAB2C02A3) +05-03-2018 15:36:13 3 Validator [E8F016EA] Using Profile: (1) Default-Filterprofile for +05-03-2018 15:36:13 3 DWL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3) +05-03-2018 15:36:13 3 AWL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3) +05-03-2018 15:36:13 3 AWL-Filter MaxMustermann@daimler.com is *NOT* on sender address whitelist. 0ms (4DBAB2C02A3) +05-03-2018 15:36:13 3 SWL-Filter Testing: AW: Urlaubsplanung (4DBAB2C02A3) +05-03-2018 15:36:13 3 RBL-Filter Testing 141.113.102.113 on bl.spamcop.net (4DBAB2C02A3) +05-03-2018 15:36:13 3 Advanced-RBL-Filter Testing (4DBAB2C02A3) +05-03-2018 15:36:13 3 Fuzzy-Filter Testing phase 1 (4DBAB2C02A3) +05-03-2018 15:36:13 3 Fuzzy-Filter (4DBAB2C02A3) phase 1 (ex) 161ms result: Major=clean Minor=normal Fallback=clean Virus= +05-03-2018 15:36:13 3 Fuzzy-Filter Testing phase 2 (4DBAB2C02A3) +05-03-2018 15:36:13 3 Fuzzy-Filter Testing 1d28528ffe972a11b3789fac292200b2 (4DBAB2C02A3) +05-03-2018 15:36:13 3 DBL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3) +05-03-2018 15:36:13 3 ABL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3) +05-03-2018 15:36:13 3 SBL-Filter Testing: AW: Urlaubsplanung (4DBAB2C02A3) +05-03-2018 15:36:13 3 VirusScanner No Virus found in message (4DBAB2C02A3) +05-03-2018 15:36:13 3 Validator [E8F016EA] - person@kunde.de validated. Result: 0 +05-03-2018 15:36:13 3 Validator Wait for MailDepot lock ... +05-03-2018 15:36:13 3 Validator MailDepot locked. +05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 start ... - (4DBAB2C02A3) +05-03-2018 15:36:13 2 Archive No policy matched. Message will be archived. +05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 before critical section ... - (4DBAB2C02A3) +05-03-2018 15:36:13 2 Archive Archiving for MailDepot 2.0 - (4DBAB2C02A3) +05-03-2018 15:36:13 3 Archive Add MailDepot 2.0 spooler entry: 82CDA2B813955EB42DCDB70BDAED57F2A6C53C29.rdxmt0 +05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 finished. - (4DBAB2C02A3) +05-03-2018 15:36:13 3 Validator MailDepot unlocked. +05-03-2018 15:36:13 3 Validator [E8F016EA] Thread terminated. +05-03-2018 15:36:13 3 Validator [E8F016EA] Validation of Message (4DBAB2C02A3) finished. +05-03-2018 15:36:13 3 SMTPClient [4041136960] Query queue +05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] Thread started. Sending message (4DBAB2C02A3) +05-03-2018 15:36:13 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Sending message for: +05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) connecting to: 10.1.0.18:25 +05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on HELO: 250 XSHADOW +05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on MAIL FROM: = 250 2.1.0 Sender OK +05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on RCPT TO: = 250 2.1.5 Recipient OK +05-03-2018 15:36:14 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Data result: 250 2.6.0 <754a924065cc4c15ace114d9196e16ef@DE35S004EXC61.wp.corpintra.net> [InternalId=1161195] Queued mail for delivery +05-03-2018 15:36:14 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Message delivered to: +05-03-2018 15:36:14 3 SMTPClient [E8F00BA5] Thread terminated.` +) + +func Test_Message_Successful(t *testing.T) { + buf := new(bytes.Buffer) + + transformer := new(Log) + transformer.CSVWriter = csv.NewWriter(buf) + err := transformer.Parse(strings.NewReader(testMessageSuccess)) + + require.NotEqual(t, buf.Len(), 0) + require.Nil(t, err) + require.Equal(t, strings.Join(sessionCSVHeader, ",")+ + "\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3", + strings.TrimSpace(buf.String())) +} + +func Test_Message_DupeSessionId(t *testing.T) { + buf := new(bytes.Buffer) + + transformer := new(Log) + transformer.CSVWriter = csv.NewWriter(buf) + err := transformer.Parse(strings.NewReader(testMessageSuccess + "\n" + testMessageSuccess)) + + require.NotEqual(t, buf.Len(), 0) + require.Nil(t, err) + require.Equal(t, strings.Join(sessionCSVHeader, ",")+ + "\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3"+ + "\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3", + strings.TrimSpace(buf.String())) +} + +func Test_Message_Failed(t *testing.T) { + buf := new(bytes.Buffer) + + transformer := new(Log) + transformer.CSVWriter = csv.NewWriter(buf) + err := transformer.Parse(strings.NewReader(testMessageFailed)) + + require.NotEqual(t, buf.Len(), 0) + require.Nil(t, err) + require.Equal(t, strings.Join(sessionCSVHeader, ",")+ + "\n2018-03-19,03:54:05,info@byphil.comt,abteilung@kunde.de,Finanzdienstleiter,false,F1AF6651,40935FC192D", + strings.TrimSpace(buf.String())) +} + +func Benchmark_Messages(b *testing.B) { + output := ioutil.Discard + + transformer := new(Log) + transformer.CSVWriter = csv.NewWriter(output) + + for n := 0; n < b.N; n++ { + transformer.Parse(strings.NewReader(testMessageSuccess)) + } +} diff --git a/internal/serversession.go b/internal/serversession.go new file mode 100644 index 0000000..08a1641 --- /dev/null +++ b/internal/serversession.go @@ -0,0 +1,50 @@ +package internal + +import ( + "fmt" + "strings" + "time" +) + +var ( + sessionCSVHeader = []string{ + "Date", + "Time", + "From", + "To", + "Subject", + "Passed", + "SMTPServer session ID", + "Message short ID", + } +) + +type SMTPServerSession struct { + SMTPServerSessionID string + MessageID string + RecipientAddress string + SenderAddress string + Subject string + IsSent bool + Timestamp time.Time +} + +func (sss *SMTPServerSession) csv() []string { + return []string{ + sss.Timestamp.Format("2006-01-02"), + sss.Timestamp.Format("15:04:05"), + sss.SenderAddress, + sss.RecipientAddress, + sss.Subject, + fmt.Sprintf("%v", sss.IsSent), + sss.SMTPServerSessionID, + sss.MessageID, + } +} + +func (sss *SMTPServerSession) IsMatchByMessageId(messageId string, timestamp time.Time) (result bool) { + result = strings.EqualFold(messageId, sss.MessageID) && + timestamp.Sub(sss.Timestamp) < maxMessageAge + return + +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..279329d --- /dev/null +++ b/main.go @@ -0,0 +1,75 @@ +package main + +import ( + "encoding/csv" + "io" + "log" + "os" + "path/filepath" + "strings" + + "github.com/alecthomas/kingpin" + + "local/smtpparse/internal" +) + +var ( + flagInputFiles = kingpin.Flag("input-file", "Input files. Can be glob patterns like `*.log` and `-` for standard input pipe.").Short('i').Required().Strings() + flagOutputFile = kingpin.Flag("output-file", "Output file. Can be `-` for standard output pipe.").Default("-").Short('o').String() + flagRecipients = kingpin.Flag("recipients", "List of recipients to find entries for. Defaults to everyone.").Short('r').Strings() + flagAnonymize = kingpin.Flag("anonymize", "Anonymizes mail subjects.").Bool() +) + +func main() { + kingpin.Parse() + + // Open input file + readers := []io.Reader{} + for _, globPattern := range *flagInputFiles { + // allow stdin + if globPattern == "-" { + readers = append(readers, os.Stdin, strings.NewReader("\n")) + continue + } + + // check for files of pattern + matches, err := filepath.Glob(globPattern) + if err != nil { + log.Fatal(err) + } + + for _, match := range matches { + f, err := os.Open(match) + if err != nil { + log.Fatal(err) + } + defer f.Close() + + readers = append(readers, f, strings.NewReader("\n")) + } + } + r := io.MultiReader(readers...) + + // Create output + var outputWriter io.Writer + if *flagOutputFile == "-" { + outputWriter = os.Stdout + } else { + f, err := os.Create(*flagOutputFile) + if err != nil { + log.Fatal(err) + } + defer f.Close() + outputWriter = f + } + + // Parse + transformer := new(internal.Log) + transformer.AnonymizeSubject = *flagAnonymize + transformer.CSVWriter = csv.NewWriter(outputWriter) + err := transformer.Parse(r) + if err != nil { + log.Fatal(err) + } + +}