Initial commit.
commit
fad5d15987
GPG Key ID:
1573F6D8EFE4D0CF
|
|
@ -0,0 +1,52 @@
|
||||||
|
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
||||||
|
*.o
|
||||||
|
*.a
|
||||||
|
*.so
|
||||||
|
|
||||||
|
# Folders
|
||||||
|
_obj
|
||||||
|
_test
|
||||||
|
|
||||||
|
# Architecture specific extensions/prefixes
|
||||||
|
*.[568vq]
|
||||||
|
[568vq].out
|
||||||
|
|
||||||
|
*.cgo1.go
|
||||||
|
*.cgo2.c
|
||||||
|
_cgo_defun.c
|
||||||
|
_cgo_gotypes.go
|
||||||
|
_cgo_export.*
|
||||||
|
|
||||||
|
_testmain.go
|
||||||
|
|
||||||
|
# Built binaries
|
||||||
|
|
||||||
|
smtplogparser
|
||||||
|
*.exe
|
||||||
|
*.test
|
||||||
|
|
||||||
|
# Windows image file caches
|
||||||
|
Thumbs.db
|
||||||
|
ehthumbs.db
|
||||||
|
|
||||||
|
# Folder config file
|
||||||
|
Desktop.ini
|
||||||
|
|
||||||
|
# Recycle Bin used on file shares
|
||||||
|
$RECYCLE.BIN/
|
||||||
|
|
||||||
|
# Windows Installer files
|
||||||
|
*.cab
|
||||||
|
*.msi
|
||||||
|
*.msm
|
||||||
|
*.msp
|
||||||
|
|
||||||
|
[._]*.s[a-w][a-z]
|
||||||
|
[._]s[a-w][a-z]
|
||||||
|
*.un~
|
||||||
|
Session.vim
|
||||||
|
.netrwhist
|
||||||
|
*~
|
||||||
|
|
||||||
|
# Log files
|
||||||
|
*.log
|
||||||
|
|
@ -0,0 +1,312 @@
|
||||||
|
package internal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/csv"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
AnonymizedText = "**ANONYMIZED**"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Syntax of a message:
|
||||||
|
// DD-MM-YYYY hh:mm:ss LogLevel Service [\[ID\]] LogMessage[ (MessageID)][.]
|
||||||
|
|
||||||
|
// Example failing mail:
|
||||||
|
// 19-03-2018 03:54:07 3 Fuzzy-Filter (40935FC192D) phase 1 (ex) 325ms result: Major=spam Minor=normal Fallback=spam Virus=
|
||||||
|
|
||||||
|
var (
|
||||||
|
// 05-03-2018 15:36:14 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Message delivered to: <person@kunde.de>
|
||||||
|
rxSuccessfulMessage = regexp.MustCompile(`^Message\s+delivered\s+to:\s+<(.+)>\s*$`)
|
||||||
|
|
||||||
|
rxMapMessageID = regexp.MustCompile(`^MessageID: .+\s*$`)
|
||||||
|
rxSubject = regexp.MustCompile(`^Testing: (.+)\s*$`)
|
||||||
|
rxSenderAddress = regexp.MustCompile(`(?i)^Recieve: MAIL\s+FROM:\s*<(.+)>\s*`)
|
||||||
|
rxRecipientAddress = regexp.MustCompile(`(?i)^Recieve: RCPT\s+TO:\s*<(.+)>\s*`)
|
||||||
|
|
||||||
|
/*
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on MAIL FROM: <MaxMustermann@daimler.com> = 250 2.1.0 Sender OK
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on RCPT TO: <person@kunde.de> = 250 2.1.5 Recipient OK
|
||||||
|
*/
|
||||||
|
rxSendMailFrom = regexp.MustCompile(`^Receive on MAIL\s+FROM:\s*<(.+)>\s+`)
|
||||||
|
rxSendRcptTo = regexp.MustCompile(`^Receive on RCPT\s+TO:\s*<(.+)>\s+`)
|
||||||
|
|
||||||
|
maxMessageAge = 30 * time.Second
|
||||||
|
)
|
||||||
|
|
||||||
|
type Log struct {
|
||||||
|
// Maps sessions by SMTP server session ID
|
||||||
|
smtpServerSessions map[string]*SMTPServerSession
|
||||||
|
CSVWriter *csv.Writer
|
||||||
|
AnonymizeSubject bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *Log) flushSingle(sessionId string) (err error) {
|
||||||
|
session := this.smtpServerSessions[sessionId]
|
||||||
|
delete(this.smtpServerSessions, sessionId)
|
||||||
|
if this.CSVWriter != nil {
|
||||||
|
if err := this.CSVWriter.Write(session.csv()); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *Log) flush(timeout time.Duration, timestamp time.Time) (err error) {
|
||||||
|
outerLoop:
|
||||||
|
for {
|
||||||
|
for id, session := range this.smtpServerSessions {
|
||||||
|
if timestamp.Sub(session.Timestamp) >= timeout {
|
||||||
|
err = this.flushSingle(id)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
continue outerLoop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *Log) Parse(r io.Reader) (err error) {
|
||||||
|
this.smtpServerSessions = map[string]*SMTPServerSession{}
|
||||||
|
|
||||||
|
textreader := bufio.NewScanner(r)
|
||||||
|
textreader.Scan()
|
||||||
|
thisLine := textreader.Text()
|
||||||
|
nextLine := ""
|
||||||
|
var session *SMTPServerSession
|
||||||
|
|
||||||
|
if this.CSVWriter != nil {
|
||||||
|
if err = this.CSVWriter.Write(sessionCSVHeader); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for {
|
||||||
|
hasNextLineAvailable := textreader.Scan()
|
||||||
|
|
||||||
|
if hasNextLineAvailable {
|
||||||
|
nextLine = textreader.Text()
|
||||||
|
|
||||||
|
// Try parsing first two fields of the upcoming line. If it doesn't
|
||||||
|
// work, assume leakage of previous line content.
|
||||||
|
fields := strings.Fields(nextLine)
|
||||||
|
if len(fields) < 2 {
|
||||||
|
// this is a continuation (too few fields)
|
||||||
|
thisLine = strings.TrimSpace(thisLine) + nextLine
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
_, err = time.Parse("02-01-2006 15:04:05", strings.Join(fields[0:2], " "))
|
||||||
|
if err != nil {
|
||||||
|
// this is a continuation (first two fields don't follow usual line syntax)
|
||||||
|
err = nil
|
||||||
|
thisLine = strings.TrimSpace(thisLine) + nextLine
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(strings.TrimSpace(thisLine)) > 0 {
|
||||||
|
fields := strings.Fields(thisLine)
|
||||||
|
if len(fields) < 2 {
|
||||||
|
log.Print("WARNING: Found a line with less than 2 fields to work with. Skipping:", thisLine)
|
||||||
|
} else {
|
||||||
|
|
||||||
|
timestamp, err := time.Parse("02-01-2006 15:04:05", strings.Join(fields[0:2], " "))
|
||||||
|
if err != nil {
|
||||||
|
// first two fields don't follow usual line syntax
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
serviceName := fields[3]
|
||||||
|
sessionId := ""
|
||||||
|
messageId := ""
|
||||||
|
logMessage := ""
|
||||||
|
logMessageStartIndex := 4
|
||||||
|
logMessageEndIndex := len(fields)
|
||||||
|
|
||||||
|
// Is the session ID stored in the message?
|
||||||
|
// For this, fields[4] needs to start with [ and needs to contain an 8-char long ID.
|
||||||
|
if strings.HasPrefix(fields[4], "[") &&
|
||||||
|
strings.HasSuffix(fields[4], "]") &&
|
||||||
|
len(fields[4]) == 1+8+1 {
|
||||||
|
// Yup, there's an ID here!
|
||||||
|
sessionId = fields[4][1:9]
|
||||||
|
logMessageStartIndex = 5
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check if there is a message ID at the end of the log message
|
||||||
|
messageIdPart := strings.TrimRight(fields[len(fields)-1], ".")
|
||||||
|
if strings.HasPrefix(messageIdPart, "(") &&
|
||||||
|
strings.HasSuffix(messageIdPart, ")") &&
|
||||||
|
len(messageIdPart) == 1+11+1 {
|
||||||
|
// Yup, there's a message ID!
|
||||||
|
messageId = messageIdPart[1:12]
|
||||||
|
logMessageEndIndex--
|
||||||
|
} else if strings.HasPrefix(fields[logMessageStartIndex], "(") &&
|
||||||
|
strings.HasSuffix(fields[logMessageStartIndex], ")") &&
|
||||||
|
len(fields[logMessageStartIndex]) == 1+11+1 {
|
||||||
|
// Message ID but in fields[logMessageStartIndex], so shift start of log message
|
||||||
|
messageId = fields[logMessageStartIndex][1:12]
|
||||||
|
logMessageStartIndex++
|
||||||
|
}
|
||||||
|
|
||||||
|
logMessage = strings.Join(fields[logMessageStartIndex:logMessageEndIndex], " ")
|
||||||
|
|
||||||
|
//log.Println(serviceName, "-", sessionId, "-", messageId, "-", logMessage)
|
||||||
|
|
||||||
|
switch serviceName {
|
||||||
|
case "SMTPClient":
|
||||||
|
// Could contain success message for delivery
|
||||||
|
//log.Println(">> parsing for delivery success")
|
||||||
|
if rxSuccessfulMessage.MatchString(logMessage) {
|
||||||
|
//log.Println(">> got delivery success!")
|
||||||
|
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||||
|
if session == nil {
|
||||||
|
log.Println("WARNING: Found a delivery success message for a message we don't know:", logMessage)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
session.IsSent = true
|
||||||
|
// immediately unregister
|
||||||
|
if err = this.flushSingle(session.SMTPServerSessionID); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Could contain sender
|
||||||
|
subm := rxSendMailFrom.FindStringSubmatch(logMessage)
|
||||||
|
if subm != nil {
|
||||||
|
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||||
|
if session == nil {
|
||||||
|
log.Println("WARNING: Found a log line about the sender for a passed message we don't know:", logMessage)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
session.SenderAddress = subm[1]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
// Could contain recipient
|
||||||
|
subm = rxSendRcptTo.FindStringSubmatch(logMessage)
|
||||||
|
if subm != nil {
|
||||||
|
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||||
|
if session == nil {
|
||||||
|
log.Println("WARNING: Found a log line about the recipient for a passed message we don't know:", logMessage)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
session.RecipientAddress = subm[1]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
case "SMTPServer":
|
||||||
|
if len(sessionId) > 0 {
|
||||||
|
session = this.createOrFetchSMTPServerSession(sessionId, timestamp)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Could contain recipient
|
||||||
|
subm := rxRecipientAddress.FindStringSubmatch(logMessage)
|
||||||
|
//log.Println(">> parsing for recipient")
|
||||||
|
if subm != nil {
|
||||||
|
//log.Println(">> got recipient")
|
||||||
|
// We got submatches, this has a recipient!
|
||||||
|
session.RecipientAddress = subm[1]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// Could contain sender
|
||||||
|
subm = rxSenderAddress.FindStringSubmatch(logMessage)
|
||||||
|
//log.Println(">> parsing for sender")
|
||||||
|
if subm != nil {
|
||||||
|
//log.Println(">> got sender")
|
||||||
|
// We got submatches, this has a sender!
|
||||||
|
session.SenderAddress = subm[1]
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// Could have metadata about message ID
|
||||||
|
//log.Println(">> parsing for message ID map")
|
||||||
|
if len(messageId) > 0 {
|
||||||
|
//log.Println(">> got message ID map 1")
|
||||||
|
subm = rxMapMessageID.FindStringSubmatch(logMessage)
|
||||||
|
if subm != nil {
|
||||||
|
//log.Println(">> got message ID map 2")
|
||||||
|
// We got submatches, this lets us map session ID to message ID!
|
||||||
|
session.MessageID = messageId
|
||||||
|
//log.Println(messageId, "->", sessionId)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case "SBL-Filter", "SWL-Filter":
|
||||||
|
// Could contain the subject
|
||||||
|
//log.Println(">> parsing for subject")
|
||||||
|
subm := rxSubject.FindStringSubmatch(logMessage)
|
||||||
|
if subm != nil {
|
||||||
|
//log.Println(">> got subject")
|
||||||
|
// We got submatches, this has a subject!
|
||||||
|
subject := subm[1]
|
||||||
|
session = this.FindSMTPServerSessionByMessageId(messageId, timestamp)
|
||||||
|
if session == nil {
|
||||||
|
log.Println("WARNING: Found a subject line for a message we don't know:", logMessage)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if this.AnonymizeSubject {
|
||||||
|
session.Subject = AnonymizedText
|
||||||
|
} else {
|
||||||
|
session.Subject = subject
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.flush(maxMessageAge, timestamp)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if !hasNextLineAvailable {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
|
||||||
|
thisLine = nextLine
|
||||||
|
}
|
||||||
|
|
||||||
|
this.flush(0, time.Now())
|
||||||
|
|
||||||
|
if this.CSVWriter != nil {
|
||||||
|
this.CSVWriter.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
/*if err = textreader.Err(); err != nil {
|
||||||
|
return
|
||||||
|
}*/
|
||||||
|
err = textreader.Err()
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *Log) createOrFetchSMTPServerSession(sessionId string, timestamp time.Time) (session *SMTPServerSession) {
|
||||||
|
if len(sessionId) <= 0 {
|
||||||
|
panic("Empty session ID")
|
||||||
|
}
|
||||||
|
session, ok := this.smtpServerSessions[sessionId]
|
||||||
|
if !ok {
|
||||||
|
session = &SMTPServerSession{
|
||||||
|
SMTPServerSessionID: sessionId,
|
||||||
|
Timestamp: timestamp,
|
||||||
|
}
|
||||||
|
this.smtpServerSessions[sessionId] = session
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *Log) FindSMTPServerSessionByMessageId(messageId string, timestamp time.Time) (session *SMTPServerSession) {
|
||||||
|
for _, currentSession := range this.smtpServerSessions {
|
||||||
|
if currentSession.IsMatchByMessageId(messageId, timestamp) {
|
||||||
|
return currentSession
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,14 @@
|
||||||
|
package internal
|
||||||
|
|
||||||
|
import (
|
||||||
|
//"errors"
|
||||||
|
"encoding/csv"
|
||||||
|
"io"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TransformLog(r io.Reader, w io.Writer) (err error) {
|
||||||
|
l := new(Log)
|
||||||
|
l.CSVWriter = csv.NewWriter(w)
|
||||||
|
err = l.Parse(r)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,176 @@
|
||||||
|
package internal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/csv"
|
||||||
|
"io/ioutil"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
testMessageFailed = `19-03-2018 03:54:05 2 SMTPServer [F1AF6651] New connection from 80.249.239.48
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer Testing 80.249.239.48 on bl.spamcop.net
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 220 mail.kunde.de SMTP server ready
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Recieve: EHLO site.azauto.com.ua
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250-OK
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250-STARTTLS
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Send: 250 SIZE 104857600
|
||||||
|
19-03-2018 03:54:05 3 SMTPServer [F1AF6651] Ehlo Greeting from: [80.249.239.48] - site.azauto.com.ua
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: MAIL FROM:<info@byphil.comt>
|
||||||
|
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Mail from: info@byphil.comt
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 OK smtp ready for info@byphil.comt
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: RCPT TO:<abteilung@kunde.de>
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer RVC-Filter testing: abteilung@kunde.de
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 OK smtp ready for abteilung@kunde.de
|
||||||
|
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Mail to: <abteilung@kunde.de> accepted
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: DATA
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 354 Send message. End with CRLF.CRLF
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] MessageID: <00D6549C_3B615A6F_reddoxx@mail.kunde.de> (40935FC192D).
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] TransactionID: F84F9E0625A025C54F7DBD19AC13DEBF4C2736FB (40935FC192D).
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Saving message ... (40935FC192D)
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 250 message queued (40935FC192D)
|
||||||
|
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Message queued (40935FC192D)
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Recieve: QUIT
|
||||||
|
19-03-2018 03:54:06 3 SMTPServer [F1AF6651] Send: 221 closing connection
|
||||||
|
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Client disconnected.
|
||||||
|
19-03-2018 03:54:06 2 SMTPServer [F1AF6651] Disconnected from 80.249.239.48
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Thread started. Validating message (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Load message ... (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Load message finished. (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 2 MailSealer [EB5FAD39] Message has no secure components. (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Starting validation of Message (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Using Profile: (1) Default-Filterprofile for <abteilung@kunde.de>
|
||||||
|
19-03-2018 03:54:07 3 DWL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 AWL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 AWL-Filter info@byphil.comt is *NOT* on sender address whitelist. 0ms (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 SWL-Filter Testing:
|
||||||
|
Finanzdienstleiter (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 RBL-Filter Testing 80.249.239.48 on bl.spamcop.net (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Advanced-RBL-Filter Testing (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Fuzzy-Filter Testing phase 1 (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Fuzzy-Filter (40935FC192D) phase 1 (ex) 325ms result: Major=spam Minor=normal Fallback=spam Virus=
|
||||||
|
19-03-2018 03:54:07 2 Fuzzy-Filter (40935FC192D) blocked by phase 1
|
||||||
|
19-03-2018 03:54:07 2 Validator [EB5FAD39] Filter: 'Fuzzy-Filter' will prevent archiving.
|
||||||
|
19-03-2018 03:54:07 3 DBL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 ABL-Filter Testing (envelope): info@byphil.comt (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 SBL-Filter Testing: Finanzdienstleiter (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 VirusScanner No Virus found in message (40935FC192D)
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] - abteilung@kunde.de validated. Result: 30
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Thread terminated.
|
||||||
|
19-03-2018 03:54:07 3 Validator [EB5FAD39] Validation of Message (40935FC192D) finished.`
|
||||||
|
|
||||||
|
testMessageSuccess = `05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: RCPT TO:<person@kunde.de>
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer RVC-Filter testing: person@kunde.de
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 250 OK smtp ready for person@kunde.de
|
||||||
|
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Mail to: <person@kunde.de> accepted
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: DATA
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 354 Send message. End with CRLF.CRLF
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] MessageID: <754a924065cc4c15ace114d9196e16ef@DE35S004EXC61.wp.corpintra.net> (4DBAB2C02A3).
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] TransactionID: 82CDA2B813955EB42DCDB70BDAED57F2A6C53C29 (4DBAB2C02A3).
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Saving message ... (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 250 message queued (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Message queued (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Recieve: QUIT
|
||||||
|
05-03-2018 15:36:13 3 SMTPServer [EEA1EB23] Send: 221 closing connection
|
||||||
|
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Client disconnected.
|
||||||
|
05-03-2018 15:36:13 2 SMTPServer [EEA1EB23] Disconnected from 141.113.102.113
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Thread started. Validating message (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Load message ... (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Load message finished. (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 2 MailSealer [E8F016EA] Message has no secure components. (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Starting validation of Message (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Using Profile: (1) Default-Filterprofile for <person@kunde.de>
|
||||||
|
05-03-2018 15:36:13 3 DWL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 AWL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 AWL-Filter MaxMustermann@daimler.com is *NOT* on sender address whitelist. 0ms (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 SWL-Filter Testing: AW: Urlaubsplanung (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 RBL-Filter Testing 141.113.102.113 on bl.spamcop.net (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Advanced-RBL-Filter Testing (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Fuzzy-Filter Testing phase 1 (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Fuzzy-Filter (4DBAB2C02A3) phase 1 (ex) 161ms result: Major=clean Minor=normal Fallback=clean Virus=
|
||||||
|
05-03-2018 15:36:13 3 Fuzzy-Filter Testing phase 2 (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Fuzzy-Filter Testing 1d28528ffe972a11b3789fac292200b2 (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 DBL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 ABL-Filter Testing (envelope): MaxMustermann@daimler.com (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 SBL-Filter Testing: AW: Urlaubsplanung (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 VirusScanner No Virus found in message (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] - person@kunde.de validated. Result: 0
|
||||||
|
05-03-2018 15:36:13 3 Validator Wait for MailDepot lock ...
|
||||||
|
05-03-2018 15:36:13 3 Validator MailDepot locked.
|
||||||
|
05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 start ... - (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 2 Archive No policy matched. Message will be archived.
|
||||||
|
05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 before critical section ... - (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 2 Archive Archiving for MailDepot 2.0 - (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Archive Add MailDepot 2.0 spooler entry: 82CDA2B813955EB42DCDB70BDAED57F2A6C53C29.rdxmt0
|
||||||
|
05-03-2018 15:36:13 3 Archive ArchiveMessage 2.0 finished. - (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 3 Validator MailDepot unlocked.
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Thread terminated.
|
||||||
|
05-03-2018 15:36:13 3 Validator [E8F016EA] Validation of Message (4DBAB2C02A3) finished.
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [4041136960] Query queue
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] Thread started. Sending message (4DBAB2C02A3)
|
||||||
|
05-03-2018 15:36:13 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Sending message for: <MaxMustermann@daimler.com>
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) connecting to: 10.1.0.18:25
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on HELO: 250 XSHADOW
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on MAIL FROM: <MaxMustermann@daimler.com> = 250 2.1.0 Sender OK
|
||||||
|
05-03-2018 15:36:13 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Receive on RCPT TO: <person@kunde.de> = 250 2.1.5 Recipient OK
|
||||||
|
05-03-2018 15:36:14 3 SMTPClient [E8F00BA5] (4DBAB2C02A3) Data result: 250 2.6.0 <754a924065cc4c15ace114d9196e16ef@DE35S004EXC61.wp.corpintra.net> [InternalId=1161195] Queued mail for delivery
|
||||||
|
05-03-2018 15:36:14 2 SMTPClient [E8F00BA5] (4DBAB2C02A3) Message delivered to: <person@kunde.de>
|
||||||
|
05-03-2018 15:36:14 3 SMTPClient [E8F00BA5] Thread terminated.`
|
||||||
|
)
|
||||||
|
|
||||||
|
func Test_Message_Successful(t *testing.T) {
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
|
||||||
|
transformer := new(Log)
|
||||||
|
transformer.CSVWriter = csv.NewWriter(buf)
|
||||||
|
err := transformer.Parse(strings.NewReader(testMessageSuccess))
|
||||||
|
|
||||||
|
require.NotEqual(t, buf.Len(), 0)
|
||||||
|
require.Nil(t, err)
|
||||||
|
require.Equal(t, strings.Join(sessionCSVHeader, ",")+
|
||||||
|
"\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3",
|
||||||
|
strings.TrimSpace(buf.String()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_Message_DupeSessionId(t *testing.T) {
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
|
||||||
|
transformer := new(Log)
|
||||||
|
transformer.CSVWriter = csv.NewWriter(buf)
|
||||||
|
err := transformer.Parse(strings.NewReader(testMessageSuccess + "\n" + testMessageSuccess))
|
||||||
|
|
||||||
|
require.NotEqual(t, buf.Len(), 0)
|
||||||
|
require.Nil(t, err)
|
||||||
|
require.Equal(t, strings.Join(sessionCSVHeader, ",")+
|
||||||
|
"\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3"+
|
||||||
|
"\n2018-03-05,15:36:13,MaxMustermann@daimler.com,person@kunde.de,AW: Urlaubsplanung,true,EEA1EB23,4DBAB2C02A3",
|
||||||
|
strings.TrimSpace(buf.String()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func Test_Message_Failed(t *testing.T) {
|
||||||
|
buf := new(bytes.Buffer)
|
||||||
|
|
||||||
|
transformer := new(Log)
|
||||||
|
transformer.CSVWriter = csv.NewWriter(buf)
|
||||||
|
err := transformer.Parse(strings.NewReader(testMessageFailed))
|
||||||
|
|
||||||
|
require.NotEqual(t, buf.Len(), 0)
|
||||||
|
require.Nil(t, err)
|
||||||
|
require.Equal(t, strings.Join(sessionCSVHeader, ",")+
|
||||||
|
"\n2018-03-19,03:54:05,info@byphil.comt,abteilung@kunde.de,Finanzdienstleiter,false,F1AF6651,40935FC192D",
|
||||||
|
strings.TrimSpace(buf.String()))
|
||||||
|
}
|
||||||
|
|
||||||
|
func Benchmark_Messages(b *testing.B) {
|
||||||
|
output := ioutil.Discard
|
||||||
|
|
||||||
|
transformer := new(Log)
|
||||||
|
transformer.CSVWriter = csv.NewWriter(output)
|
||||||
|
|
||||||
|
for n := 0; n < b.N; n++ {
|
||||||
|
transformer.Parse(strings.NewReader(testMessageSuccess))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,50 @@
|
||||||
|
package internal
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
sessionCSVHeader = []string{
|
||||||
|
"Date",
|
||||||
|
"Time",
|
||||||
|
"From",
|
||||||
|
"To",
|
||||||
|
"Subject",
|
||||||
|
"Passed",
|
||||||
|
"SMTPServer session ID",
|
||||||
|
"Message short ID",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
type SMTPServerSession struct {
|
||||||
|
SMTPServerSessionID string
|
||||||
|
MessageID string
|
||||||
|
RecipientAddress string
|
||||||
|
SenderAddress string
|
||||||
|
Subject string
|
||||||
|
IsSent bool
|
||||||
|
Timestamp time.Time
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sss *SMTPServerSession) csv() []string {
|
||||||
|
return []string{
|
||||||
|
sss.Timestamp.Format("2006-01-02"),
|
||||||
|
sss.Timestamp.Format("15:04:05"),
|
||||||
|
sss.SenderAddress,
|
||||||
|
sss.RecipientAddress,
|
||||||
|
sss.Subject,
|
||||||
|
fmt.Sprintf("%v", sss.IsSent),
|
||||||
|
sss.SMTPServerSessionID,
|
||||||
|
sss.MessageID,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (sss *SMTPServerSession) IsMatchByMessageId(messageId string, timestamp time.Time) (result bool) {
|
||||||
|
result = strings.EqualFold(messageId, sss.MessageID) &&
|
||||||
|
timestamp.Sub(sss.Timestamp) < maxMessageAge
|
||||||
|
return
|
||||||
|
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,75 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/csv"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/alecthomas/kingpin"
|
||||||
|
|
||||||
|
"local/smtpparse/internal"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
flagInputFiles = kingpin.Flag("input-file", "Input files. Can be glob patterns like `*.log` and `-` for standard input pipe.").Short('i').Required().Strings()
|
||||||
|
flagOutputFile = kingpin.Flag("output-file", "Output file. Can be `-` for standard output pipe.").Default("-").Short('o').String()
|
||||||
|
flagRecipients = kingpin.Flag("recipients", "List of recipients to find entries for. Defaults to everyone.").Short('r').Strings()
|
||||||
|
flagAnonymize = kingpin.Flag("anonymize", "Anonymizes mail subjects.").Bool()
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
kingpin.Parse()
|
||||||
|
|
||||||
|
// Open input file
|
||||||
|
readers := []io.Reader{}
|
||||||
|
for _, globPattern := range *flagInputFiles {
|
||||||
|
// allow stdin
|
||||||
|
if globPattern == "-" {
|
||||||
|
readers = append(readers, os.Stdin, strings.NewReader("\n"))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// check for files of pattern
|
||||||
|
matches, err := filepath.Glob(globPattern)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, match := range matches {
|
||||||
|
f, err := os.Open(match)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
readers = append(readers, f, strings.NewReader("\n"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
r := io.MultiReader(readers...)
|
||||||
|
|
||||||
|
// Create output
|
||||||
|
var outputWriter io.Writer
|
||||||
|
if *flagOutputFile == "-" {
|
||||||
|
outputWriter = os.Stdout
|
||||||
|
} else {
|
||||||
|
f, err := os.Create(*flagOutputFile)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
outputWriter = f
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse
|
||||||
|
transformer := new(internal.Log)
|
||||||
|
transformer.AnonymizeSubject = *flagAnonymize
|
||||||
|
transformer.CSVWriter = csv.NewWriter(outputWriter)
|
||||||
|
err := transformer.Parse(r)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue