diff --git a/main.go b/main.go index 8f5b265..bcec298 100644 --- a/main.go +++ b/main.go @@ -31,6 +31,8 @@ func main() { var soundcloudClientId string var soundcloudClientSecret string + var webEnableImages bool + var debug bool var noInvite bool var useTLS bool @@ -59,9 +61,14 @@ func main() { // Youtube config kingpin.Flag("youtube-key", "The API key to use to access the YouTube API.").StringVar(&youtubeApiKey) + + // SoundCloud config kingpin.Flag("soundcloud-id", "The SoundCloud ID.").StringVar(&soundcloudClientId) kingpin.Flag("soundcloud-secret", "The SoundCloud secret.").StringVar(&soundcloudClientSecret) + // Web parser config + kingpin.Flag("images", "Enables parsing links of images. Disabled by default for legal reasons.").BoolVar(&webEnableImages) + kingpin.Parse() if len(nickname) == 0 { @@ -75,25 +82,36 @@ func main() { m := manager.NewManager() // Load youtube parser - youtubeParser := &youtube.Parser{ - Config: &youtube.Config{ApiKey: youtubeApiKey}, + if len(youtubeApiKey) > 0 { + youtubeParser := &youtube.Parser{ + Config: &youtube.Config{ApiKey: youtubeApiKey}, + } + must(m.RegisterParser(youtubeParser)) + } else { + log.Println("No YouTube API key provided, YouTube parsing via API is disabled.") } - must(m.RegisterParser(youtubeParser)) // Load soundcloud parser - soundcloudParser := &soundcloud.Parser{ - Config: &soundcloud.Config{ - ClientId: soundcloudClientId, - ClientSecret: soundcloudClientSecret, - }, + if len(soundcloudClientId) > 0 && len(soundcloudClientSecret) > 0 { + soundcloudParser := &soundcloud.Parser{ + Config: &soundcloud.Config{ + ClientId: soundcloudClientId, + ClientSecret: soundcloudClientSecret, + }, + } + must(m.RegisterParser(soundcloudParser)) + } else { + log.Println("No SoundCloud client ID or secret provided, SoundCloud parsing via API is disabled.") } - must(m.RegisterParser(soundcloudParser)) // Load wikipedia parser must(m.RegisterParser(new(wikipedia.Parser))) // Load web parser - must(m.RegisterParser(new(web.Parser))) + webParser := &web.Parser{ + EnableImages: webEnableImages, + } + must(m.RegisterParser(webParser)) // IRC conn := m.AntifloodIrcConn(irc.IRC(nickname, ident)) @@ -130,6 +148,8 @@ func main() { conn.AddCallback("JOIN", func(e *irc.Event) { // Is this JOIN not about us? if !strings.EqualFold(e.Nick, conn.GetNick()) { + // Save this user's details for a temporary ignore + m.NotifyUserJoined(e.Arguments[0], e.Source) return } @@ -199,6 +219,12 @@ func main() { log.Printf("<%s @ %s> %s", event.Nick, target, msg) + // Ignore user if they just joined + if shouldIgnore := m.TrackUser(target, event.Source); shouldIgnore { + log.Print("This message will be ignored since the user just joined.") + return + } + urlStr := xurls.Relaxed.FindString(msg) switch { diff --git a/main.tpl b/main.tpl index 6567222..f5a2a59 100644 --- a/main.tpl +++ b/main.tpl @@ -67,19 +67,22 @@ ({{ . }}) {{ end }} {{ else }} - {{ if index . "Description" }} - {{ excerpt 384 (index . "Description") }} - {{ else }} - {{ with index . "ImageType" }} - {{ . }} image, + {{ with index . "Description" }} + {{ excerpt 384 . }} + {{ end }} + {{ end }} + + {{ if index . "ImageType" }} + {{ if index . "Title" }} + · + {{ end }} + {{ .ImageType }} image, + {{ if (index . "ImageSize") (index . "Size") }} + {{ with index . "ImageSize" }} + {{ .X }}×{{ .Y }} {{ end }} - {{ if (index . "ImageSize") (index . "Size") }} - {{ with index . "ImageSize" }} - {{ .X }}×{{ .Y }} - {{ end }} - {{ with index . "Size" }} - ({{ size . }}) - {{ end }} + {{ with index . "Size" }} + ({{ size . }}) {{ end }} {{ end }} {{ end }} diff --git a/manager/antiflood.go b/manager/antiflood.go index 42d4692..9a100b2 100644 --- a/manager/antiflood.go +++ b/manager/antiflood.go @@ -17,6 +17,28 @@ func (m *Manager) initAntiflood() { m.cache = cache.New(1*time.Minute, 5*time.Second) } +func (m *Manager) TrackUser(target string, source string) (shouldIgnore bool) { + key := normalizeUserAntiflood(target, source) + + if _, ok := m.cache.Get(key); ok { + // User just joined here recently, ignore them + shouldIgnore = true + } + + return +} + +func (m *Manager) NotifyUserJoined(target string, source string) { + key := normalizeUserAntiflood(target, source) + + // When a user joins, he will be ignored for the first 30 seconds, + // enough to prevent parsing links from people who only join to spam their + // links immediately + if _, exists := m.cache.Get(key); !exists { + m.cache.Add(key, nil, 30*time.Second) + } +} + func (m *Manager) TrackUrl(target string, u *url.URL) (shouldIgnore bool) { key := normalizeUrlAntiflood(target, u) @@ -70,6 +92,17 @@ func normalizeTextAntiflood(target, text string) string { return fmt.Sprintf("TEXT/%s/%X", strings.ToUpper(target), s.Sum([]byte{})) } +func normalizeUserAntiflood(target, source string) string { + sourceSplitHost := strings.SplitN(source, "@", 2) + sourceSplitHostname := strings.Split(sourceSplitHost[1], ".") + if len(sourceSplitHostname) > 1 && + strings.EqualFold(sourceSplitHostname[len(sourceSplitHostname)-1], "IP") { + sourceSplitHostname[0] = "*" + } + source = fmt.Sprintf("%s!%s@%s", "*", "*", strings.Join(sourceSplitHostname, ".")) + return fmt.Sprintf("USER/%s/%s", strings.ToUpper(target), source) +} + // Proxies several methods of the IRC connection in order to drop repeated messages type ircConnectionProxy struct { *irc.Connection diff --git a/parsers/web/parser.go b/parsers/web/parser.go index 0097859..9abbb89 100644 --- a/parsers/web/parser.go +++ b/parsers/web/parser.go @@ -2,7 +2,6 @@ package web import ( "errors" - "log" "net/http" "net/url" "regexp" @@ -33,7 +32,9 @@ const ( maxHtmlSize = 8 * 1024 ) -type Parser struct{} +type Parser struct { + EnableImages bool +} func (p *Parser) Init() error { return nil @@ -65,6 +66,7 @@ func (p *Parser) Parse(u *url.URL, referer *url.URL) (result parsers.ParseResult if referer != nil { req.Header.Set("Referer", referer.String()) } + req.Header.Set("User-Agent", "MediaLink IRC Bot") if resp, err := http.DefaultTransport.RoundTrip(req); err != nil { result.Error = err return @@ -119,23 +121,28 @@ func (p *Parser) Parse(u *url.URL, referer *url.URL) (result parsers.ParseResult result.Information[0]["Title"] = noTitleStr } case "image/png", "image/jpeg", "image/gif": + if p.EnableImages { - // No need to limit the reader to a specific size here as - // image.DecodeConfig only reads as much as needed anyways. - if m, imgType, err := image.DecodeConfig(resp.Body); err != nil { - result.UserError = ErrCorruptedImage - } else { - info := map[string]interface{}{ - "IsUpload": true, - "ImageSize": image.Point{X: m.Width, Y: m.Height}, - "ImageType": strings.ToUpper(imgType), + // No need to limit the reader to a specific size here as + // image.DecodeConfig only reads as much as needed anyways. + if m, imgType, err := image.DecodeConfig(resp.Body); err != nil { + result.UserError = ErrCorruptedImage + } else { + info := map[string]interface{}{ + "IsUpload": true, + "ImageSize": image.Point{X: m.Width, Y: m.Height}, + "ImageType": strings.ToUpper(imgType), + "Title": u.Path[strings.LastIndex(u.Path, "/")+1:], + } + if resp.ContentLength > 0 { + info["Size"] = uint64(resp.ContentLength) + } + result.Information = []map[string]interface{}{info} } - if resp.ContentLength > 0 { - info["Size"] = uint64(resp.ContentLength) - } - result.Information = []map[string]interface{}{info} - log.Printf("Got through: %+v!", info) + break } + + fallthrough default: // TODO - Implement generic head info? result.Ignored = true diff --git a/util_test.go b/util_test.go index 858dc8b..58d8413 100644 --- a/util_test.go +++ b/util_test.go @@ -1,33 +1,3 @@ package main -import ( - "net/url" - "testing" - - "github.com/stretchr/testify/assert" -) - -func mustParseUrl(u string) *url.URL { - if uri, err := url.Parse(u); err == nil { - return uri - } else { - panic(err) - } -} - -func Test_GetYouTubeId(t *testing.T) { - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://youtube.com/watch?v=aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://youtube.com/watch?v=aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://www.youtube.com/watch?v=aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://www.youtube.com/watch?v=aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://youtu.be/aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://youtu.be/aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://www.youtu.be/aYz-9jUlav-"))) - assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://www.youtu.be/aYz-9jUlav-"))) -} - -func Benchmark_GetYouTubeId(b *testing.B) { - for n := 0; n < b.N; n++ { - getYouTubeId(mustParseUrl("http://youtube.com/watch?v=aYz-9jUlav-")) - } -} +// TODO - unit test stripIrcFormatting