Merge remote-tracking branch 'origin/develop'

main.go

@@ -31,6 +31,8 @@ func main() {
 	var soundcloudClientId string
 	var soundcloudClientSecret string
 
+	var webEnableImages bool
+
 	var debug bool
 	var noInvite bool
 	var useTLS bool
@@ -59,9 +61,14 @@ func main() {
 
 	// Youtube config
 	kingpin.Flag("youtube-key", "The API key to use to access the YouTube API.").StringVar(&youtubeApiKey)
+
+	// SoundCloud config
 	kingpin.Flag("soundcloud-id", "The SoundCloud ID.").StringVar(&soundcloudClientId)
 	kingpin.Flag("soundcloud-secret", "The SoundCloud secret.").StringVar(&soundcloudClientSecret)
 
+	// Web parser config
+	kingpin.Flag("images", "Enables parsing links of images. Disabled by default for legal reasons.").BoolVar(&webEnableImages)
+
 	kingpin.Parse()
 
 	if len(nickname) == 0 {
@@ -75,12 +82,17 @@ func main() {
 	m := manager.NewManager()
 
 	// Load youtube parser
+	if len(youtubeApiKey) > 0 {
 		youtubeParser := &youtube.Parser{
 			Config: &youtube.Config{ApiKey: youtubeApiKey},
 		}
 		must(m.RegisterParser(youtubeParser))
+	} else {
+		log.Println("No YouTube API key provided, YouTube parsing via API is disabled.")
+	}
 
 	// Load soundcloud parser
+	if len(soundcloudClientId) > 0 && len(soundcloudClientSecret) > 0 {
 		soundcloudParser := &soundcloud.Parser{
 			Config: &soundcloud.Config{
 				ClientId:     soundcloudClientId,
@@ -88,12 +100,18 @@ func main() {
 			},
 		}
 		must(m.RegisterParser(soundcloudParser))
+	} else {
+		log.Println("No SoundCloud client ID or secret provided, SoundCloud parsing via API is disabled.")
+	}
 
 	// Load wikipedia parser
 	must(m.RegisterParser(new(wikipedia.Parser)))
 
 	// Load web parser
-	must(m.RegisterParser(new(web.Parser)))
+	webParser := &web.Parser{
+		EnableImages: webEnableImages,
+	}
+	must(m.RegisterParser(webParser))
 
 	// IRC
 	conn := m.AntifloodIrcConn(irc.IRC(nickname, ident))
@@ -130,6 +148,8 @@ func main() {
 	conn.AddCallback("JOIN", func(e *irc.Event) {
 		// Is this JOIN not about us?
 		if !strings.EqualFold(e.Nick, conn.GetNick()) {
+			// Save this user's details for a temporary ignore
+			m.NotifyUserJoined(e.Arguments[0], e.Source)
 			return
 		}
 
@@ -199,6 +219,12 @@ func main() {
 
 			log.Printf("<%s @ %s> %s", event.Nick, target, msg)
 
+			// Ignore user if they just joined
+			if shouldIgnore := m.TrackUser(target, event.Source); shouldIgnore {
+				log.Print("This message will be ignored since the user just joined.")
+				return
+			}
+
 			urlStr := xurls.Relaxed.FindString(msg)
 
 			switch {

main.tpl

@@ -67,12 +67,16 @@
 				({{ . }})
 			{{ end }}
 		{{ else }}
-			{{ if index . "Description" }}
-				{{ excerpt 384 (index . "Description") }}
-			{{ else }}
-				{{ with index . "ImageType" }}
-					{{ . }} image,
+			{{ with index . "Description" }}
+				{{ excerpt 384 . }}
 			{{ end }}
+		{{ end }}
+		
+		{{ if index . "ImageType" }}
+			{{ if index . "Title" }}
+				·
+			{{ end }}
+			{{ .ImageType }} image,
 			{{ if (index . "ImageSize") (index . "Size") }}
 				{{ with index . "ImageSize" }}
 					{{ .X }}×{{ .Y }}
@@ -83,7 +87,6 @@
 			{{ end }}
 		{{ end }}
 	{{ end }}
-	{{ end }}
 
 	{{ if or (index . "Author") }}
 		{{ if index . "Author" }}

manager/antiflood.go

@@ -17,6 +17,28 @@ func (m *Manager) initAntiflood() {
 	m.cache = cache.New(1*time.Minute, 5*time.Second)
 }
 
+func (m *Manager) TrackUser(target string, source string) (shouldIgnore bool) {
+	key := normalizeUserAntiflood(target, source)
+
+	if _, ok := m.cache.Get(key); ok {
+		// User just joined here recently, ignore them
+		shouldIgnore = true
+	}
+
+	return
+}
+
+func (m *Manager) NotifyUserJoined(target string, source string) {
+	key := normalizeUserAntiflood(target, source)
+
+	// When a user joins, he will be ignored for the first 30 seconds,
+	// enough to prevent parsing links from people who only join to spam their
+	// links immediately
+	if _, exists := m.cache.Get(key); !exists {
+		m.cache.Add(key, nil, 30*time.Second)
+	}
+}
+
 func (m *Manager) TrackUrl(target string, u *url.URL) (shouldIgnore bool) {
 	key := normalizeUrlAntiflood(target, u)
 
@@ -70,6 +92,17 @@ func normalizeTextAntiflood(target, text string) string {
 	return fmt.Sprintf("TEXT/%s/%X", strings.ToUpper(target), s.Sum([]byte{}))
 }
 
+func normalizeUserAntiflood(target, source string) string {
+	sourceSplitHost := strings.SplitN(source, "@", 2)
+	sourceSplitHostname := strings.Split(sourceSplitHost[1], ".")
+	if len(sourceSplitHostname) > 1 &&
+		strings.EqualFold(sourceSplitHostname[len(sourceSplitHostname)-1], "IP") {
+		sourceSplitHostname[0] = "*"
+	}
+	source = fmt.Sprintf("%s!%s@%s", "*", "*", strings.Join(sourceSplitHostname, "."))
+	return fmt.Sprintf("USER/%s/%s", strings.ToUpper(target), source)
+}
+
 // Proxies several methods of the IRC connection in order to drop repeated messages
 type ircConnectionProxy struct {
 	*irc.Connection

parsers/web/parser.go

@@ -2,7 +2,6 @@ package web
 
 import (
 	"errors"
-	"log"
 	"net/http"
 	"net/url"
 	"regexp"
@@ -33,7 +32,9 @@ const (
 	maxHtmlSize = 8 * 1024
 )
 
-type Parser struct{}
+type Parser struct {
+	EnableImages bool
+}
 
 func (p *Parser) Init() error {
 	return nil
@@ -65,6 +66,7 @@ func (p *Parser) Parse(u *url.URL, referer *url.URL) (result parsers.ParseResult
 	if referer != nil {
 		req.Header.Set("Referer", referer.String())
 	}
+	req.Header.Set("User-Agent", "MediaLink IRC Bot")
 	if resp, err := http.DefaultTransport.RoundTrip(req); err != nil {
 		result.Error = err
 		return
@@ -119,6 +121,7 @@ func (p *Parser) Parse(u *url.URL, referer *url.URL) (result parsers.ParseResult
 				result.Information[0]["Title"] = noTitleStr
 			}
 		case "image/png", "image/jpeg", "image/gif":
+			if p.EnableImages {
 
 				// No need to limit the reader to a specific size here as
 				// image.DecodeConfig only reads as much as needed anyways.
@@ -129,13 +132,17 @@ func (p *Parser) Parse(u *url.URL, referer *url.URL) (result parsers.ParseResult
 						"IsUpload":  true,
 						"ImageSize": image.Point{X: m.Width, Y: m.Height},
 						"ImageType": strings.ToUpper(imgType),
+						"Title":     u.Path[strings.LastIndex(u.Path, "/")+1:],
 					}
 					if resp.ContentLength > 0 {
 						info["Size"] = uint64(resp.ContentLength)
 					}
 					result.Information = []map[string]interface{}{info}
-				log.Printf("Got through: %+v!", info)
 				}
+				break
+			}
+
+			fallthrough
 		default:
 			// TODO - Implement generic head info?
 			result.Ignored = true

util_test.go

@@ -1,33 +1,3 @@
 package main
 
-import (
-	"net/url"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func mustParseUrl(u string) *url.URL {
-	if uri, err := url.Parse(u); err == nil {
-		return uri
-	} else {
-		panic(err)
-	}
-}
-
-func Test_GetYouTubeId(t *testing.T) {
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://youtube.com/watch?v=aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://youtube.com/watch?v=aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://www.youtube.com/watch?v=aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://www.youtube.com/watch?v=aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://youtu.be/aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://youtu.be/aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("http://www.youtu.be/aYz-9jUlav-")))
-	assert.Equal(t, "aYz-9jUlav-", getYouTubeId(mustParseUrl("https://www.youtu.be/aYz-9jUlav-")))
-}
-
-func Benchmark_GetYouTubeId(b *testing.B) {
-	for n := 0; n < b.N; n++ {
-		getYouTubeId(mustParseUrl("http://youtube.com/watch?v=aYz-9jUlav-"))
-	}
-}
+// TODO - unit test stripIrcFormatting