diff --git a/.travis.yml b/.travis.yml index c217d39..4a617b1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,14 @@ language: go sudo: false go: - - 1.1 - - 1.2 + - 1.2.2 - 1.3 - 1.4 - 1.5 + - 1.12 + - 1.13 - tip +go_import_path: github.com/satyrius/gonx install: make deps script: - go test -v -bench . diff --git a/example/common/common.go b/example/common/common.go index 03696bd..30d4e5f 100644 --- a/example/common/common.go +++ b/example/common/common.go @@ -1,9 +1,9 @@ package main import ( - gonx "../.." "flag" "fmt" + gonx "github.com/satyrius/gonx" "io" "os" "strings" diff --git a/example/nginx/nginx.go b/example/nginx/nginx.go index 6af4cd4..cc349b8 100644 --- a/example/nginx/nginx.go +++ b/example/nginx/nginx.go @@ -1,9 +1,9 @@ package main import ( - gonx "../.." "flag" "fmt" + gonx "github.com/satyrius/gonx" "io" "os" "strings" diff --git a/example/reduce/reduce.go b/example/reduce/reduce.go index 95a8cb5..ea5c953 100644 --- a/example/reduce/reduce.go +++ b/example/reduce/reduce.go @@ -1,9 +1,9 @@ package main import ( - gonx "../.." "flag" "fmt" + gonx "github.com/satyrius/gonx" "io" "os" "strings" diff --git a/parser.go b/parser.go index 6e4d950..6411d4b 100644 --- a/parser.go +++ b/parser.go @@ -19,6 +19,11 @@ type Parser struct { regexp *regexp.Regexp } +func getSpecialNginxRegexes() map[string]string { + return map[string]string{ + "http_x_forwarded_for": `[^, ]*(?:, ?[^, ]+)*`} +} + // NewParser returns a new Parser, use given log format to create its internal // strings parsing regexp. func NewParser(format string) *Parser { @@ -32,10 +37,21 @@ func NewParser(format string) *Parser { ) } - // Second replace each fileds to regexp grouping - quotedFormat := regexp.QuoteMeta(preparedFormat + " ") - re := regexp.MustCompile(`\\\$([A-Za-z0-9_]+)(?:\\\$[A-Za-z0-9_])*(\\?([^$\\A-Za-z0-9_]))`).ReplaceAllString( - quotedFormat, "(?P<$1>[^$3]*)$2") + formatRegex := regexp.MustCompile(`([^$ ]*)\$([a-z_]+)([^$ ]*)([ ]?)`) + specialNginxRegexes := getSpecialNginxRegexes() + fields := formatRegex.FindAllStringSubmatch(preparedFormat+" ", -1) + re := formatRegex.ReplaceAllString(preparedFormat+" ", "$2$4") + for _, field := range fields { + terminateChar := field[3] + if len([]rune(terminateChar)) == 0 { + terminateChar = field[4] + } + if specialRegex, found := specialNginxRegexes[field[2]]; found { + re = strings.Replace(re, field[2]+field[4], regexp.QuoteMeta(field[1])+"(?P<"+field[2]+">"+specialRegex+")"+regexp.QuoteMeta(field[3])+field[4], 1) + } else { + re = strings.Replace(re, field[2]+field[4], regexp.QuoteMeta(field[1])+"(?P<"+field[2]+">[^"+terminateChar+"]*)"+regexp.QuoteMeta(field[3]+field[4]), 1) + } + } // Finally remove placeholder re = regexp.MustCompile(fmt.Sprintf(".%s", placeholder)).ReplaceAllString(re, "") diff --git a/parser_test.go b/parser_test.go index 90e7007..f6fe63c 100644 --- a/parser_test.go +++ b/parser_test.go @@ -9,7 +9,7 @@ import ( func TestParser(t *testing.T) { Convey("Test Parser", t, func() { Convey("Parse custom format", func() { - format := "$remote_addr [$time_local] \"$request\" $status" + format := "$remote_addr [$time_local] \"$request\" $status $http_x_forwarded_for" parser := NewParser(format) Convey("Ensure parser format is ok", func() { @@ -18,16 +18,17 @@ func TestParser(t *testing.T) { Convey("Test format to regexp translation", func() { So(parser.regexp.String(), ShouldEqual, - `^(?P[^ ]*) \[(?P[^]]*)\] "(?P[^"]*)" (?P[^ ]*)`) + `^(?P[^ ]*) \[(?P[^]]*)\] "(?P[^"]*)" (?P[^ ]*) (?P[^, ]*(?:, ?[^, ]+)*)`) }) Convey("ParseString", func() { - line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "GET /api/foo/bar HTTP/1.1" 200` + line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "GET /api/foo/bar HTTP/1.1" 200 1.2.3.4,5.6.7.8, 9.10.11.12` expected := NewEntry(Fields{ - "remote_addr": "89.234.89.123", - "time_local": "08/Nov/2013:13:39:18 +0000", - "request": "GET /api/foo/bar HTTP/1.1", - "status": "200", + "remote_addr": "89.234.89.123", + "time_local": "08/Nov/2013:13:39:18 +0000", + "request": "GET /api/foo/bar HTTP/1.1", + "status": "200", + "http_x_forwarded_for": "1.2.3.4,5.6.7.8, 9.10.11.12", }) entry, err := parser.ParseString(line) So(err, ShouldBeNil) @@ -35,12 +36,13 @@ func TestParser(t *testing.T) { }) Convey("Handle empty values", func() { - line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "" 200` + line := `89.234.89.123 [08/Nov/2013:13:39:18 +0000] "" 200 1.2.3.4` expected := NewEntry(Fields{ - "remote_addr": "89.234.89.123", - "time_local": "08/Nov/2013:13:39:18 +0000", - "request": "", - "status": "200", + "remote_addr": "89.234.89.123", + "time_local": "08/Nov/2013:13:39:18 +0000", + "request": "", + "status": "200", + "http_x_forwarded_for": "1.2.3.4", }) entry, err := parser.ParseString(line) So(err, ShouldBeNil)