1
0
Fork 0

Ultimate Access log filter

This commit is contained in:
Michael 2018-03-14 14:12:04 +01:00 committed by Traefiker Bot
parent f99363674b
commit 8d468925d3
24 changed files with 1722 additions and 683 deletions

View file

@ -15,7 +15,6 @@ import (
"time"
"github.com/containous/traefik/types"
shellwords "github.com/mattn/go-shellwords"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -127,156 +126,392 @@ func TestLoggerCLF(t *testing.T) {
logData, err := ioutil.ReadFile(logFilePath)
require.NoError(t, err)
assertValidLogData(t, logData)
expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testFrontend" "http://127.0.0.1/testBackend" 1ms`
assertValidLogData(t, expectedLog, logData)
}
func assertString(exp string) func(t *testing.T, actual interface{}) {
return func(t *testing.T, actual interface{}) {
t.Helper()
assert.Equal(t, exp, actual)
}
}
func assertNotEqual(exp string) func(t *testing.T, actual interface{}) {
return func(t *testing.T, actual interface{}) {
t.Helper()
assert.NotEqual(t, exp, actual)
}
}
func assertFloat64(exp float64) func(t *testing.T, actual interface{}) {
return func(t *testing.T, actual interface{}) {
t.Helper()
assert.Equal(t, exp, actual)
}
}
func assertFloat64NotZero() func(t *testing.T, actual interface{}) {
return func(t *testing.T, actual interface{}) {
t.Helper()
assert.NotZero(t, actual)
}
}
func TestLoggerJSON(t *testing.T) {
tmpDir := createTempDir(t, JSONFormat)
defer os.RemoveAll(tmpDir)
logFilePath := filepath.Join(tmpDir, logFileNameSuffix)
config := &types.AccessLog{FilePath: logFilePath, Format: JSONFormat}
doLogging(t, config)
logData, err := ioutil.ReadFile(logFilePath)
require.NoError(t, err)
jsonData := make(map[string]interface{})
err = json.Unmarshal(logData, &jsonData)
require.NoError(t, err)
expectedKeys := []string{
RequestHost,
RequestAddr,
RequestMethod,
RequestPath,
RequestProtocol,
RequestPort,
RequestLine,
DownstreamStatus,
DownstreamStatusLine,
DownstreamContentSize,
OriginContentSize,
OriginStatus,
"request_Referer",
"request_User-Agent",
FrontendName,
BackendURL,
ClientUsername,
ClientHost,
ClientPort,
ClientAddr,
"level",
"msg",
"downstream_Content-Type",
RequestCount,
Duration,
Overhead,
RetryAttempts,
"time",
"StartLocal",
"StartUTC",
testCases := []struct {
desc string
config *types.AccessLog
expected map[string]func(t *testing.T, value interface{})
}{
{
desc: "default config",
config: &types.AccessLog{
FilePath: "",
Format: JSONFormat,
},
expected: map[string]func(t *testing.T, value interface{}){
RequestHost: assertString(testHostname),
RequestAddr: assertString(testHostname),
RequestMethod: assertString(testMethod),
RequestPath: assertString(testPath),
RequestProtocol: assertString(testProto),
RequestPort: assertString("-"),
RequestLine: assertString(fmt.Sprintf("%s %s %s", testMethod, testPath, testProto)),
DownstreamStatus: assertFloat64(float64(testStatus)),
DownstreamStatusLine: assertString(fmt.Sprintf("%d ", testStatus)),
DownstreamContentSize: assertFloat64(float64(len(testContent))),
OriginContentSize: assertFloat64(float64(len(testContent))),
OriginStatus: assertFloat64(float64(testStatus)),
RequestRefererHeader: assertString(testReferer),
RequestUserAgentHeader: assertString(testUserAgent),
FrontendName: assertString(testFrontendName),
BackendURL: assertString(testBackendName),
ClientUsername: assertString(testUsername),
ClientHost: assertString(testHostname),
ClientPort: assertString(fmt.Sprintf("%d", testPort)),
ClientAddr: assertString(fmt.Sprintf("%s:%d", testHostname, testPort)),
"level": assertString("info"),
"msg": assertString(""),
"downstream_Content-Type": assertString("text/plain; charset=utf-8"),
RequestCount: assertFloat64NotZero(),
Duration: assertFloat64NotZero(),
Overhead: assertFloat64NotZero(),
RetryAttempts: assertFloat64(float64(testRetryAttempts)),
"time": assertNotEqual(""),
"StartLocal": assertNotEqual(""),
"StartUTC": assertNotEqual(""),
},
},
{
desc: "default config drop all fields",
config: &types.AccessLog{
FilePath: "",
Format: JSONFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
},
},
expected: map[string]func(t *testing.T, value interface{}){
"level": assertString("info"),
"msg": assertString(""),
"time": assertNotEqual(""),
"downstream_Content-Type": assertString("text/plain; charset=utf-8"),
RequestRefererHeader: assertString(testReferer),
RequestUserAgentHeader: assertString(testUserAgent),
},
},
{
desc: "default config drop all fields and headers",
config: &types.AccessLog{
FilePath: "",
Format: JSONFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Headers: &types.FieldHeaders{
DefaultMode: "drop",
},
},
},
expected: map[string]func(t *testing.T, value interface{}){
"level": assertString("info"),
"msg": assertString(""),
"time": assertNotEqual(""),
},
},
{
desc: "default config drop all fields and redact headers",
config: &types.AccessLog{
FilePath: "",
Format: JSONFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Headers: &types.FieldHeaders{
DefaultMode: "redact",
},
},
},
expected: map[string]func(t *testing.T, value interface{}){
"level": assertString("info"),
"msg": assertString(""),
"time": assertNotEqual(""),
"downstream_Content-Type": assertString("REDACTED"),
RequestRefererHeader: assertString("REDACTED"),
RequestUserAgentHeader: assertString("REDACTED"),
},
},
{
desc: "default config drop all fields and headers but kept someone",
config: &types.AccessLog{
FilePath: "",
Format: JSONFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Names: types.FieldNames{
RequestHost: "keep",
},
Headers: &types.FieldHeaders{
DefaultMode: "drop",
Names: types.FieldHeaderNames{
"Referer": "keep",
},
},
},
},
expected: map[string]func(t *testing.T, value interface{}){
RequestHost: assertString(testHostname),
"level": assertString("info"),
"msg": assertString(""),
"time": assertNotEqual(""),
RequestRefererHeader: assertString(testReferer),
},
},
}
containsKeys(t, expectedKeys, jsonData)
var assertCount int
assert.Equal(t, testHostname, jsonData[RequestHost])
assertCount++
assert.Equal(t, testHostname, jsonData[RequestAddr])
assertCount++
assert.Equal(t, testMethod, jsonData[RequestMethod])
assertCount++
assert.Equal(t, testPath, jsonData[RequestPath])
assertCount++
assert.Equal(t, testProto, jsonData[RequestProtocol])
assertCount++
assert.Equal(t, "-", jsonData[RequestPort])
assertCount++
assert.Equal(t, fmt.Sprintf("%s %s %s", testMethod, testPath, testProto), jsonData[RequestLine])
assertCount++
assert.Equal(t, float64(testStatus), jsonData[DownstreamStatus])
assertCount++
assert.Equal(t, fmt.Sprintf("%d ", testStatus), jsonData[DownstreamStatusLine])
assertCount++
assert.Equal(t, float64(len(testContent)), jsonData[DownstreamContentSize])
assertCount++
assert.Equal(t, float64(len(testContent)), jsonData[OriginContentSize])
assertCount++
assert.Equal(t, float64(testStatus), jsonData[OriginStatus])
assertCount++
assert.Equal(t, testReferer, jsonData["request_Referer"])
assertCount++
assert.Equal(t, testUserAgent, jsonData["request_User-Agent"])
assertCount++
assert.Equal(t, testFrontendName, jsonData[FrontendName])
assertCount++
assert.Equal(t, testBackendName, jsonData[BackendURL])
assertCount++
assert.Equal(t, testUsername, jsonData[ClientUsername])
assertCount++
assert.Equal(t, testHostname, jsonData[ClientHost])
assertCount++
assert.Equal(t, fmt.Sprintf("%d", testPort), jsonData[ClientPort])
assertCount++
assert.Equal(t, fmt.Sprintf("%s:%d", testHostname, testPort), jsonData[ClientAddr])
assertCount++
assert.Equal(t, "info", jsonData["level"])
assertCount++
assert.Equal(t, "", jsonData["msg"])
assertCount++
assert.Equal(t, "text/plain; charset=utf-8", jsonData["downstream_Content-Type"].(string))
assertCount++
assert.NotZero(t, jsonData[RequestCount].(float64))
assertCount++
assert.NotZero(t, jsonData[Duration].(float64))
assertCount++
assert.NotZero(t, jsonData[Overhead].(float64))
assertCount++
assert.Equal(t, float64(testRetryAttempts), jsonData[RetryAttempts].(float64))
assertCount++
assert.NotEqual(t, "", jsonData["time"].(string))
assertCount++
assert.NotEqual(t, "", jsonData["StartLocal"].(string))
assertCount++
assert.NotEqual(t, "", jsonData["StartUTC"].(string))
assertCount++
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
t.Parallel()
assert.Equal(t, len(jsonData), assertCount, string(logData))
tmpDir := createTempDir(t, JSONFormat)
defer os.RemoveAll(tmpDir)
logFilePath := filepath.Join(tmpDir, logFileNameSuffix)
test.config.FilePath = logFilePath
doLogging(t, test.config)
logData, err := ioutil.ReadFile(logFilePath)
require.NoError(t, err)
jsonData := make(map[string]interface{})
err = json.Unmarshal(logData, &jsonData)
require.NoError(t, err)
assert.Equal(t, len(test.expected), len(jsonData))
for field, assertion := range test.expected {
assertion(t, jsonData[field])
}
})
}
}
func TestNewLogHandlerOutputStdout(t *testing.T) {
file, restoreStdout := captureStdout(t)
defer restoreStdout()
testCases := []struct {
desc string
config *types.AccessLog
expectedLog string
}{
{
desc: "default config",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
},
expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`,
},
{
desc: "Status code filter not matching",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Filters: &types.AccessLogFilters{
StatusCodes: []string{"200"},
},
},
expectedLog: ``,
},
{
desc: "Status code filter matching",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Filters: &types.AccessLogFilters{
StatusCodes: []string{"123"},
},
},
expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`,
},
{
desc: "Default mode keep",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "keep",
},
},
expectedLog: `TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`,
},
{
desc: "Default mode keep with override",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "keep",
Names: types.FieldNames{
ClientHost: "drop",
},
},
},
expectedLog: `- - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 23 "testFrontend" "http://127.0.0.1/testBackend" 1ms`,
},
{
desc: "Default mode drop",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
},
},
expectedLog: `- - - [-] "- - -" - - "testReferer" "testUserAgent" - - - 0ms`,
},
{
desc: "Default mode drop with override",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Names: types.FieldNames{
ClientHost: "drop",
ClientUsername: "keep",
},
},
},
expectedLog: `- - TestUser [-] "- - -" - - "testReferer" "testUserAgent" - - - 0ms`,
},
{
desc: "Default mode drop with header dropped",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Names: types.FieldNames{
ClientHost: "drop",
ClientUsername: "keep",
},
Headers: &types.FieldHeaders{
DefaultMode: "drop",
},
},
},
expectedLog: `- - TestUser [-] "- - -" - - "-" "-" - - - 0ms`,
},
{
desc: "Default mode drop with header redacted",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Names: types.FieldNames{
ClientHost: "drop",
ClientUsername: "keep",
},
Headers: &types.FieldHeaders{
DefaultMode: "redact",
},
},
},
expectedLog: `- - TestUser [-] "- - -" - - "REDACTED" "REDACTED" - - - 0ms`,
},
{
desc: "Default mode drop with header redacted",
config: &types.AccessLog{
FilePath: "",
Format: CommonFormat,
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Names: types.FieldNames{
ClientHost: "drop",
ClientUsername: "keep",
},
Headers: &types.FieldHeaders{
DefaultMode: "keep",
Names: types.FieldHeaderNames{
"Referer": "redact",
},
},
},
},
expectedLog: `- - TestUser [-] "- - -" - - "REDACTED" "testUserAgent" - - - 0ms`,
},
}
config := &types.AccessLog{FilePath: "", Format: CommonFormat}
doLogging(t, config)
for _, test := range testCases {
test := test
t.Run(test.desc, func(t *testing.T) {
written, err := ioutil.ReadFile(file.Name())
require.NoError(t, err, "unable to read captured stdout from file")
require.NotZero(t, len(written), "expected access log message on stdout")
assertValidLogData(t, written)
// NOTE: It is not possible to run these cases in parallel because we capture Stdout
file, restoreStdout := captureStdout(t)
defer restoreStdout()
doLogging(t, test.config)
written, err := ioutil.ReadFile(file.Name())
require.NoError(t, err, "unable to read captured stdout from file")
assertValidLogData(t, test.expectedLog, written)
})
}
}
func assertValidLogData(t *testing.T, logData []byte) {
tokens, err := shellwords.Parse(string(logData))
require.NoError(t, err)
func assertValidLogData(t *testing.T, expected string, logData []byte) {
if len(expected) > 0 {
result, err := ParseAccessLog(string(logData))
require.NoError(t, err)
formatErrMessage := fmt.Sprintf(`
Expected: TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testFrontend" "http://127.0.0.1/testBackend" 1ms
Actual: %s
`,
string(logData))
require.Equal(t, 14, len(tokens), formatErrMessage)
assert.Equal(t, testHostname, tokens[0], formatErrMessage)
assert.Equal(t, testUsername, tokens[2], formatErrMessage)
assert.Equal(t, fmt.Sprintf("%s %s %s", testMethod, testPath, testProto), tokens[5], formatErrMessage)
assert.Equal(t, fmt.Sprintf("%d", testStatus), tokens[6], formatErrMessage)
assert.Equal(t, fmt.Sprintf("%d", len(testContent)), tokens[7], formatErrMessage)
assert.Equal(t, testReferer, tokens[8], formatErrMessage)
assert.Equal(t, testUserAgent, tokens[9], formatErrMessage)
assert.Regexp(t, regexp.MustCompile("[0-9]*"), tokens[10], formatErrMessage)
assert.Equal(t, testFrontendName, tokens[11], formatErrMessage)
assert.Equal(t, testBackendName, tokens[12], formatErrMessage)
resultExpected, err := ParseAccessLog(expected)
require.NoError(t, err)
formatErrMessage := fmt.Sprintf(`
Expected: %s
Actual: %s`, expected, string(logData))
require.Equal(t, len(resultExpected), len(result), formatErrMessage)
assert.Equal(t, resultExpected[ClientHost], result[ClientHost], formatErrMessage)
assert.Equal(t, resultExpected[ClientUsername], result[ClientUsername], formatErrMessage)
assert.Equal(t, resultExpected[RequestMethod], result[RequestMethod], formatErrMessage)
assert.Equal(t, resultExpected[RequestPath], result[RequestPath], formatErrMessage)
assert.Equal(t, resultExpected[RequestProtocol], result[RequestProtocol], formatErrMessage)
assert.Equal(t, resultExpected[OriginStatus], result[OriginStatus], formatErrMessage)
assert.Equal(t, resultExpected[OriginContentSize], result[OriginContentSize], formatErrMessage)
assert.Equal(t, resultExpected[RequestRefererHeader], result[RequestRefererHeader], formatErrMessage)
assert.Equal(t, resultExpected[RequestUserAgentHeader], result[RequestUserAgentHeader], formatErrMessage)
assert.Regexp(t, regexp.MustCompile("[0-9]*"), result[RequestCount], formatErrMessage)
assert.Equal(t, resultExpected[FrontendName], result[FrontendName], formatErrMessage)
assert.Equal(t, resultExpected[BackendURL], result[BackendURL], formatErrMessage)
assert.Regexp(t, regexp.MustCompile("[0-9]*ms"), result[Duration], formatErrMessage)
}
}
func captureStdout(t *testing.T) (out *os.File, restoreStdout func()) {
@ -328,28 +563,6 @@ func doLogging(t *testing.T, config *types.AccessLog) {
logger.ServeHTTP(httptest.NewRecorder(), req, logWriterTestHandlerFunc)
}
func containsKeys(t *testing.T, expectedKeys []string, data map[string]interface{}) {
for key, value := range data {
if !contains(expectedKeys, key) {
t.Errorf("Unexpected log key: %s [value: %s]", key, value)
}
}
for _, k := range expectedKeys {
if _, ok := data[k]; !ok {
t.Errorf("the expected key '%s' is not present in the map. %+v", k, data)
}
}
}
func contains(values []string, value string) bool {
for _, v := range values {
if value == v {
return true
}
}
return false
}
func logWriterTestHandlerFunc(rw http.ResponseWriter, r *http.Request) {
rw.Write([]byte(testContent))
rw.WriteHeader(testStatus)