homelab/traefik
1
0
Fork 0
Code Activity

Add GenericCLF log format for access logs

Browse source
This commit is contained in:
Simon Delicata 2025-09-08 11:24:05 +02:00 • committed by GitHub
parent a051f20876
commit e2282b1379
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 234 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

5
pkg/middlewares/accesslog/logger.go
View file

@ -37,6 +37,9 @@ const (
// CommonFormat is the common logging format (CLF).
CommonFormat string = "common"
// GenericCLFFormat is the generic CLF format.
GenericCLFFormat string = "genericCLF"
// JSONFormat is the JSON logging format.
JSONFormat string = "json"
)
@ -101,6 +104,8 @@ func NewHandler(ctx context.Context, config *types.AccessLog) (*Handler, error)
switch config.Format {
case CommonFormat:
formatter = new(CommonLogFormatter)
case GenericCLFFormat:
formatter = new(GenericCLFLogFormatter)
case JSONFormat:
formatter = new(logrus.JSONFormatter)
default:

29
pkg/middlewares/accesslog/logger_formatters.go
View file

@ -52,6 +52,35 @@ func (f *CommonLogFormatter) Format(entry *logrus.Entry) ([]byte, error) {
return b.Bytes(), err
}
// GenericCLFLogFormatter provides formatting in the generic CLF log format.
type GenericCLFLogFormatter struct{}
// Format formats the log entry in the generic CLF log format.
func (f *GenericCLFLogFormatter) Format(entry *logrus.Entry) ([]byte, error) {
b := &bytes.Buffer{}
timestamp := defaultValue
if v, ok := entry.Data[StartUTC]; ok {
timestamp = v.(time.Time).Format(commonLogTimeFormat)
} else if v, ok := entry.Data[StartLocal]; ok {
timestamp = v.(time.Time).Local().Format(commonLogTimeFormat)
}
_, err := fmt.Fprintf(b, "%s - %s [%s] \"%s %s %s\" %v %v %s %s\n",
toLog(entry.Data, ClientHost, defaultValue, false),
toLog(entry.Data, ClientUsername, defaultValue, false),
timestamp,
toLog(entry.Data, RequestMethod, defaultValue, false),
toLog(entry.Data, RequestPath, defaultValue, false),
toLog(entry.Data, RequestProtocol, defaultValue, false),
toLog(entry.Data, DownstreamStatus, defaultValue, true),
toLog(entry.Data, DownstreamContentSize, defaultValue, true),
toLog(entry.Data, "request_Referer", `"-"`, true),
toLog(entry.Data, "request_User-Agent", `"-"`, true))
return b.Bytes(), err
}
func toLog(fields logrus.Fields, key, defaultValue string, quoted bool) interface{} {
if v, ok := fields[key]; ok {
if v == nil {

91
pkg/middlewares/accesslog/logger_formatters_test.go
View file

@ -101,6 +101,97 @@ func TestCommonLogFormatter_Format(t *testing.T) {
}
}
func TestGenericCLFLogFormatter_Format(t *testing.T) {
clf := GenericCLFLogFormatter{}
testCases := []struct {
name string
data map[string]interface{}
expectedLog string
}{
{
name: "DownstreamStatus & DownstreamContentSize are nil",
data: map[string]interface{}{
StartUTC: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
Duration: 123 * time.Second,
ClientHost: "10.0.0.1",
ClientUsername: "Client",
RequestMethod: http.MethodGet,
RequestPath: "/foo",
RequestProtocol: "http",
DownstreamStatus: nil,
DownstreamContentSize: nil,
RequestRefererHeader: "",
RequestUserAgentHeader: "",
RequestCount: 0,
RouterName: "",
ServiceURL: "",
},
expectedLog: `10.0.0.1 - Client [10/Nov/2009:23:00:00 +0000] "GET /foo http" - - "-" "-"
`,
},
{
name: "all data",
data: map[string]interface{}{
StartUTC: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
Duration: 123 * time.Second,
ClientHost: "10.0.0.1",
ClientUsername: "Client",
RequestMethod: http.MethodGet,
RequestPath: "/foo",
RequestProtocol: "http",
DownstreamStatus: 123,
DownstreamContentSize: 132,
RequestRefererHeader: "referer",
RequestUserAgentHeader: "agent",
RequestCount: nil,
RouterName: "foo",
ServiceURL: "http://10.0.0.2/toto",
},
expectedLog: `10.0.0.1 - Client [10/Nov/2009:23:00:00 +0000] "GET /foo http" 123 132 "referer" "agent"
`,
},
{
name: "all data with local time",
data: map[string]interface{}{
StartLocal: time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC),
Duration: 123 * time.Second,
ClientHost: "10.0.0.1",
ClientUsername: "Client",
RequestMethod: http.MethodGet,
RequestPath: "/foo",
RequestProtocol: "http",
DownstreamStatus: 123,
DownstreamContentSize: 132,
RequestRefererHeader: "referer",
RequestUserAgentHeader: "agent",
RequestCount: nil,
RouterName: "foo",
ServiceURL: "http://10.0.0.2/toto",
},
expectedLog: `10.0.0.1 - Client [10/Nov/2009:14:00:00 -0900] "GET /foo http" 123 132 "referer" "agent"
`,
},
}
var err error
time.Local, err = time.LoadLocation("Etc/GMT+9")
require.NoError(t, err)
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
t.Parallel()
entry := &logrus.Entry{Data: test.data}
raw, err := clf.Format(entry)
assert.NoError(t, err)
assert.Equal(t, test.expectedLog, string(raw))
})
}
}
func Test_toLog(t *testing.T) {
testCases := []struct {
desc string

80
pkg/middlewares/accesslog/logger_test.go
View file

@ -68,7 +68,17 @@ func TestOTelAccessLogWithBody(t *testing.T) {
bodyCheckFn: func(t *testing.T, log string) {
t.Helper()
// For common format, verify the body contains the CLF formatted string
// For common format, verify the body contains the Traefik common log formatted string
assert.Regexp(t, `"body":{"stringValue":".*- /health -.*200.*[0-9]+ms.*"}`, log)
},
},
{
desc: "Generic CLF format with log body",
format: GenericCLFFormat,
bodyCheckFn: func(t *testing.T, log string) {
t.Helper()
// For generic CLF format, verify the body contains the CLF formatted string
assert.Regexp(t, `"body":{"stringValue":".*- /health -.*200.*"}`, log)
},
},
@ -375,7 +385,7 @@ func TestLoggerHeaderFields(t *testing.T) {
}
}
func TestLoggerCLF(t *testing.T) {
func TestCommonLogger(t *testing.T) {
logFilePath := filepath.Join(t.TempDir(), logFileNameSuffix)
config := &types.AccessLog{FilePath: logFilePath, Format: CommonFormat}
doLogging(t, config, false)
@ -384,10 +394,10 @@ func TestLoggerCLF(t *testing.T) {
require.NoError(t, err)
expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testRouter" "http://127.0.0.1/testService" 1ms`
assertValidLogData(t, expectedLog, logData)
assertValidCommonLogData(t, expectedLog, logData)
}
func TestLoggerCLFWithBufferingSize(t *testing.T) {
func TestCommonLoggerWithBufferingSize(t *testing.T) {
logFilePath := filepath.Join(t.TempDir(), logFileNameSuffix)
config := &types.AccessLog{FilePath: logFilePath, Format: CommonFormat, BufferingSize: 1024}
doLogging(t, config, false)
@ -399,7 +409,34 @@ func TestLoggerCLFWithBufferingSize(t *testing.T) {
require.NoError(t, err)
expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent" 1 "testRouter" "http://127.0.0.1/testService" 1ms`
assertValidLogData(t, expectedLog, logData)
assertValidCommonLogData(t, expectedLog, logData)
}
func TestLoggerGenericCLF(t *testing.T) {
logFilePath := filepath.Join(t.TempDir(), logFileNameSuffix)
config := &types.AccessLog{FilePath: logFilePath, Format: GenericCLFFormat}
doLogging(t, config, false)
logData, err := os.ReadFile(logFilePath)
require.NoError(t, err)
expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent"`
assertValidGenericCLFLogData(t, expectedLog, logData)
}
func TestLoggerGenericCLFWithBufferingSize(t *testing.T) {
logFilePath := filepath.Join(t.TempDir(), logFileNameSuffix)
config := &types.AccessLog{FilePath: logFilePath, Format: GenericCLFFormat, BufferingSize: 1024}
doLogging(t, config, false)
// wait a bit for the buffer to be written in the file.
time.Sleep(50 * time.Millisecond)
logData, err := os.ReadFile(logFilePath)
require.NoError(t, err)
expectedLog := ` TestHost - TestUser [13/Apr/2016:07:14:19 -0700] "POST testpath HTTP/0.0" 123 12 "testReferer" "testUserAgent"`
assertValidGenericCLFLogData(t, expectedLog, logData)
}
func assertString(exp string) func(t *testing.T, actual interface{}) {
@ -963,12 +1000,12 @@ func TestNewLogHandlerOutputStdout(t *testing.T) {
written, err := os.ReadFile(file.Name())
require.NoError(t, err, "unable to read captured stdout from file")
assertValidLogData(t, test.expectedLog, written)
assertValidCommonLogData(t, test.expectedLog, written)
})
}
}
func assertValidLogData(t *testing.T, expected string, logData []byte) {
func assertValidCommonLogData(t *testing.T, expected string, logData []byte) {
t.Helper()
if len(expected) == 0 {
@ -1001,6 +1038,35 @@ func assertValidLogData(t *testing.T, expected string, logData []byte) {
assert.Regexp(t, `\d*ms`, result[Duration], formatErrMessage)
}
func assertValidGenericCLFLogData(t *testing.T, expected string, logData []byte) {
t.Helper()
if len(expected) == 0 {
assert.Empty(t, logData)
t.Log(string(logData))
return
}
result, err := ParseAccessLog(string(logData))
require.NoError(t, err)
resultExpected, err := ParseAccessLog(expected)
require.NoError(t, err)
formatErrMessage := fmt.Sprintf("Expected:\t%q\nActual:\t%q", expected, string(logData))
require.Len(t, result, len(resultExpected), formatErrMessage)
assert.Equal(t, resultExpected[ClientHost], result[ClientHost], formatErrMessage)
assert.Equal(t, resultExpected[ClientUsername], result[ClientUsername], formatErrMessage)
assert.Equal(t, resultExpected[RequestMethod], result[RequestMethod], formatErrMessage)
assert.Equal(t, resultExpected[RequestPath], result[RequestPath], formatErrMessage)
assert.Equal(t, resultExpected[RequestProtocol], result[RequestProtocol], formatErrMessage)
assert.Equal(t, resultExpected[OriginStatus], result[OriginStatus], formatErrMessage)
assert.Equal(t, resultExpected[OriginContentSize], result[OriginContentSize], formatErrMessage)
assert.Equal(t, resultExpected[RequestRefererHeader], result[RequestRefererHeader], formatErrMessage)
assert.Equal(t, resultExpected[RequestUserAgentHeader], result[RequestUserAgentHeader], formatErrMessage)
}
func captureStdout(t *testing.T) (out *os.File, restoreStdout func()) {
t.Helper()

2
pkg/types/logs.go
View file

@ -58,7 +58,7 @@ func (l *TraefikLog) SetDefaults() {
// AccessLog holds the configuration settings for the access logger (middlewares/accesslog).
type AccessLog struct {
FilePath string `description:"Access log file path. Stdout is used when omitted or empty." json:"filePath,omitempty" toml:"filePath,omitempty" yaml:"filePath,omitempty"`
Format string `description:"Access log format: json | common" json:"format,omitempty" toml:"format,omitempty" yaml:"format,omitempty" export:"true"`
Format string `description:"Access log format: json, common, or genericCLF" json:"format,omitempty" toml:"format,omitempty" yaml:"format,omitempty" export:"true"`
Filters *AccessLogFilters `description:"Access log filters, used to keep only specific access logs." json:"filters,omitempty" toml:"filters,omitempty" yaml:"filters,omitempty" export:"true"`
Fields *AccessLogFields `description:"AccessLogFields." json:"fields,omitempty" toml:"fields,omitempty" yaml:"fields,omitempty" export:"true"`
BufferingSize int64 `description:"Number of access log lines to process in a buffered way." json:"bufferingSize,omitempty" toml:"bufferingSize,omitempty" yaml:"bufferingSize,omitempty" export:"true"`