1
0
Fork 0

Move code to pkg

This commit is contained in:
Ludovic Fernandez 2019-03-15 09:42:03 +01:00 committed by Traefiker Bot
parent bd4c822670
commit f1b085fa36
465 changed files with 656 additions and 680 deletions

136
pkg/anonymize/anonymize.go Normal file
View file

@ -0,0 +1,136 @@
package anonymize
import (
"encoding/json"
"fmt"
"reflect"
"regexp"
"github.com/mitchellh/copystructure"
"github.com/mvdan/xurls"
)
const (
maskShort = "xxxx"
maskLarge = maskShort + maskShort + maskShort + maskShort + maskShort + maskShort + maskShort + maskShort
)
// Do configuration.
func Do(baseConfig interface{}, indent bool) (string, error) {
anomConfig, err := copystructure.Copy(baseConfig)
if err != nil {
return "", err
}
val := reflect.ValueOf(anomConfig)
err = doOnStruct(val)
if err != nil {
return "", err
}
configJSON, err := marshal(anomConfig, indent)
if err != nil {
return "", err
}
return doOnJSON(string(configJSON)), nil
}
func doOnJSON(input string) string {
mailExp := regexp.MustCompile(`\w[-._\w]*\w@\w[-._\w]*\w\.\w{2,3}"`)
return xurls.Relaxed.ReplaceAllString(mailExp.ReplaceAllString(input, maskLarge+"\""), maskLarge)
}
func doOnStruct(field reflect.Value) error {
switch field.Kind() {
case reflect.Ptr:
if !field.IsNil() {
if err := doOnStruct(field.Elem()); err != nil {
return err
}
}
case reflect.Struct:
for i := 0; i < field.NumField(); i++ {
fld := field.Field(i)
stField := field.Type().Field(i)
if !isExported(stField) {
continue
}
if stField.Tag.Get("export") == "true" {
if err := doOnStruct(fld); err != nil {
return err
}
} else {
if err := reset(fld, stField.Name); err != nil {
return err
}
}
}
case reflect.Map:
for _, key := range field.MapKeys() {
if err := doOnStruct(field.MapIndex(key)); err != nil {
return err
}
}
case reflect.Slice:
for j := 0; j < field.Len(); j++ {
if err := doOnStruct(field.Index(j)); err != nil {
return err
}
}
}
return nil
}
func reset(field reflect.Value, name string) error {
if !field.CanSet() {
return fmt.Errorf("cannot reset field %s", name)
}
switch field.Kind() {
case reflect.Ptr:
if !field.IsNil() {
field.Set(reflect.Zero(field.Type()))
}
case reflect.Struct:
if field.IsValid() {
field.Set(reflect.Zero(field.Type()))
}
case reflect.String:
if field.String() != "" {
field.Set(reflect.ValueOf(maskShort))
}
case reflect.Map:
if field.Len() > 0 {
field.Set(reflect.MakeMap(field.Type()))
}
case reflect.Slice:
if field.Len() > 0 {
field.Set(reflect.MakeSlice(field.Type(), 0, 0))
}
case reflect.Interface:
if !field.IsNil() {
return reset(field.Elem(), "")
}
default:
// Primitive type
field.Set(reflect.Zero(field.Type()))
}
return nil
}
// isExported return true is a struct field is exported, else false
func isExported(f reflect.StructField) bool {
if f.PkgPath != "" && !f.Anonymous {
return false
}
return true
}
func marshal(anomConfig interface{}, indent bool) ([]byte, error) {
if indent {
return json.MarshalIndent(anomConfig, "", " ")
}
return json.Marshal(anomConfig)
}

View file

@ -0,0 +1,329 @@
package anonymize
import (
"os"
"testing"
"time"
"github.com/containous/flaeg/parse"
"github.com/containous/traefik/pkg/config/static"
"github.com/containous/traefik/pkg/ping"
"github.com/containous/traefik/pkg/provider"
"github.com/containous/traefik/pkg/provider/acme"
acmeprovider "github.com/containous/traefik/pkg/provider/acme"
"github.com/containous/traefik/pkg/provider/docker"
"github.com/containous/traefik/pkg/provider/file"
"github.com/containous/traefik/pkg/provider/kubernetes/crd"
"github.com/containous/traefik/pkg/provider/kubernetes/ingress"
traefiktls "github.com/containous/traefik/pkg/tls"
"github.com/containous/traefik/pkg/tracing/datadog"
"github.com/containous/traefik/pkg/tracing/instana"
"github.com/containous/traefik/pkg/tracing/jaeger"
"github.com/containous/traefik/pkg/tracing/zipkin"
"github.com/containous/traefik/pkg/types"
assetfs "github.com/elazarl/go-bindata-assetfs"
)
func TestDo_globalConfiguration(t *testing.T) {
config := &static.Configuration{}
sendAnonymousUsage := true
config.Global = &static.Global{
Debug: true,
CheckNewVersion: true,
SendAnonymousUsage: &sendAnonymousUsage,
}
config.AccessLog = &types.AccessLog{
FilePath: "AccessLog FilePath",
Format: "AccessLog Format",
Filters: &types.AccessLogFilters{
StatusCodes: types.StatusCodes{"200", "500"},
RetryAttempts: true,
MinDuration: 10,
},
Fields: &types.AccessLogFields{
DefaultMode: "drop",
Names: types.FieldNames{
"RequestHost": "keep",
},
Headers: &types.FieldHeaders{
DefaultMode: "drop",
Names: types.FieldHeaderNames{
"Referer": "keep",
},
},
},
BufferingSize: 4,
}
config.Log = &types.TraefikLog{
LogLevel: "LogLevel",
FilePath: "/foo/path",
Format: "json",
}
config.EntryPoints = static.EntryPoints{
"foo": {
Address: "foo Address",
Transport: &static.EntryPointsTransport{
RespondingTimeouts: &static.RespondingTimeouts{
ReadTimeout: parse.Duration(111 * time.Second),
WriteTimeout: parse.Duration(111 * time.Second),
IdleTimeout: parse.Duration(111 * time.Second),
},
},
ProxyProtocol: &static.ProxyProtocol{
TrustedIPs: []string{"127.0.0.1/32", "192.168.0.1"},
},
},
"fii": {
Address: "fii Address",
Transport: &static.EntryPointsTransport{
RespondingTimeouts: &static.RespondingTimeouts{
ReadTimeout: parse.Duration(111 * time.Second),
WriteTimeout: parse.Duration(111 * time.Second),
IdleTimeout: parse.Duration(111 * time.Second),
},
},
ProxyProtocol: &static.ProxyProtocol{
TrustedIPs: []string{"127.0.0.1/32", "192.168.0.1"},
},
},
}
config.ACME = &acme.Configuration{
Email: "acme Email",
ACMELogging: true,
CAServer: "CAServer",
Storage: "Storage",
EntryPoint: "EntryPoint",
KeyType: "MyKeyType",
OnHostRule: true,
DNSChallenge: &acmeprovider.DNSChallenge{Provider: "DNSProvider"},
HTTPChallenge: &acmeprovider.HTTPChallenge{
EntryPoint: "MyEntryPoint",
},
TLSChallenge: &acmeprovider.TLSChallenge{},
Domains: []types.Domain{
{
Main: "Domains Main",
SANs: []string{"Domains acme SANs 1", "Domains acme SANs 2", "Domains acme SANs 3"},
},
},
}
config.Providers = &static.Providers{
ProvidersThrottleDuration: parse.Duration(111 * time.Second),
}
config.ServersTransport = &static.ServersTransport{
InsecureSkipVerify: true,
RootCAs: traefiktls.FilesOrContents{"RootCAs 1", "RootCAs 2", "RootCAs 3"},
MaxIdleConnsPerHost: 111,
ForwardingTimeouts: &static.ForwardingTimeouts{
DialTimeout: parse.Duration(111 * time.Second),
ResponseHeaderTimeout: parse.Duration(111 * time.Second),
},
}
config.API = &static.API{
EntryPoint: "traefik",
Dashboard: true,
Statistics: &types.Statistics{
RecentErrors: 111,
},
DashboardAssets: &assetfs.AssetFS{
Asset: func(path string) ([]byte, error) {
return nil, nil
},
AssetDir: func(path string) ([]string, error) {
return nil, nil
},
AssetInfo: func(path string) (os.FileInfo, error) {
return nil, nil
},
Prefix: "fii",
},
Middlewares: []string{"first", "second"},
}
config.Providers.File = &file.Provider{
BaseProvider: provider.BaseProvider{
Watch: true,
Filename: "file Filename",
Constraints: types.Constraints{
{
Key: "file Constraints Key 1",
Regex: "file Constraints Regex 2",
MustMatch: true,
},
{
Key: "file Constraints Key 1",
Regex: "file Constraints Regex 2",
MustMatch: true,
},
},
Trace: true,
DebugLogGeneratedTemplate: true,
},
Directory: "file Directory",
}
config.Providers.Docker = &docker.Provider{
BaseProvider: provider.BaseProvider{
Watch: true,
Filename: "myfilename",
Constraints: nil,
Trace: true,
DebugLogGeneratedTemplate: true,
},
Endpoint: "MyEndPoint",
DefaultRule: "PathPrefix(`/`)",
TLS: &types.ClientTLS{
CA: "myCa",
CAOptional: true,
Cert: "mycert.pem",
Key: "mycert.key",
InsecureSkipVerify: true,
},
ExposedByDefault: true,
UseBindPortIP: true,
SwarmMode: true,
Network: "MyNetwork",
SwarmModeRefreshSeconds: 42,
}
config.Providers.Kubernetes = &ingress.Provider{
BaseProvider: provider.BaseProvider{
Watch: true,
Filename: "myFileName",
Constraints: types.Constraints{
{
Key: "k8s Constraints Key 1",
Regex: "k8s Constraints Regex 2",
MustMatch: true,
},
{
Key: "k8s Constraints Key 1",
Regex: "k8s Constraints Regex 2",
MustMatch: true,
},
},
Trace: true,
DebugLogGeneratedTemplate: true,
},
Endpoint: "MyEndpoint",
Token: "MyToken",
CertAuthFilePath: "MyCertAuthPath",
DisablePassHostHeaders: true,
EnablePassTLSCert: true,
Namespaces: []string{"a", "b"},
LabelSelector: "myLabelSelector",
IngressClass: "MyIngressClass",
}
config.Providers.KubernetesCRD = &crd.Provider{
BaseProvider: provider.BaseProvider{
Watch: true,
Filename: "myFileName",
Constraints: types.Constraints{
{
Key: "k8s Constraints Key 1",
Regex: "k8s Constraints Regex 2",
MustMatch: true,
},
{
Key: "k8s Constraints Key 1",
Regex: "k8s Constraints Regex 2",
MustMatch: true,
},
},
Trace: true,
DebugLogGeneratedTemplate: true,
},
Endpoint: "MyEndpoint",
Token: "MyToken",
CertAuthFilePath: "MyCertAuthPath",
DisablePassHostHeaders: true,
EnablePassTLSCert: true,
Namespaces: []string{"a", "b"},
LabelSelector: "myLabelSelector",
IngressClass: "MyIngressClass",
}
// FIXME Test the other providers once they are migrated
config.Metrics = &types.Metrics{
Prometheus: &types.Prometheus{
Buckets: types.Buckets{0.1, 0.3, 1.2, 5},
EntryPoint: "MyEntryPoint",
Middlewares: []string{"m1", "m2"},
},
Datadog: &types.Datadog{
Address: "localhost:8181",
PushInterval: "12",
},
StatsD: &types.Statsd{
Address: "localhost:8182",
PushInterval: "42",
},
InfluxDB: &types.InfluxDB{
Address: "localhost:8183",
Protocol: "http",
PushInterval: "22",
Database: "myDB",
RetentionPolicy: "12",
Username: "a",
Password: "aaaa",
},
}
config.Ping = &ping.Handler{
EntryPoint: "MyEntryPoint",
Middlewares: []string{"m1", "m2", "m3"},
}
config.Tracing = &static.Tracing{
Backend: "myBackend",
ServiceName: "myServiceName",
SpanNameLimit: 3,
Jaeger: &jaeger.Config{
SamplingServerURL: "aaa",
SamplingType: "bbb",
SamplingParam: 43,
LocalAgentHostPort: "ccc",
Gen128Bit: true,
Propagation: "ddd",
TraceContextHeaderName: "eee",
},
Zipkin: &zipkin.Config{
HTTPEndpoint: "fff",
SameSpan: true,
ID128Bit: true,
Debug: true,
SampleRate: 53,
},
DataDog: &datadog.Config{
LocalAgentHostPort: "ggg",
GlobalTag: "eee",
Debug: true,
PrioritySampling: true,
},
Instana: &instana.Config{
LocalAgentHost: "fff",
LocalAgentPort: 32,
LogLevel: "ggg",
},
}
config.HostResolver = &types.HostResolverConfig{
CnameFlattening: true,
ResolvConfig: "aaa",
ResolvDepth: 3,
}
cleanJSON, err := Do(config, true)
if err != nil {
t.Fatal(err, cleanJSON)
}
}

View file

@ -0,0 +1,225 @@
package anonymize
import (
"testing"
"github.com/stretchr/testify/assert"
)
func Test_doOnJSON(t *testing.T) {
baseConfiguration := `
{
"GraceTimeOut": 10000000000,
"Debug": false,
"CheckNewVersion": true,
"AccessLogsFile": "",
"TraefikLogsFile": "",
"LogLevel": "ERROR",
"EntryPoints": {
"http": {
"Network": "",
"Address": ":80",
"TLS": null,
"Auth": null,
"Compress": false
},
"https": {
"Address": ":443",
"TLS": {
"MinVersion": "",
"CipherSuites": null,
"Certificates": null,
"ClientCAFiles": null
},
"Auth": null,
"Compress": false
}
},
"Cluster": null,
"Constraints": [],
"ACME": {
"Email": "foo@bar.com",
"Domains": [
{
"Main": "foo@bar.com",
"SANs": null
},
{
"Main": "foo@bar.com",
"SANs": null
}
],
"Storage": "",
"StorageFile": "/acme/acme.json",
"OnDemand": true,
"OnHostRule": true,
"CAServer": "",
"EntryPoint": "https",
"DNSProvider": "",
"DelayDontCheckDNS": 0,
"ACMELogging": false,
"TLSOptions": null
},
"DefaultEntryPoints": [
"https",
"http"
],
"ProvidersThrottleDuration": 2000000000,
"MaxIdleConnsPerHost": 200,
"IdleTimeout": 180000000000,
"InsecureSkipVerify": false,
"Retry": null,
"HealthCheck": {
"Interval": 30000000000
},
"Docker": null,
"File": null,
"Web": null,
"Marathon": null,
"Consul": null,
"ConsulCatalog": null,
"Etcd": null,
"Zookeeper": null,
"Boltdb": null,
"Kubernetes": null,
"Mesos": null,
"Eureka": null,
"ECS": null,
"Rancher": null,
"DynamoDB": null,
"ConfigFile": "/etc/traefik/traefik.toml"
}
`
expectedConfiguration := `
{
"GraceTimeOut": 10000000000,
"Debug": false,
"CheckNewVersion": true,
"AccessLogsFile": "",
"TraefikLogsFile": "",
"LogLevel": "ERROR",
"EntryPoints": {
"http": {
"Network": "",
"Address": ":80",
"TLS": null,
"Auth": null,
"Compress": false
},
"https": {
"Address": ":443",
"TLS": {
"MinVersion": "",
"CipherSuites": null,
"Certificates": null,
"ClientCAFiles": null
},
"Auth": null,
"Compress": false
}
},
"Cluster": null,
"Constraints": [],
"ACME": {
"Email": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"Domains": [
{
"Main": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"SANs": null
},
{
"Main": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"SANs": null
}
],
"Storage": "",
"StorageFile": "/acme/acme.json",
"OnDemand": true,
"OnHostRule": true,
"CAServer": "",
"EntryPoint": "https",
"DNSProvider": "",
"DelayDontCheckDNS": 0,
"ACMELogging": false,
"TLSOptions": null
},
"DefaultEntryPoints": [
"https",
"http"
],
"ProvidersThrottleDuration": 2000000000,
"MaxIdleConnsPerHost": 200,
"IdleTimeout": 180000000000,
"InsecureSkipVerify": false,
"Retry": null,
"HealthCheck": {
"Interval": 30000000000
},
"Docker": null,
"File": null,
"Web": null,
"Marathon": null,
"Consul": null,
"ConsulCatalog": null,
"Etcd": null,
"Zookeeper": null,
"Boltdb": null,
"Kubernetes": null,
"Mesos": null,
"Eureka": null,
"ECS": null,
"Rancher": null,
"DynamoDB": null,
"ConfigFile": "/etc/traefik/traefik.toml"
}
`
anomConfiguration := doOnJSON(baseConfiguration)
if anomConfiguration != expectedConfiguration {
t.Errorf("Got %s, want %s.", anomConfiguration, expectedConfiguration)
}
}
func Test_doOnJSON_simple(t *testing.T) {
testCases := []struct {
name string
input string
expectedOutput string
}{
{
name: "email",
input: `{
"email1": "goo@example.com",
"email2": "foo.bargoo@example.com",
"email3": "foo.bargoo@example.com.us"
}`,
expectedOutput: `{
"email1": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"email2": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
"email3": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
}`,
},
{
name: "url",
input: `{
"URL": "foo domain.com foo",
"URL": "foo sub.domain.com foo",
"URL": "foo sub.sub.domain.com foo",
"URL": "foo sub.sub.sub.domain.com.us foo"
}`,
expectedOutput: `{
"URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo",
"URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo",
"URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo",
"URL": "foo xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx foo"
}`,
},
}
for _, test := range testCases {
t.Run(test.name, func(t *testing.T) {
output := doOnJSON(test.input)
assert.Equal(t, test.expectedOutput, output)
})
}
}

View file

@ -0,0 +1,176 @@
package anonymize
import (
"reflect"
"testing"
"github.com/stretchr/testify/assert"
)
type Courgette struct {
Ji string
Ho string
}
type Tomate struct {
Ji string
Ho string
}
type Carotte struct {
Name string
Value int
Courgette Courgette
ECourgette Courgette `export:"true"`
Pourgette *Courgette
EPourgette *Courgette `export:"true"`
Aubergine map[string]string
EAubergine map[string]string `export:"true"`
SAubergine map[string]Tomate
ESAubergine map[string]Tomate `export:"true"`
PSAubergine map[string]*Tomate
EPAubergine map[string]*Tomate `export:"true"`
}
func Test_doOnStruct(t *testing.T) {
testCase := []struct {
name string
base *Carotte
expected *Carotte
hasError bool
}{
{
name: "primitive",
base: &Carotte{
Name: "koko",
Value: 666,
},
expected: &Carotte{
Name: "xxxx",
},
},
{
name: "struct",
base: &Carotte{
Name: "koko",
Courgette: Courgette{
Ji: "huu",
},
},
expected: &Carotte{
Name: "xxxx",
},
},
{
name: "pointer",
base: &Carotte{
Name: "koko",
Pourgette: &Courgette{
Ji: "hoo",
},
},
expected: &Carotte{
Name: "xxxx",
Pourgette: nil,
},
},
{
name: "export struct",
base: &Carotte{
Name: "koko",
ECourgette: Courgette{
Ji: "huu",
},
},
expected: &Carotte{
Name: "xxxx",
ECourgette: Courgette{
Ji: "xxxx",
},
},
},
{
name: "export pointer struct",
base: &Carotte{
Name: "koko",
ECourgette: Courgette{
Ji: "huu",
},
},
expected: &Carotte{
Name: "xxxx",
ECourgette: Courgette{
Ji: "xxxx",
},
},
},
{
name: "export map string/string",
base: &Carotte{
Name: "koko",
EAubergine: map[string]string{
"foo": "bar",
},
},
expected: &Carotte{
Name: "xxxx",
EAubergine: map[string]string{
"foo": "bar",
},
},
},
{
name: "export map string/pointer",
base: &Carotte{
Name: "koko",
EPAubergine: map[string]*Tomate{
"foo": {
Ji: "fdskljf",
},
},
},
expected: &Carotte{
Name: "xxxx",
EPAubergine: map[string]*Tomate{
"foo": {
Ji: "xxxx",
},
},
},
},
{
name: "export map string/struct (UNSAFE)",
base: &Carotte{
Name: "koko",
ESAubergine: map[string]Tomate{
"foo": {
Ji: "JiJiJi",
},
},
},
expected: &Carotte{
Name: "xxxx",
ESAubergine: map[string]Tomate{
"foo": {
Ji: "JiJiJi",
},
},
},
hasError: true,
},
}
for _, test := range testCase {
t.Run(test.name, func(t *testing.T) {
val := reflect.ValueOf(test.base).Elem()
err := doOnStruct(val)
if !test.hasError && err != nil {
t.Fatal(err)
}
if test.hasError && err == nil {
t.Fatal("Got no error but want an error.")
}
assert.EqualValues(t, test.expected, test.base)
})
}
}