homelab/traefik
1
0
Fork 0
Code Activity

Align OpenTelemetry tracing and metrics configurations

Browse source 
Co-authored-by: Michael <michael.matur@gmail.com>
This commit is contained in:
Romain 2024-02-06 10:04:05 +01:00 • committed by GitHub
parent d7ec0cedbf
commit bb6cd581a6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
18 changed files with 587 additions and 341 deletions
Download patch file Download diff file Expand all files Collapse all files

15
pkg/config/static/static_config.go
View file

@ -194,7 +194,6 @@ func (a *LifeCycle) SetDefaults() {
// Tracing holds the tracing configuration.
type Tracing struct {
ServiceName string `description:"Set the name for this service." json:"serviceName,omitempty" toml:"serviceName,omitempty" yaml:"serviceName,omitempty" export:"true"`
Headers map[string]string `description:"Defines additional headers to be sent with the payloads." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty" export:"true"`
GlobalAttributes map[string]string `description:"Defines additional attributes (key:value) on all spans." json:"globalAttributes,omitempty" toml:"globalAttributes,omitempty" yaml:"globalAttributes,omitempty" export:"true"`
SampleRate float64 `description:"Sets the rate between 0.0 and 1.0 of requests to trace." json:"sampleRate,omitempty" toml:"sampleRate,omitempty" yaml:"sampleRate,omitempty" export:"true"`
AddInternals bool `description:"Enables tracing for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"`
@ -342,19 +341,17 @@ func (c *Configuration) ValidateConfiguration() error {
}
if c.Tracing != nil && c.Tracing.OTLP != nil {
if c.Tracing.OTLP.HTTP == nil && c.Tracing.OTLP.GRPC == nil {
return errors.New("tracing OTLP: at least one of HTTP and gRPC options should be defined")
}
if c.Tracing.OTLP.HTTP != nil && c.Tracing.OTLP.GRPC != nil {
return errors.New("tracing OTLP: HTTP and gRPC options are mutually exclusive")
}
if c.Tracing.OTLP.GRPC != nil && c.Tracing.OTLP.GRPC.TLS != nil && c.Tracing.OTLP.GRPC.Insecure {
return errors.New("tracing OTLP GRPC: TLS and Insecure options are mutually exclusive")
}
}
if c.Metrics != nil && c.Metrics.OTLP != nil {
if c.Metrics.OTLP.GRPC != nil && c.Metrics.OTLP.GRPC.TLS != nil && c.Metrics.OTLP.GRPC.Insecure {
return errors.New("metrics OTLP GRPC: TLS and Insecure options are mutually exclusive")
}
}
return nil
}

29
pkg/metrics/opentelemetry.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"net"
"net/url"
"strings"
"sync"
"time"
@ -30,7 +31,7 @@ var (
)
// RegisterOpenTelemetry registers all OpenTelemetry metrics.
func RegisterOpenTelemetry(ctx context.Context, config *types.OpenTelemetry) Registry {
func RegisterOpenTelemetry(ctx context.Context, config *types.OTLP) Registry {
if openTelemetryMeterProvider == nil {
var err error
if openTelemetryMeterProvider, err = newOpenTelemetryMeterProvider(ctx, config); err != nil {
@ -123,15 +124,15 @@ func StopOpenTelemetry() {
}
// newOpenTelemetryMeterProvider creates a new controller.Controller.
func newOpenTelemetryMeterProvider(ctx context.Context, config *types.OpenTelemetry) (*sdkmetric.MeterProvider, error) {
func newOpenTelemetryMeterProvider(ctx context.Context, config *types.OTLP) (*sdkmetric.MeterProvider, error) {
var (
exporter sdkmetric.Exporter
err error
)
if config.GRPC != nil {
exporter, err = newGRPCExporter(ctx, config)
exporter, err = newGRPCExporter(ctx, config.GRPC)
} else {
exporter, err = newHTTPExporter(ctx, config)
exporter, err = newHTTPExporter(ctx, config.HTTP)
}
if err != nil {
return nil, fmt.Errorf("creating exporter: %w", err)
@ -168,24 +169,24 @@ func newOpenTelemetryMeterProvider(ctx context.Context, config *types.OpenTeleme
return meterProvider, nil
}
func newHTTPExporter(ctx context.Context, config *types.OpenTelemetry) (sdkmetric.Exporter, error) {
host, port, err := net.SplitHostPort(config.Address)
func newHTTPExporter(ctx context.Context, config *types.OtelHTTP) (sdkmetric.Exporter, error) {
endpoint, err := url.Parse(config.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector address %q: %w", config.Address, err)
return nil, fmt.Errorf("invalid collector endpoint %q: %w", config.Endpoint, err)
}
opts := []otlpmetrichttp.Option{
otlpmetrichttp.WithEndpoint(fmt.Sprintf("%s:%s", host, port)),
otlpmetrichttp.WithEndpoint(endpoint.Host),
otlpmetrichttp.WithHeaders(config.Headers),
otlpmetrichttp.WithCompression(otlpmetrichttp.GzipCompression),
}
if config.Insecure {
if endpoint.Scheme == "http" {
opts = append(opts, otlpmetrichttp.WithInsecure())
}
if config.Path != "" {
opts = append(opts, otlpmetrichttp.WithURLPath(config.Path))
if endpoint.Path != "" {
opts = append(opts, otlpmetrichttp.WithURLPath(endpoint.Path))
}
if config.TLS != nil {
@ -200,10 +201,10 @@ func newHTTPExporter(ctx context.Context, config *types.OpenTelemetry) (sdkmetri
return otlpmetrichttp.New(ctx, opts...)
}
func newGRPCExporter(ctx context.Context, config *types.OpenTelemetry) (sdkmetric.Exporter, error) {
host, port, err := net.SplitHostPort(config.Address)
func newGRPCExporter(ctx context.Context, config *types.OtelGRPC) (sdkmetric.Exporter, error) {
host, port, err := net.SplitHostPort(config.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector address %q: %w", config.Address, err)
return nil, fmt.Errorf("invalid collector endpoint %q: %w", config.Endpoint, err)
}
opts := []otlpmetricgrpc.Option{

11
pkg/metrics/opentelemetry_test.go
View file

@ -8,7 +8,6 @@ import (
"io"
"net/http"
"net/http/httptest"
"net/url"
"regexp"
"strconv"
"testing"
@ -316,14 +315,12 @@ func TestOpenTelemetry(t *testing.T) {
ts.Close()
})
sURL, err := url.Parse(ts.URL)
require.NoError(t, err)
var cfg types.OpenTelemetry
var cfg types.OTLP
(&cfg).SetDefaults()
cfg.AddRoutersLabels = true
cfg.Address = sURL.Host
cfg.Insecure = true
cfg.HTTP = &types.OtelHTTP{
Endpoint: ts.URL,
}
cfg.PushInterval = ptypes.Duration(10 * time.Millisecond)
registry := RegisterOpenTelemetry(context.Background(), &cfg)

3
pkg/middlewares/auth/forward_test.go
View file

@ -17,6 +17,7 @@ import (
"github.com/traefik/traefik/v3/pkg/testhelpers"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/tracing/opentelemetry"
"github.com/traefik/traefik/v3/pkg/types"
"github.com/traefik/traefik/v3/pkg/version"
"github.com/vulcand/oxy/v2/forward"
"go.opentelemetry.io/otel"
@ -500,7 +501,7 @@ func TestForwardAuthUsesTracing(t *testing.T) {
ServiceName: "testApp",
SampleRate: 1,
OTLP: &opentelemetry.Config{
HTTP: &opentelemetry.HTTP{
HTTP: &types.OtelHTTP{
Endpoint: "http://127.0.0.1:8080",
},
},

8
pkg/redactor/redactor_config_test.go
View file

@ -846,17 +846,17 @@ func TestDo_staticConfiguration(t *testing.T) {
config.Tracing = &static.Tracing{
ServiceName: "myServiceName",
Headers: map[string]string{
"foobar": "foobar",
},
GlobalAttributes: map[string]string{
"foobar": "foobar",
},
SampleRate: 42,
OTLP: &opentelemetry.Config{
HTTP: &opentelemetry.HTTP{
HTTP: &types.OtelHTTP{
Endpoint: "foobar",
TLS: nil,
Headers: map[string]string{
"foobar": "foobar",
},
},
},
}

3
pkg/redactor/testdata/anonymized-static-config.json vendored
View file

@ -344,9 +344,6 @@
},
"tracing": {
"serviceName": "myServiceName",
"headers": {
"foobar": "foobar"
},
"globalAttributes": {
"foobar": "foobar"
},

44
pkg/tracing/opentelemetry/opentelemetry.go
View file

@ -27,44 +27,26 @@ import (
// Config provides configuration settings for the open-telemetry tracer.
type Config struct {
GRPC *GRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" export:"true"`
HTTP *HTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" export:"true"`
}
// GRPC provides configuration settings for the gRPC open-telemetry tracer.
type GRPC struct {
Endpoint string `description:"Sets the gRPC endpoint (host:port) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
Insecure bool `description:"Disables client transport security for the exporter." json:"insecure,omitempty" toml:"insecure,omitempty" yaml:"insecure,omitempty" export:"true"`
TLS *types.ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
GRPC *types.OtelGRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" export:"true"`
HTTP *types.OtelHTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" export:"true"`
}
// SetDefaults sets the default values.
func (c *GRPC) SetDefaults() {
c.Endpoint = "localhost:4317"
}
// HTTP provides configuration settings for the HTTP open-telemetry tracer.
type HTTP struct {
Endpoint string `description:"Sets the HTTP endpoint (scheme://host:port/v1/traces) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
TLS *types.ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
}
// SetDefaults sets the default values.
func (c *HTTP) SetDefaults() {
c.Endpoint = "localhost:4318"
func (c *Config) SetDefaults() {
c.HTTP = &types.OtelHTTP{}
c.HTTP.SetDefaults()
}
// Setup sets up the tracer.
func (c *Config) Setup(serviceName string, sampleRate float64, globalAttributes map[string]string, headers map[string]string) (trace.Tracer, io.Closer, error) {
func (c *Config) Setup(serviceName string, sampleRate float64, globalAttributes map[string]string) (trace.Tracer, io.Closer, error) {
var (
err error
exporter *otlptrace.Exporter
)
if c.GRPC != nil {
exporter, err = c.setupGRPCExporter(headers)
exporter, err = c.setupGRPCExporter()
} else {
exporter, err = c.setupHTTPExporter(headers)
exporter, err = c.setupHTTPExporter()
}
if err != nil {
return nil, nil, fmt.Errorf("setting up exporter: %w", err)
@ -107,7 +89,7 @@ func (c *Config) Setup(serviceName string, sampleRate float64, globalAttributes
return tracerProvider.Tracer("github.com/traefik/traefik"), &tpCloser{provider: tracerProvider}, err
}
func (c *Config) setupHTTPExporter(headers map[string]string) (*otlptrace.Exporter, error) {
func (c *Config) setupHTTPExporter() (*otlptrace.Exporter, error) {
endpoint, err := url.Parse(c.HTTP.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector endpoint %q: %w", c.HTTP.Endpoint, err)
@ -115,7 +97,7 @@ func (c *Config) setupHTTPExporter(headers map[string]string) (*otlptrace.Export
opts := []otlptracehttp.Option{
otlptracehttp.WithEndpoint(endpoint.Host),
otlptracehttp.WithHeaders(headers),
otlptracehttp.WithHeaders(c.HTTP.Headers),
otlptracehttp.WithCompression(otlptracehttp.GzipCompression),
}
@ -139,15 +121,15 @@ func (c *Config) setupHTTPExporter(headers map[string]string) (*otlptrace.Export
return otlptrace.New(context.Background(), otlptracehttp.NewClient(opts...))
}
func (c *Config) setupGRPCExporter(headers map[string]string) (*otlptrace.Exporter, error) {
func (c *Config) setupGRPCExporter() (*otlptrace.Exporter, error) {
host, port, err := net.SplitHostPort(c.GRPC.Endpoint)
if err != nil {
return nil, fmt.Errorf("invalid collector address %q: %w", c.GRPC.Endpoint, err)
return nil, fmt.Errorf("invalid collector endpoint %q: %w", c.GRPC.Endpoint, err)
}
opts := []otlptracegrpc.Option{
otlptracegrpc.WithEndpoint(fmt.Sprintf("%s:%s", host, port)),
otlptracegrpc.WithHeaders(headers),
otlptracegrpc.WithHeaders(c.GRPC.Headers),
otlptracegrpc.WithCompressor(gzip.Name),
}

3
pkg/tracing/opentelemetry/opentelemetry_test.go
View file

@ -17,6 +17,7 @@ import (
tracingMiddle "github.com/traefik/traefik/v3/pkg/middlewares/tracing"
"github.com/traefik/traefik/v3/pkg/tracing"
"github.com/traefik/traefik/v3/pkg/tracing/opentelemetry"
"github.com/traefik/traefik/v3/pkg/types"
"go.opentelemetry.io/collector/pdata/ptrace/ptraceotlp"
)
@ -74,7 +75,7 @@ func TestTracing(t *testing.T) {
ServiceName: "traefik",
SampleRate: 1.0,
OTLP: &opentelemetry.Config{
HTTP: &opentelemetry.HTTP{
HTTP: &types.OtelHTTP{
Endpoint: collector.URL,
},
},

4
pkg/tracing/tracing.go
View file

@ -20,7 +20,7 @@ import (
// Backend is an abstraction for tracking backend (OpenTelemetry, ...).
type Backend interface {
Setup(serviceName string, sampleRate float64, globalAttributes map[string]string, headers map[string]string) (trace.Tracer, io.Closer, error)
Setup(serviceName string, sampleRate float64, globalAttributes map[string]string) (trace.Tracer, io.Closer, error)
}
// NewTracing Creates a Tracing.
@ -37,7 +37,7 @@ func NewTracing(conf *static.Tracing) (trace.Tracer, io.Closer, error) {
backend = defaultBackend
}
return backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes, conf.Headers)
return backend.Setup(conf.ServiceName, conf.SampleRate, conf.GlobalAttributes)
}
// TracerFromContext extracts the trace.Tracer from the given context.

64
pkg/types/metrics.go
View file

@ -12,11 +12,11 @@ import (
type Metrics struct {
AddInternals bool `description:"Enables metrics for internal services (ping, dashboard, etc...)." json:"addInternals,omitempty" toml:"addInternals,omitempty" yaml:"addInternals,omitempty" export:"true"`
Prometheus *Prometheus `description:"Prometheus metrics exporter type." json:"prometheus,omitempty" toml:"prometheus,omitempty" yaml:"prometheus,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
Datadog *Datadog `description:"Datadog metrics exporter type." json:"datadog,omitempty" toml:"datadog,omitempty" yaml:"datadog,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
StatsD *Statsd `description:"StatsD metrics exporter type." json:"statsD,omitempty" toml:"statsD,omitempty" yaml:"statsD,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
InfluxDB2 *InfluxDB2 `description:"InfluxDB v2 metrics exporter type." json:"influxDB2,omitempty" toml:"influxDB2,omitempty" yaml:"influxDB2,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
OpenTelemetry *OpenTelemetry `description:"OpenTelemetry metrics exporter type." json:"openTelemetry,omitempty" toml:"openTelemetry,omitempty" yaml:"openTelemetry,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
Prometheus *Prometheus `description:"Prometheus metrics exporter type." json:"prometheus,omitempty" toml:"prometheus,omitempty" yaml:"prometheus,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
Datadog *Datadog `description:"Datadog metrics exporter type." json:"datadog,omitempty" toml:"datadog,omitempty" yaml:"datadog,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
StatsD *Statsd `description:"StatsD metrics exporter type." json:"statsD,omitempty" toml:"statsD,omitempty" yaml:"statsD,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
InfluxDB2 *InfluxDB2 `description:"InfluxDB v2 metrics exporter type." json:"influxDB2,omitempty" toml:"influxDB2,omitempty" yaml:"influxDB2,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
OTLP *OTLP `description:"OpenTelemetry metrics exporter type." json:"otlp,omitempty" toml:"otlp,omitempty" yaml:"otlp,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
}
// Prometheus can contain specific configuration used by the Prometheus Metrics exporter.
@ -106,26 +106,23 @@ func (i *InfluxDB2) SetDefaults() {
i.AddServicesLabels = true
}
// OpenTelemetry contains specific configuration used by the OpenTelemetry Metrics exporter.
type OpenTelemetry struct {
// NOTE: as no gRPC option is implemented yet, the type is empty and is used as a boolean for upward compatibility purposes.
GRPC *struct{} `description:"gRPC specific configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" label:"allowEmpty" file:"allowEmpty" export:"true"`
// OTLP contains specific configuration used by the OpenTelemetry Metrics exporter.
type OTLP struct {
GRPC *OtelGRPC `description:"gRPC configuration for the OpenTelemetry collector." json:"grpc,omitempty" toml:"grpc,omitempty" yaml:"grpc,omitempty" export:"true"`
HTTP *OtelHTTP `description:"HTTP configuration for the OpenTelemetry collector." json:"http,omitempty" toml:"http,omitempty" yaml:"http,omitempty" export:"true"`
Address string `description:"Address (host:port) of the collector endpoint." json:"address,omitempty" toml:"address,omitempty" yaml:"address,omitempty"`
AddEntryPointsLabels bool `description:"Enable metrics on entry points." json:"addEntryPointsLabels,omitempty" toml:"addEntryPointsLabels,omitempty" yaml:"addEntryPointsLabels,omitempty" export:"true"`
AddRoutersLabels bool `description:"Enable metrics on routers." json:"addRoutersLabels,omitempty" toml:"addRoutersLabels,omitempty" yaml:"addRoutersLabels,omitempty" export:"true"`
AddServicesLabels bool `description:"Enable metrics on services." json:"addServicesLabels,omitempty" toml:"addServicesLabels,omitempty" yaml:"addServicesLabels,omitempty" export:"true"`
ExplicitBoundaries []float64 `description:"Boundaries for latency metrics." json:"explicitBoundaries,omitempty" toml:"explicitBoundaries,omitempty" yaml:"explicitBoundaries,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty" export:"true"`
Insecure bool `description:"Disables client transport security for the exporter." json:"insecure,omitempty" toml:"insecure,omitempty" yaml:"insecure,omitempty" export:"true"`
Path string `description:"Set the URL path of the collector endpoint." json:"path,omitempty" toml:"path,omitempty" yaml:"path,omitempty" export:"true"`
PushInterval types.Duration `description:"Period between calls to collect a checkpoint." json:"pushInterval,omitempty" toml:"pushInterval,omitempty" yaml:"pushInterval,omitempty" export:"true"`
TLS *ClientTLS `description:"Enable TLS support." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
AddEntryPointsLabels bool `description:"Enable metrics on entry points." json:"addEntryPointsLabels,omitempty" toml:"addEntryPointsLabels,omitempty" yaml:"addEntryPointsLabels,omitempty" export:"true"`
AddRoutersLabels bool `description:"Enable metrics on routers." json:"addRoutersLabels,omitempty" toml:"addRoutersLabels,omitempty" yaml:"addRoutersLabels,omitempty" export:"true"`
AddServicesLabels bool `description:"Enable metrics on services." json:"addServicesLabels,omitempty" toml:"addServicesLabels,omitempty" yaml:"addServicesLabels,omitempty" export:"true"`
ExplicitBoundaries []float64 `description:"Boundaries for latency metrics." json:"explicitBoundaries,omitempty" toml:"explicitBoundaries,omitempty" yaml:"explicitBoundaries,omitempty" export:"true"`
PushInterval types.Duration `description:"Period between calls to collect a checkpoint." json:"pushInterval,omitempty" toml:"pushInterval,omitempty" yaml:"pushInterval,omitempty" export:"true"`
}
// SetDefaults sets the default values.
func (o *OpenTelemetry) SetDefaults() {
o.Address = "localhost:4318"
func (o *OTLP) SetDefaults() {
o.HTTP = &OtelHTTP{}
o.HTTP.SetDefaults()
o.AddEntryPointsLabels = true
o.AddServicesLabels = true
o.ExplicitBoundaries = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10}
@ -141,3 +138,28 @@ type Statistics struct {
func (s *Statistics) SetDefaults() {
s.RecentErrors = 10
}
// OtelGRPC provides configuration settings for the gRPC open-telemetry.
type OtelGRPC struct {
Endpoint string `description:"Sets the gRPC endpoint (host:port) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
Insecure bool `description:"Disables client transport security for the exporter." json:"insecure,omitempty" toml:"insecure,omitempty" yaml:"insecure,omitempty" export:"true"`
TLS *ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty"`
}
// SetDefaults sets the default values.
func (c *OtelGRPC) SetDefaults() {
c.Endpoint = "localhost:4317"
}
// OtelHTTP provides configuration settings for the HTTP open-telemetry.
type OtelHTTP struct {
Endpoint string `description:"Sets the HTTP endpoint (scheme://host:port/path) of the collector." json:"endpoint,omitempty" toml:"endpoint,omitempty" yaml:"endpoint,omitempty"`
TLS *ClientTLS `description:"Defines client transport security parameters." json:"tls,omitempty" toml:"tls,omitempty" yaml:"tls,omitempty" export:"true"`
Headers map[string]string `description:"Headers sent with payload." json:"headers,omitempty" toml:"headers,omitempty" yaml:"headers,omitempty"`
}
// SetDefaults sets the default values.
func (c *OtelHTTP) SetDefaults() {
c.Endpoint = "https://localhost:4318"
}