From 18828ca7207fbd11662b40ca5a9759ea6cdf348a Mon Sep 17 00:00:00 2001 From: "Arthur K." Date: Tue, 21 Jan 2025 14:21:40 +0300 Subject: [PATCH] feat: update Caddy config, add cache, improve response handling and logging --- caddy/Caddyfile | 35 ++++++-- caddy/Caddyfile.puppies | 6 -- caddy/Dockerfile | 8 ++ docker-compose.yml | 8 +- proxy/Dockerfile | 10 +++ proxy/main.go | 191 +++++++++++++++++++++++----------------- 6 files changed, 162 insertions(+), 96 deletions(-) delete mode 100644 caddy/Caddyfile.puppies create mode 100644 caddy/Dockerfile create mode 100644 proxy/Dockerfile diff --git a/caddy/Caddyfile b/caddy/Caddyfile index a729538..38827b2 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -1,10 +1,35 @@ -https://, ядро.орг, *.ядро.орг { - tls internal +{ + cache + log { + format console + level WARN + } +} + +http://ядро.орг, http://*.ядро.орг { + tls internal encode gzip zstd - import Caddyfile.yadro localhost:8000 + cache { + # we don't want to flood the upstream from the same IP + mode bypass + ttl 30m + } + + import Caddyfile.yadro proxy:80 } -http://localhost:9000 { - import Caddyfile.puppies localhost:9001-9010 +:9000 { + reverse_proxy { + dynamic a puppy 80 + lb_policy least_conn + } + + cache { + allowed_http_verbs POST + ttl 7d + timeout { + backend 1m + } + } } diff --git a/caddy/Caddyfile.puppies b/caddy/Caddyfile.puppies deleted file mode 100644 index 38621fc..0000000 --- a/caddy/Caddyfile.puppies +++ /dev/null @@ -1,6 +0,0 @@ -reverse_proxy { - to {args[0]} - lb_policy least_conn -} - -# TODO: infinite cache by path+body diff --git a/caddy/Dockerfile b/caddy/Dockerfile new file mode 100644 index 0000000..3d21c17 --- /dev/null +++ b/caddy/Dockerfile @@ -0,0 +1,8 @@ +FROM caddy:builder AS builder + +RUN --mount=type=cache,target=/go/pkg/mod xcaddy build \ + --with github.com/caddyserver/cache-handler + +FROM caddy:latest + +COPY --from=builder /usr/bin/caddy /usr/bin/caddy diff --git a/docker-compose.yml b/docker-compose.yml index c46405c..1bb0852 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,14 +3,14 @@ services: networks: - promty build: puppy - ports: - - "127.0.0.1:9000-9002:80" + hostname: puppy stop_signal: SIGINT volumes: - /dev/shm/puppy-temp:/tmpfs + - ./cache:/cache deploy: mode: replicated - replicas: 2 + replicas: 1 endpoint_mode: vip proxy: @@ -23,7 +23,7 @@ services: networks: - promty container_name: caddy - image: caddy:2.8-alpine + build: caddy volumes: - ./caddy:/etc/caddy ports: diff --git a/proxy/Dockerfile b/proxy/Dockerfile new file mode 100644 index 0000000..52f9945 --- /dev/null +++ b/proxy/Dockerfile @@ -0,0 +1,10 @@ +FROM golang:1.23-alpine AS builder +WORKDIR /build +COPY . . +RUN --mount=type=cache,target=/go/pkg/mod go build + +FROM alpine AS runner +WORKDIR /app +COPY --from=builder /build/proxy . +EXPOSE 80/tcp +CMD ./proxy diff --git a/proxy/main.go b/proxy/main.go index 94f6ee6..51f9d5a 100644 --- a/proxy/main.go +++ b/proxy/main.go @@ -1,115 +1,144 @@ package main import ( - "fmt" - "io" - "bytes" - "regexp" - "strings" - "compress/gzip" - "net/url" - "net/http" - "net/http/httputil" - "golang.org/x/text/encoding/charmap" + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" + "os" + "regexp" + "strings" ) func main() { - proxy := &httputil.ReverseProxy{ - Rewrite: func(r *httputil.ProxyRequest) { - host, _ := Yadro2Kernel(r.In.Host, true) - url, err := url.Parse("https://" + host) - - if err != nil { - // TODO... - } + proxy := &httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + host, _ := Yadro2Kernel(r.In.Host, true) + url, _ := url.Parse("https://" + host) - r.SetURL(url) + r.SetURL(url) - // We only support gzip decoding - r.Out.Header.Set("Accept-Encoding", "gzip") - }, - ModifyResponse: func(r *http.Response) error { - // Disable security policy because of the domain restrictions. - r.Header.Del("Content-Security-Policy") + // We only support gzip decoding + r.Out.Header.Set("Accept-Encoding", "gzip") + }, + ModifyResponse: func(r *http.Response) error { + // Disable security policy because of the domain restrictions. + r.Header.Del("Content-Security-Policy") - // Skip non-html pages. - contentType := r.Header.Get("Content-Type") - if !strings.HasPrefix(contentType, "text/") { - return nil - } + // Skip non-html pages. + contentType := r.Header.Get("Content-Type") + if !strings.HasPrefix(contentType, "text/") { + return nil + } + r.Header.Set("Content-Type", contentType+";charset=utf-8") - // Read response body into data. If body is encoded, decode it. - var data []byte + // Read response body into data. If body is encoded, decode it. + var data []byte - switch r.Header.Get("Content-Encoding") { - case "gzip": - reader, _ := gzip.NewReader(r.Body) - data, _ = io.ReadAll(reader) - r.Body.Close() - default: - data, _ = io.ReadAll(r.Body) - r.Body.Close() - } + switch r.Header.Get("Content-Encoding") { + case "gzip": + reader, _ := gzip.NewReader(r.Body) + data, _ = io.ReadAll(reader) + r.Body.Close() + default: + data, _ = io.ReadAll(r.Body) + r.Body.Close() + } - // Modify the response. - data = modifyResponse(data) + // Rewrite 30x redirect location + locHeader := r.Header.Get("Location") + if locHeader != "" { + re := regexp.MustCompile(`https?:\/\/[A-Za-z\-\.]*.?kernel\.org\/`) + r.Header.Set("Location", string(re.ReplaceAllFunc([]byte(locHeader), func(original_raw []byte) []byte { + kernel := strings.ToLower(string(original_raw)) - // Remove headers that mess with body encoding and set the body. - r.Header.Del("Content-Encoding") - r.Header.Del("Content-Length") - // r.Header.Set("Content-Type", "text/html; charset=windows-1251") + kernel = strings.TrimPrefix(kernel, "https://") + kernel = strings.TrimPrefix(kernel, "http://") + kernel = strings.TrimPrefix(kernel, "www.") + kernel = strings.TrimSuffix(kernel, "/") - r.Body = io.NopCloser(bytes.NewReader(data)) - - return nil - }, - } + yadro, exists := Kernel2Yadro(kernel) - http.ListenAndServe("localhost:8000", proxy) + if !exists { + fmt.Println("Missing:", kernel) + return original_raw + } + + return []byte("http://" + yadro + "/") // TODO: https + }))) + } + + // Modify the response. + data = modifyResponse(data) + + // Remove headers that mess with body encoding and set the body. + r.Header.Del("Content-Encoding") + r.Header.Del("Content-Length") + + r.Body = io.NopCloser(bytes.NewReader(data)) + + return nil + }, + } + + http.ListenAndServe("0.0.0.0:80", proxy) } func replaceDomains(response []byte) []byte { - re := regexp.MustCompile(`(?i)[A-Za-z\-\.]*\.?kernel\.org`) - response = re.ReplaceAllFunc(response, func(original_raw []byte) []byte { - kernel := strings.ToLower(string(original_raw)) + re := regexp.MustCompile(`(?i)[A-Za-z\-\.]*\.?kernel\.org`) - // Strip `www.` - kernel = strings.TrimPrefix(kernel, "www.") + response = re.ReplaceAllFunc(response, func(original_raw []byte) []byte { + kernel := strings.ToLower(string(original_raw)) - yadro, exists := Kernel2Yadro(kernel) + // Strip `www.` + kernel = strings.TrimPrefix(kernel, "www.") - if !exists { - return original_raw - } + yadro, exists := Kernel2Yadro(kernel) - return []byte(yadro) - }) + if !exists { + fmt.Println("Missing:", kernel) + return original_raw + } - return response + return []byte(yadro) + }) + + response = bytes.ReplaceAll(response, []byte("%3F"), []byte("?")) + response = bytes.ReplaceAll(response, []byte("%26"), []byte("&")) + response = bytes.ReplaceAll(response, []byte("https"), []byte("http")) // TODO: TEMP + + return response } func translateWithPromtPuppies(response []byte) []byte { - enc := charmap.Windows1251.NewEncoder() - cp1521, _ := enc.Bytes(response) + // Don't try to translate empty body (30x, etc) + if len(response) == 0 { + return response + } - req, _ := http.NewRequest("POST", "http://localhost:2390/", bytes.NewReader(cp1521)) // TODO + req, _ := http.NewRequest("POST", "http://caddy:9000/translate", bytes.NewReader(response)) - resp, err := http.DefaultClient.Do(req) - fmt.Println(err) - fmt.Println(resp.StatusCode) + req.Header.Add("Content-Type", "text/html") - response, _ = io.ReadAll(resp.Body) - dec, _ := charmap.Windows1251.NewDecoder().Bytes(response) - // fmt.Println(len(dec)) - // fmt.Println(string(dec)) + resp, err := http.DefaultClient.Do(req) - resp.Body.Close() - - return dec + if err != nil { + fmt.Fprintln(os.Stderr, "Error in first", err) + return []byte{0} + } + + response, _ = io.ReadAll(resp.Body) + + resp.Body.Close() + + return response } func modifyResponse(response []byte) []byte { - response = replaceDomains(response) - response = translateWithPromtPuppies(response) - return response + response = translateWithPromtPuppies(response) + response = replaceDomains(response) + return response }