feat: update Caddy config, add cache, improve response handling and logging

caddy/Caddyfile

@@ -1,10 +1,35 @@
-https://, ядро.орг, *.ядро.орг {
+{
-	tls internal
+    cache
+    log {
+        format console
+        level WARN
+    }
+}
+
+http://ядро.орг, http://*.ядро.орг {
+    tls internal
     encode gzip zstd
 
-    import Caddyfile.yadro localhost:8000
+    cache {
+        # we don't want to flood the upstream from the same IP
+        mode bypass
+        ttl 30m
+    }
+
+    import Caddyfile.yadro proxy:80
 }
 
-http://localhost:9000 {
+:9000 {
-    import Caddyfile.puppies localhost:9001-9010
+    reverse_proxy {
+        dynamic a puppy 80
+        lb_policy least_conn
+    }
+
+    cache {
+        allowed_http_verbs POST
+        ttl 7d
+        timeout {
+            backend 1m
+        }
+    }
 }

caddy/Caddyfile.puppies

@@ -1,6 +0,0 @@
-reverse_proxy {
-    to {args[0]}
-    lb_policy least_conn
-}
-
-# TODO: infinite cache by path+body

caddy/Dockerfile

@@ -0,0 +1,8 @@
+FROM caddy:builder AS builder
+
+RUN --mount=type=cache,target=/go/pkg/mod xcaddy build \
+    --with github.com/caddyserver/cache-handler
+
+FROM caddy:latest
+
+COPY --from=builder /usr/bin/caddy /usr/bin/caddy

docker-compose.yml

@@ -3,14 +3,14 @@ services:
     networks:
       - promty
     build: puppy
-    ports:
+    hostname: puppy
-      - "127.0.0.1:9000-9002:80"
     stop_signal: SIGINT
     volumes:
       - /dev/shm/puppy-temp:/tmpfs
+      - ./cache:/cache
     deploy:
       mode: replicated
-      replicas: 2
+      replicas: 1
       endpoint_mode: vip
 
   proxy:
@@ -23,7 +23,7 @@ services:
     networks:
       - promty
     container_name: caddy
-    image: caddy:2.8-alpine
+    build: caddy
     volumes:
       - ./caddy:/etc/caddy
     ports:

proxy/Dockerfile

@@ -0,0 +1,10 @@
+FROM golang:1.23-alpine AS builder
+WORKDIR /build
+COPY . .
+RUN --mount=type=cache,target=/go/pkg/mod go build
+
+FROM alpine AS runner
+WORKDIR /app
+COPY --from=builder /build/proxy .
+EXPOSE 80/tcp
+CMD ./proxy

proxy/main.go

@@ -1,115 +1,144 @@
 package main
 
 import (
-    "fmt"
+	"bytes"
-    "io"
+	"compress/gzip"
-    "bytes"
+	"fmt"
-    "regexp"
+	"io"
-    "strings"
+	"net/http"
-    "compress/gzip"
+	"net/http/httputil"
-    "net/url"
+	"net/url"
-    "net/http"
+	"os"
-    "net/http/httputil"
+	"regexp"
-    "golang.org/x/text/encoding/charmap"
+	"strings"
 )
 
 func main() {
-    proxy := &httputil.ReverseProxy{
+	proxy := &httputil.ReverseProxy{
-        Rewrite: func(r *httputil.ProxyRequest) {
+		Rewrite: func(r *httputil.ProxyRequest) {
-            host, _ := Yadro2Kernel(r.In.Host, true)
+			host, _ := Yadro2Kernel(r.In.Host, true)
-            url, err := url.Parse("https://" + host)
+			url, _ := url.Parse("https://" + host)
 
-            if err != nil {
+			r.SetURL(url)
-                // TODO...
-            }
 
-            r.SetURL(url)
+			// We only support gzip decoding
+			r.Out.Header.Set("Accept-Encoding", "gzip")
+		},
+		ModifyResponse: func(r *http.Response) error {
+			// Disable security policy because of the domain restrictions.
+			r.Header.Del("Content-Security-Policy")
 
-            // We only support gzip decoding
+			// Skip non-html pages.
-            r.Out.Header.Set("Accept-Encoding", "gzip")
+			contentType := r.Header.Get("Content-Type")
-        },
+			if !strings.HasPrefix(contentType, "text/") {
-        ModifyResponse: func(r *http.Response) error {
+				return nil
-            // Disable security policy because of the domain restrictions.
+			}
-            r.Header.Del("Content-Security-Policy")
+			r.Header.Set("Content-Type", contentType+";charset=utf-8")
 
-            // Skip non-html pages.
+			// Read response body into data. If body is encoded, decode it.
-            contentType := r.Header.Get("Content-Type")
+			var data []byte
-            if !strings.HasPrefix(contentType, "text/") {
-                return nil
-            }
 
-            // Read response body into data. If body is encoded, decode it.
+			switch r.Header.Get("Content-Encoding") {
-            var data []byte
+			case "gzip":
+				reader, _ := gzip.NewReader(r.Body)
+				data, _ = io.ReadAll(reader)
+				r.Body.Close()
+			default:
+				data, _ = io.ReadAll(r.Body)
+				r.Body.Close()
+			}
 
-            switch r.Header.Get("Content-Encoding") {
+			// Rewrite 30x redirect location
-            case "gzip":
+			locHeader := r.Header.Get("Location")
-                reader, _ := gzip.NewReader(r.Body)
+			if locHeader != "" {
-                data, _ = io.ReadAll(reader)
+				re := regexp.MustCompile(`https?:\/\/[A-Za-z\-\.]*.?kernel\.org\/`)
-                r.Body.Close()
+				r.Header.Set("Location", string(re.ReplaceAllFunc([]byte(locHeader), func(original_raw []byte) []byte {
-            default:
+					kernel := strings.ToLower(string(original_raw))
-                data, _ = io.ReadAll(r.Body)
-                r.Body.Close()
-            }
 
-            // Modify the response.
+					kernel = strings.TrimPrefix(kernel, "https://")
-            data = modifyResponse(data)
+					kernel = strings.TrimPrefix(kernel, "http://")
+					kernel = strings.TrimPrefix(kernel, "www.")
+					kernel = strings.TrimSuffix(kernel, "/")
 
-            // Remove headers that mess with body encoding and set the body.
+					yadro, exists := Kernel2Yadro(kernel)
-            r.Header.Del("Content-Encoding")
-            r.Header.Del("Content-Length")
-            // r.Header.Set("Content-Type", "text/html; charset=windows-1251")
 
-            r.Body = io.NopCloser(bytes.NewReader(data))
+					if !exists {
+						fmt.Println("Missing:", kernel)
+						return original_raw
+					}
 
-            return nil
+					return []byte("http://" + yadro + "/") // TODO: https
-        },
+				})))
-    }
+			}
 
-    http.ListenAndServe("localhost:8000", proxy)
+			// Modify the response.
+			data = modifyResponse(data)
+
+			// Remove headers that mess with body encoding and set the body.
+			r.Header.Del("Content-Encoding")
+			r.Header.Del("Content-Length")
+
+			r.Body = io.NopCloser(bytes.NewReader(data))
+
+			return nil
+		},
+	}
+
+	http.ListenAndServe("0.0.0.0:80", proxy)
 }
 
 func replaceDomains(response []byte) []byte {
-    re := regexp.MustCompile(`(?i)[A-Za-z\-\.]*\.?kernel\.org`)
+	re := regexp.MustCompile(`(?i)[A-Za-z\-\.]*\.?kernel\.org`)
-    response = re.ReplaceAllFunc(response, func(original_raw []byte) []byte {
-        kernel := strings.ToLower(string(original_raw))
 
-        // Strip `www.`
+	response = re.ReplaceAllFunc(response, func(original_raw []byte) []byte {
-        kernel = strings.TrimPrefix(kernel, "www.")
+		kernel := strings.ToLower(string(original_raw))
 
-        yadro, exists := Kernel2Yadro(kernel)
+		// Strip `www.`
+		kernel = strings.TrimPrefix(kernel, "www.")
 
-        if !exists {
+		yadro, exists := Kernel2Yadro(kernel)
-            return original_raw
-        }
 
-        return []byte(yadro)
+		if !exists {
-    })
+			fmt.Println("Missing:", kernel)
+			return original_raw
+		}
 
-    return response
+		return []byte(yadro)
+	})
+
+	response = bytes.ReplaceAll(response, []byte("%3F"), []byte("?"))
+	response = bytes.ReplaceAll(response, []byte("%26"), []byte("&"))
+	response = bytes.ReplaceAll(response, []byte("https"), []byte("http")) // TODO: TEMP
+
+	return response
 }
 
 func translateWithPromtPuppies(response []byte) []byte {
-    enc := charmap.Windows1251.NewEncoder()
+	// Don't try to translate empty body (30x, etc)
-    cp1521, _ := enc.Bytes(response)
+	if len(response) == 0 {
+		return response
+	}
 
-    req, _ := http.NewRequest("POST", "http://localhost:2390/", bytes.NewReader(cp1521))  // TODO
+	req, _ := http.NewRequest("POST", "http://caddy:9000/translate", bytes.NewReader(response))
 
-    resp, err := http.DefaultClient.Do(req)
+	req.Header.Add("Content-Type", "text/html")
-    fmt.Println(err)
-    fmt.Println(resp.StatusCode)
 
-    response, _ = io.ReadAll(resp.Body)
+	resp, err := http.DefaultClient.Do(req)
-    dec, _ := charmap.Windows1251.NewDecoder().Bytes(response)
-    // fmt.Println(len(dec))
-    // fmt.Println(string(dec))
 
-    resp.Body.Close()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "Error in first", err)
+		return []byte{0}
+	}
 
-    return dec
+	response, _ = io.ReadAll(resp.Body)
+
+	resp.Body.Close()
+
+	return response
 }
 
 func modifyResponse(response []byte) []byte {
-    response = replaceDomains(response)
+	response = translateWithPromtPuppies(response)
-    response = translateWithPromtPuppies(response)
+	response = replaceDomains(response)
-    return response
+	return response
 }