feat: harden forwarded headers and automate Cloudflare IP sync

README.md

@@ -80,4 +80,17 @@ labels:
 
 I mount the access.log for crowdsec firewall to read. 
 
-PS: Because I access my traefik dashboard through my local network. I commented out the authetication method for dashboard. 
+PS: Because I access my traefik dashboard through my local network. I commented out the authetication method for dashboard. 
+
+## Discussion and Changelog
+
+1. Traefik vs Nginx
+- Performance: Nginx is still better at high traffic. After all it is written in C. Traefik 3 though claims it has higher 20% performance than before. The latency still showed a little higher than nginx. 
+- Docker Deployment Ease: Traefik is easier for docker service deployment. In my environment, I can assign each docker stack with labels and then guides the traefik to add Let's encrypt SSL. 
+
+2. ChangeLog: 
+
+- 2025.4.21 Add the defaulthost rule for container name for lazy writing. But commented out for precision.  
+- 2025.4.21 Fix the trused IP settings to let the traefik-plugin-cloudflare tackle it. 
+- 2025.4.18 Add Souin HTTP Cache Middleware (in feature branch, not merge into main)
+- 2025.4.18 Temp disable the compression middleware. It has MIME type bugs. 

docker-compose.yaml

@@ -39,3 +39,12 @@ services:
 #     external: true
 #   internal_traefik_default:
 #     external: true
+networks:
+  default:
+    name: traefik_default
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.19.0.0/16
+          gateway: 172.19.0.1
+

dynamic.yml

@@ -9,7 +9,7 @@ http:
         cloudflare:
           trustedCIDRs: []
           overwriteRequestHeader: true
-          debug: true
+          debug: false
     crowdsec:
       plugin:
         bouncer:
@@ -34,21 +34,21 @@ http:
     block-direct-access:
       rule: "HostRegexp(`{host:.+}`)"  # Matches any host
       service: noop@internal
-      priority: 1  # Low priority to catch unmatched requests
+      priority: -1  # Low priority to catch unmatched requests
       entryPoints:
         - web
         - websecure
       middlewares:
         - block-ip-access
-    netdata:
-      rule: Host(`netdata.gbanyan.net`)
-      service: netdata
+    qbit:
+      rule: Host(`qbit.gbanyan.net`)
+      service: qbit
       entryPoints: ["internal_websecure"]
       tls:
         certResolver: letsencrypt
 
   services:
-    netdata:
+    qbit:
       loadBalancer:
         servers:
-          - url: "http://127.0.0.1:19999"
+          - url: "http://192.168.50.4:8083"

scripts/update_cloudflare_ips.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Refresh Cloudflare IP ranges in Traefik config and restart Traefik if needed."""
+from __future__ import annotations
+
+import hashlib
+import subprocess
+import sys
+from pathlib import Path
+from typing import Iterable, List
+from urllib.request import Request, urlopen
+
+CLOUDFLARE_IPV4_URL = "https://www.cloudflare.com/ips-v4"
+CLOUDFLARE_IPV6_URL = "https://www.cloudflare.com/ips-v6"
+REPO_ROOT = Path(__file__).resolve().parents[1]
+TRAEFIK_CONFIG_PATH = REPO_ROOT / "traefik.yml"
+ENTRYPOINTS = ["  web:", "  websecure:"]
+USER_AGENT = "TraefikCloudflareUpdater/1.0 (+https://gbanyan.net)"
+
+
+def fetch_ip_ranges() -> List[str]:
+    ips: List[str] = []
+    for url in (CLOUDFLARE_IPV4_URL, CLOUDFLARE_IPV6_URL):
+        req = Request(url, headers={"User-Agent": USER_AGENT})
+        with urlopen(req, timeout=10) as resp:  # noqa: S310 (trusted upstream)
+            for raw_line in resp.read().decode().splitlines():
+                line = raw_line.strip()
+                if line and not line.startswith("#"):
+                    ips.append(line)
+    # Remove duplicates while preserving order
+    seen = set()
+    deduped: List[str] = []
+    for ip in ips:
+        if ip not in seen:
+            deduped.append(ip)
+            seen.add(ip)
+    return deduped
+
+
+def update_trusted_ips(original: str, ips: Iterable[str], entrypoint_label: str) -> str:
+    try:
+        entry_idx = original.index(entrypoint_label)
+    except ValueError:
+        raise RuntimeError(f"EntryPoint {entrypoint_label.strip()} not found in traefik.yml") from None
+
+    forwarded_token = "forwardedHeaders:\n"
+    fwd_idx = original.index(forwarded_token, entry_idx)
+    line_start = original.rfind("\n", 0, fwd_idx) + 1
+    indent = original[line_start:fwd_idx]
+    inner_indent = indent + "  "
+    list_indent = inner_indent + "  "
+
+    block_start = fwd_idx + len(forwarded_token)
+    cursor = block_start
+    while cursor < len(original):
+        next_newline = original.find("\n", cursor)
+        if next_newline == -1:
+            next_newline = len(original)
+        line = original[cursor:next_newline]
+        stripped = line.lstrip()
+        current_indent = len(line) - len(stripped)
+        if not stripped.startswith("-") and current_indent <= len(indent):
+            break
+        cursor = next_newline + 1
+    block_end = cursor
+
+    new_block = forwarded_token
+    new_block += f"{inner_indent}trustedIPs:\n"
+    for ip in ips:
+        new_block += f"{list_indent}- \"{ip}\"\n"
+
+    return original[:fwd_idx] + new_block + original[block_end:]
+
+
+def main() -> None:
+    ips = fetch_ip_ranges()
+    config_text = TRAEFIK_CONFIG_PATH.read_text()
+    before_hash = hashlib.sha256(config_text.encode()).hexdigest()
+
+    for entry_label in ENTRYPOINTS:
+        config_text = update_trusted_ips(config_text, ips, entry_label)
+
+    after_hash = hashlib.sha256(config_text.encode()).hexdigest()
+    if after_hash == before_hash:
+        print("Traefik trusted IP ranges already up-to-date.")
+        return
+
+    TRAEFIK_CONFIG_PATH.write_text(config_text)
+    print(f"Updated {TRAEFIK_CONFIG_PATH} with {len(ips)} Cloudflare IP ranges.")
+
+    try:
+        subprocess.run(
+            ["docker", "compose", "up", "-d", "traefik"],
+            check=True,
+            cwd=REPO_ROOT,
+        )
+        print("Traefik container restarted via docker compose.")
+    except subprocess.CalledProcessError as exc:  # pragma: no cover
+        print("Failed to restart Traefik via docker compose:", exc, file=sys.stderr)
+        sys.exit(exc.returncode)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except Exception as exc:  # noqa: BLE001
+        print(f"Error: {exc}", file=sys.stderr)
+        sys.exit(1)

traefik.yml

@@ -1,7 +1,7 @@
 ## STATIC CONFIGURATION
 
 log:
-  level: "DEBUG"
+  level: "INFO"
   filePath: "/var/log/traefik/traefik.log"
 accessLog:
   filePath: "/var/log/traefik/access.log"
@@ -16,34 +16,32 @@ api:
 
 entryPoints:
   web:
-   address: "10.0.0.225:80"
-   forwardedHeaders:
-      trustedIPs: &trustedIps
-        # Start of Cloudlare's public IP list
-        - 103.21.244.0/22
-        - 103.22.200.0/22
-        - 103.31.4.0/22
-        - 104.16.0.0/13
-        - 104.24.0.0/14
-        - 108.162.192.0/18
-        - 131.0.72.0/22
-        - 141.101.64.0/18
-        - 162.158.0.0/15
-        - 172.64.0.0/13
-        - 173.245.48.0/20
-        - 188.114.96.0/20
-        - 190.93.240.0/20
-        - 197.234.240.0/22
-        - 198.41.128.0/17
-        - 2400:cb00::/32
-        - 2606:4700::/32
-        - 2803:f800::/32
-        - 2405:b500::/32
-        - 2405:8100::/32
-        - 2a06:98c0::/29
-        - 2c0f:f248::/32
-        # End of Cloudlare's public IP list
-   http:
+    address: "10.0.0.225:80"
+    forwardedHeaders:
+      trustedIPs:
+        - "173.245.48.0/20"
+        - "103.21.244.0/22"
+        - "103.22.200.0/22"
+        - "103.31.4.0/22"
+        - "141.101.64.0/18"
+        - "108.162.192.0/18"
+        - "190.93.240.0/20"
+        - "188.114.96.0/20"
+        - "197.234.240.0/22"
+        - "198.41.128.0/17"
+        - "162.158.0.0/15"
+        - "104.16.0.0/13"
+        - "104.24.0.0/14"
+        - "172.64.0.0/13"
+        - "131.0.72.0/22"
+        - "2400:cb00::/32"
+        - "2606:4700::/32"
+        - "2803:f800::/32"
+        - "2405:b500::/32"
+        - "2405:8100::/32"
+        - "2a06:98c0::/29"
+        - "2c0f:f248::/32"
+    http:
       redirections:                           # HTTPS redirection (80 to 443)
         entryPoint:
           to: "websecure"                         # The target element
@@ -51,8 +49,29 @@ entryPoints:
   websecure:
     address: "10.0.0.225:443"
     forwardedHeaders:
-      # Reuse the list of Cloudflare's public IPs from above
-      trustedIPs: *trustedIps
+      trustedIPs:
+        - "173.245.48.0/20"
+        - "103.21.244.0/22"
+        - "103.22.200.0/22"
+        - "103.31.4.0/22"
+        - "141.101.64.0/18"
+        - "108.162.192.0/18"
+        - "190.93.240.0/20"
+        - "188.114.96.0/20"
+        - "197.234.240.0/22"
+        - "198.41.128.0/17"
+        - "162.158.0.0/15"
+        - "104.16.0.0/13"
+        - "104.24.0.0/14"
+        - "172.64.0.0/13"
+        - "131.0.72.0/22"
+        - "2400:cb00::/32"
+        - "2606:4700::/32"
+        - "2803:f800::/32"
+        - "2405:b500::/32"
+        - "2405:8100::/32"
+        - "2a06:98c0::/29"
+        - "2c0f:f248::/32"
     http3: {}
   internal_web:
     address: "192.168.50.4:80"
@@ -65,7 +84,7 @@ entryPoints:
     address: "192.168.50.4:443"
     http3: {}
   metrics:
-    address: "127.0.0.1:8082"
+    address: ":8082"
   dashboard:
     address: "127.0.0.1:9090"
 
@@ -76,7 +95,7 @@ global:
 providers:
   docker:
     exposedByDefault: false
-#    network: traefik_default  # Ensure this matches the Docker network
+    # defaultRule: "Host(`{{ .ContainerName }}.gbanyan.net`)"
   file:
     filename: "/dynamic.yml"  # Enable dynamic configuration file
 certificatesResolvers: