Merge pull request #3824 from werf/tuf-redirector

site: Router for download URLs

docs/site/.helm/templates/14-tuf-router.yaml

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: tuf-router
+  annotations:
+    "werf.io/replicas-on-creation": "2"
+  labels:
+    service: tuf-router
+spec:
+  minReadySeconds: 5
+  revisionHistoryLimit: 5
+  selector:
+    matchLabels:
+      service: tuf-router
+  template:
+    metadata:
+      labels:
+        service: tuf-router
+    spec:
+      imagePullSecrets:
+        - name: github-werfio
+      containers:
+        - name: nginx
+          command: ["nginx", "-g", "daemon off;"]
+          image: {{ index .Values.werf.image "tuf-router" }}
+{{- include "resources" . | indent 10 }}
+          ports:
+          - containerPort: 8080
+            name: http
+            protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: 8080
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tuf-router
+spec:
+  clusterIP: None
+  selector:
+    service: tuf-router
+  ports:
+  - name: http
+    port: 80
+    protocol: TCP
+    targetPort: 8080
+---
+apiVersion: autoscaling.k8s.io/v1beta2
+kind: VerticalPodAutoscaler
+metadata:
+  name: tuf-router
+spec:
+  targetRef:
+    apiVersion: "apps/v1"
+    kind: Deployment
+    name: tuf-router
+  updatePolicy:
+    updateMode: "Initial"
+---
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: tuf-router
+spec:
+  maxUnavailable: 1
+  selector:
+    matchLabels:
+      service: tuf-router

docs/site/.helm/templates/20-ingress-tuf-router.yaml

@@ -0,0 +1,40 @@
+{{- $host := pluck .Values.werf.env .Values.host | first | default .Values.host._default  }}
+{{- if  hasPrefix "review" .Values.werf.env }}
+{{- $host = ( printf "%s.%s" .Values.werf.env (pluck "dev" .Values.host | first | default .Values.host._default ) | lower ) }}
+{{- end }}
+
+apiVersion: networking.k8s.io/v1beta1
+kind: Ingress
+metadata:
+  name: tuf-router
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+spec:
+  tls:
+  - hosts:
+      - {{ $host }}
+      - ru.{{ $host }}
+    secretName: tls-{{ $host }}
+  rules:
+  - host: {{ $host }}
+    http:
+      paths:
+      - path: /targets
+        backend:
+          serviceName: tuf-router
+          servicePort: http
+      - path: /download
+        backend:
+          serviceName: tuf-router
+          servicePort: http
+  - host: ru.{{ $host }}
+    http:
+      paths:
+      - path: /targets
+        backend:
+          serviceName: tuf-router
+          servicePort: http
+      - path: /download
+        backend:
+          serviceName: tuf-router
+          servicePort: http

docs/site/.werf/nginx-dev.conf

@@ -46,6 +46,9 @@ http {
 
             proxy_pass http://site:8080;
           }
+        location ~ ^(/download/release/|/targets/channels/) {
+            proxy_pass http://tuf:8080;
+          }
         location /guides/ {
             proxy_redirect    off;
             proxy_set_header  Host              werf.io;
@@ -85,6 +88,9 @@ http {
 
             proxy_pass http://site:8080;
         }
+        location ~ ^(/download/release/|/targets/channels/) {
+            proxy_pass http://tuf:8080;
+          }
         location /guides/ {
             proxy_redirect    off;
             proxy_set_header  Host              ru.werf.io;

docs/site/.werf/nginx-tuf-router.conf

@@ -0,0 +1,104 @@
+user  nginx;
+worker_processes auto;
+pid /run/nginx.pid;
+
+events {
+    worker_connections 500;
+    multi_accept on;
+    use epoll;
+}
+
+http {
+
+    # set search paths for pure Lua external libraries (';;' is the default path):
+    lua_package_path '/app/?.lua;;';
+
+    # set search paths for Lua external libraries written in C (can also use ';;'):
+    lua_package_cpath '/app/?.so;;';
+
+    # You might consider changing this
+    resolver 192.168.0.10;
+    # resolver 8.8.8.8;
+
+    proxy_cache_path   /var/cache/nginx levels=1:2 keys_zone=tufcache:5m inactive=1h max_size=500g;
+
+    log_format json_combined escape=json '{ "time_local": "$time_local", '
+     '"host": "$host", '
+     '"remote_addr": "$remote_addr", '
+     '"remote_user": "$remote_user", '
+     '"request": "$request", '
+     '"status": "$status", '
+     '"body_bytes_sent": "$body_bytes_sent", '
+     '"request_time": "$request_time", '
+     '"http_referrer": "$http_referer", '
+     '"http_user_agent": "$http_user_agent" }';
+
+    server_tokens off;
+    sendfile on;
+    tcp_nopush on;
+    tcp_nodelay on;
+    keepalive_timeout 65;
+    types_hash_max_size 2048;
+    client_max_body_size 100M;
+    server_names_hash_bucket_size 64;
+
+    include /etc/nginx/mime.types;
+    default_type application/octet-stream;
+
+    error_log /dev/stderr info;
+
+    gzip off;
+
+    server {
+        charset utf-8;
+        listen 8080;
+        server_name _;
+
+        root        /app;
+        index       index.html;
+
+        set_real_ip_from  0.0.0.0/0;
+        access_log       /dev/stdout json_combined;
+        error_log        /dev/stderr info;
+
+        set $tuf_repo_url 'https://tuf.werf.io';
+
+        location = /healthz {
+            access_log  off;
+            add_header Content-Type text/plain;
+            return 200;
+        }
+
+        # src - /download/release/1.2/alpha/windows-arm64/werf
+        # target - tuf_repo_url/targets/releases/0.1.3/darwin-arm64/bin/werf
+        # $1 - group (MAJ.MIN)
+        # $2 - channel
+        # $3 - OS
+        # $4 - arch
+        # $5 - target_file
+        location ~ ^/download/release/([0-9.]+)/(alpha|beta|early-access|stable|rock-solid)/([a-zA-Z0-9]+)-([a-zA-Z0-9_-]+)/(.+) {
+            set $group $1;
+            set $channel $2;
+            set $os $3;
+            set $arch $4;
+            set $target $5;
+            set $version '';
+
+            rewrite_by_lua_file tuf-router.lua;
+        }
+
+        location /targets/channels {
+          internal;
+          add_header Cache-Control 'no-store, no-cache';
+          proxy_cache tufcache;
+          proxy_cache_valid 200 301 302 5m;
+          proxy_cache_valid any 1m;
+          proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504;
+          proxy_ignore_headers Expires Cache-Control Set-Cookie;
+          # Change this to variable after merging repos
+          proxy_pass https://storage.googleapis.com/werf-tuf$uri;
+        }
+
+        error_page 404 https://$host/404.html?uri=$uri;
+    }
+}

docs/site/.werf/tuf-router.lua

@@ -0,0 +1,37 @@
+local dst_url
+local res
+
+ngx.var.arch = string.lower(ngx.var.arch)
+ngx.var.arch = string.gsub(ngx.var.arch, 'x86_64', 'amd64')
+ngx.var.arch = string.gsub(ngx.var.arch, 'aarch64', 'arm64')
+local m, err = ngx.re.match(ngx.var.target, '.sig$')
+
+if m then
+   dst_url = '/targets/signature/releases/'
+else
+   dst_url = '/targets/releases/'
+end
+
+local version_request_url = string.format('/targets/channels/%s/%s', ngx.var.group, ngx.var.channel)
+
+-- A chain of redirects is not working well, so, avoid it here
+local max_hop = 1
+repeat
+  res = ngx.location.capture(version_request_url)
+  if res.status == ngx.HTTP_MOVED_PERMANENTLY or res.status == ngx.HTTP_MOVED_TEMPORARILY then
+    ngx.log(ngx.NOTICE, string.format('Got redirect %s to %s (requested - %s)', res.status, res.header['Location'], version_request_url))
+    version_request_url = res.header['Location']
+  end
+  max_hop = max_hop - 1
+until not ( max_hop > 0 and res.status ~= ngx.HTTP_OK )
+
+if res.status == ngx.HTTP_OK then
+  -- Is this enough to validate the version?
+  ngx.var.version = string.gsub(res.body,'[^a-zA-Z0-9.+-]','')
+  ngx.header["Cache-Control"] = 'no-store, no-cache'
+  return ngx.redirect(string.format('%s%s%s/%s-%s/bin/%s', ngx.var.tuf_repo_url, dst_url, ngx.var.version, ngx.var.os, ngx.var.arch, ngx.var.target), 302)
+else
+  ngx.log(ngx.WARN, string.format('Got status %s when trying to request version (URL - %s)', res.status, version_request_url))
+  ngx.status = res.status
+  return ngx.exit(res.status)
+end

docs/site/docker-compose.yml

@@ -31,3 +31,7 @@ services:
       - ".werf/nginx-dev.conf:/etc/nginx/nginx.conf:ro"
     ports:
       - "80:80"
+
+  tuf:
+    image: $WERF_TUF_ROUTER_DOCKER_IMAGE_NAME
+    command: ["nginx", "-g", "daemon off;"]

docs/site/werf.yaml

@@ -119,3 +119,17 @@ import:
     add: /app/_site
     to: /app/root
     before: setup
+---
+image: tuf-router
+from: fabiocicerchia/nginx-lua:1.21-alpine@sha256:525aa8fd0ac9bf5c7e82cd83b3f8ebf620bedb0d0b1e52955745fd53917eb24c
+ansible:
+  install:
+  - copy:
+      content: |
+{{ .Files.Get ".werf/nginx-tuf-router.conf" | indent 8 }}
+      dest: /etc/nginx/nginx.conf
+  - copy:
+      content: |
+{{ .Files.Get ".werf/tuf-router.lua" | indent 8 }}
+      dest: /etc/nginx/tuf-router.lua
+