From 6a61bbcc8181c37e7c424ab507c904625bc3bea4 Mon Sep 17 00:00:00 2001 From: neodk2004 Date: Sun, 16 Aug 2026 13:17:04 +0200 Subject: [PATCH] feat(docker): zero-downtime rolling deploy for the 3 replicas Adds a dependency-free liveness endpoint (GET /api/health), a Docker healthcheck against it, and docker/rolling-deploy.sh which rebuilds the shared image once and restarts pawfeed-1/2/3 sequentially, gated on each becoming healthy before moving to the next. Verified zero-downtime via continuous curl monitoring through a full run against production. --- CLAUDE.md | 2 +- README.md | 2 +- docker/SCALING-ROADMAP.md | 15 +++++++++- docker/docker-compose.yml | 9 ++++++ docker/rolling-deploy.sh | 59 +++++++++++++++++++++++++++++++++++++ src/app/api/health/route.ts | 14 +++++++++ src/proxy.ts | 1 + 7 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 docker/rolling-deploy.sh create mode 100644 src/app/api/health/route.ts diff --git a/CLAUDE.md b/CLAUDE.md index 440a399..fab13f0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -98,7 +98,7 @@ Vitest + `jsdom`. Prisma is fully mocked per-model via `createMockPrisma()` in ` Despite `next.config.ts`'s `output: "standalone"` and the original stack research recommending Vercel, this app is deployed to a self-hosted OpenMediaVault NAS via Docker (`docker/Dockerfile`, `docker/docker-compose.yml`, `docker/start.sh`). **There is no `git pull` on the server** — deploys are manual FTP uploads of changed files followed by `./start.sh`. Don't suggest Vercel-specific features (ISR revalidation webhooks, Vercel KV, edge config) without checking this still holds. -**Runs as 3 load-balanced replicas, not a single container (since 2026-08-16, Gitea #26).** `docker/docker-compose.yml` defines `pawfeed-1`/`pawfeed-2`/`pawfeed-3` (via the `x-pawfeed-common` YAML anchor) instead of one `pawfeed` service — Next.js standalone output is single-process (`node server.js`), so one container only ever used one CPU core regardless of how many the NAS has. A load test on 2026-08-15 hit a hard ceiling at ~60–65 req/sec with one core pegged at 133% while 3 cores sat idle; splitting into 3 replicas fixed it (confirmed via a follow-up load test showing ~100% CPU spread evenly across all three). Nginx Proxy Manager (container `nginx-app-1` on this NAS, admin UI at `192.168.1.222:81`) load-balances across them via a `least_conn` upstream block — that block lives in `/Docker-Data/nginxproxymanager/data/nginx/custom/http_top.conf` on the NAS host (an nginx `upstream {}` is only valid in the `http{}` context, which NPM's per-host "Custom Nginx Configuration" field can't reach — that field only writes into the `server{}`/`location{}` context, and holds the `proxy_pass http://pawfeed_upstream;` override for the `pawfeed.org` Proxy Host instead). Full runbook, decisions, and risk notes: `docker/SCALING-ROADMAP.md`. **Deploying a config or code change means rebuilding all 3 replicas** — `docker compose up -d --build` from `docker/` rebuilds the shared `pawfeed:latest` image once and recreates all three; there is no separate single-container path anymore, and NPM's proxy target is `pawfeed_upstream`, not a bare `pawfeed:3000` hostname. `src/lib/prisma.ts`'s Prisma pg pool size is capped per-replica via `DB_POOL_MAX` (compose sets `10`) so 3 replicas stay well under the Supabase pooler's client-connection limit. +**Runs as 3 load-balanced replicas, not a single container (since 2026-08-16, Gitea #26).** `docker/docker-compose.yml` defines `pawfeed-1`/`pawfeed-2`/`pawfeed-3` (via the `x-pawfeed-common` YAML anchor) instead of one `pawfeed` service — Next.js standalone output is single-process (`node server.js`), so one container only ever used one CPU core regardless of how many the NAS has. A load test on 2026-08-15 hit a hard ceiling at ~60–65 req/sec with one core pegged at 133% while 3 cores sat idle; splitting into 3 replicas fixed it (confirmed via a follow-up load test showing ~100% CPU spread evenly across all three). Nginx Proxy Manager (container `nginx-app-1` on this NAS, admin UI at `192.168.1.222:81`) load-balances across them via a `least_conn` upstream block — that block lives in `/Docker-Data/nginxproxymanager/data/nginx/custom/http_top.conf` on the NAS host (an nginx `upstream {}` is only valid in the `http{}` context, which NPM's per-host "Custom Nginx Configuration" field can't reach — that field only writes into the `server{}`/`location{}` context, and holds the `proxy_pass http://pawfeed_upstream;` override for the `pawfeed.org` Proxy Host instead). Full runbook, decisions, and risk notes: `docker/SCALING-ROADMAP.md`. **Deploying a config or code change means rebuilding all 3 replicas** — there is no separate single-container path anymore, and NPM's proxy target is `pawfeed_upstream`, not a bare `pawfeed:3000` hostname. Use `./docker/rolling-deploy.sh` for deploys against the live site — it rebuilds the shared `pawfeed:latest` image once, then restarts `pawfeed-1`/`2`/`3` one at a time, gated on each becoming `healthy` per the Docker `healthcheck` (`GET /api/health`, a dependency-free liveness check — see `src/app/api/health/route.ts`) before moving to the next, so at least 2 replicas always serve traffic. Verified zero-downtime on 2026-08-16 (continuous curl monitoring showed unbroken HTTP 200 through a full rolling deploy). `docker compose up -d --build` (`start.sh`) still works but restarts all 3 at once — only use it for first-time setup or when a brief outage is acceptable. `src/lib/prisma.ts`'s Prisma pg pool size is capped per-replica via `DB_POOL_MAX` (compose sets `10`) so 3 replicas stay well under the Supabase pooler's client-connection limit. `src/app/api/cron/trim-feeds/route.ts` trims each pet's Redis feed sorted set to the 90-day retention window and expects a `CRON_SECRET` bearer token. `vercel.json`'s `crons` array that points at it is **inert** on this self-hosted deployment — nothing calls this route yet. It needs an external trigger (e.g. a NAS-side cron job hitting the route with the secret header) before it actually runs. diff --git a/README.md b/README.md index c75ac33..7eed997 100644 --- a/README.md +++ b/README.md @@ -262,7 +262,7 @@ docker compose build `docker-compose.yml` passes the `NEXT_PUBLIC_*` vars and the Sentry/GlitchTip build-time vars (`SENTRY_ORG`, `SENTRY_PROJECT`, `SENTRY_AUTH_TOKEN`) as Docker build args — everything else is injected at runtime via `env_file`. -**As of 2026-08-16 this runs as 3 replicas (`pawfeed-1/2/3`), not 1 container** — Next.js standalone output is single-process, so a single container can't use more than one CPU core. `docker compose up -d --build` rebuilds the shared image once and recreates all three; there's no single-container path anymore. Nginx Proxy Manager load-balances across them via a `least_conn` upstream (`pawfeed_upstream`) — see `docker/SCALING-ROADMAP.md` for the full rationale and the NPM-side config location. +**As of 2026-08-16 this runs as 3 replicas (`pawfeed-1/2/3`), not 1 container** — Next.js standalone output is single-process, so a single container can't use more than one CPU core; there's no single-container path anymore. Nginx Proxy Manager load-balances across them via a `least_conn` upstream (`pawfeed_upstream`) — see `docker/SCALING-ROADMAP.md` for the full rationale and the NPM-side config location. For deploys against the live site, run `./docker/rolling-deploy.sh` instead of `docker compose up -d --build` — it rebuilds the image once and restarts the 3 replicas one at a time (gated on `GET /api/health`), verified zero-downtime. `docker compose up -d --build` still works but restarts all 3 at once. --- diff --git a/docker/SCALING-ROADMAP.md b/docker/SCALING-ROADMAP.md index a9163d9..9b211b8 100644 --- a/docker/SCALING-ROADMAP.md +++ b/docker/SCALING-ROADMAP.md @@ -115,8 +115,21 @@ Zwei Teile, unterschiedliche Nginx-Kontexte — geht nicht komplett über die NP - [ ] Login/Session-Konsistenz über mehrere Reloads noch nicht gezielt manuell verifiziert (Clerk ist JWT-basiert/stateless, sollte unkritisch sein) — bei Gelegenheit nachholen. - [ ] Mux-Webhook und Stripe-Webhook noch nicht gezielt gegen die neue Replica-Config getestet — bei Gelegenheit nachholen. +## Schritt 5 — Rolling-Deploy (Zero-Downtime) — ✅ erledigt 2026-08-16 + +`docker compose up -d --build` startete bis hierhin alle 3 Container gleichzeitig neu → kurzes gemeinsames Downtime-Fenster bei jedem Deploy. Behoben mit drei Teilen: + +1. **`src/app/api/health/route.ts`** — reiner Liveness-Check (`GET` → `{status:"ok"}`), keine DB-/Redis-Abhängigkeit (ein DB-Ausfall darf nicht alle Replicas als "unhealthy" markieren). In `src/proxy.ts`s `isPublicRoute` eingetragen, sonst blockt Clerk den Check. +2. **Docker-`healthcheck`** in `docker-compose.yml`s `x-pawfeed-common`-Anchor — pollt `GET http://127.0.0.1:3000/api/health` per `wget` (einziger HTTP-Client im `node:22-alpine`-Runner-Image), `interval: 5s`, `start_period: 15s`, `retries: 5`. +3. **`docker/rolling-deploy.sh`** — baut das Image einmal, geht dann `pawfeed-1` → `pawfeed-2` → `pawfeed-3` einzeln durch, wartet nach jedem Neustart bis `docker inspect` `healthy` meldet (Timeout 60s), bricht **vor** der nächsten Replica ab, falls eine nicht gesund wird. + +**Verifiziert:** Skript einmal live gegen die Produktionsseite gefahren, parallel jede Sekunde `curl` auf `https://pawfeed.org/` — **durchgehend HTTP 200, keine einzige fehlgeschlagene Anfrage** während des kompletten Durchlaufs (Build + 3× sequenzieller Neustart). + +**Deploy-Workflow ab jetzt:** `./rolling-deploy.sh` statt `docker compose up -d --build` für Deploys gegen die laufende Seite. `start.sh` (`docker compose up -d --build`) bleibt für Erstinstallation bzw. wenn Downtime egal ist. + ## Risiken / offene Punkte - **DB-Connection-Pool — gelöst:** `src/lib/prisma.ts` cappt jetzt per `DB_POOL_MAX` (Default/gesetzt: 10). 3 × 10 = 30, weit unter dem verifizierten Supabase-Limit von 200 Max Client Connections. -- **Rolling-Deploy fehlt noch:** `docker compose up --build` startet weiterhin alle 3 Container gleichzeitig neu → kurzes gemeinsames Downtime-Fenster bei jedem künftigen Deploy. Echter Zero-Downtime-Rolling-Deploy (Container einzeln nacheinander mit `--no-deps` neu bauen) ist ein sinnvoller Folgeschritt, aber nicht Teil dieser Roadmap. +- **Rolling-Deploy — gelöst,** siehe Schritt 5. - **Host-Port-Mapping — geklärt:** NPM erreicht die Replicas per Docker-DNS-Name im `nginx_default`-Network (`pawfeed-1/2/3:3000`), kein Host-Port-Mapping nötig oder vorhanden. +- **Self-Heal-Fanout-Bug (2026-08-16, gefunden + behoben):** `videosRouter.getByPostId`s Self-Heal-Pfad (Client pollt PROCESSING, fragt direkt bei Mux nach) setzte Video-Status auf READY, rief aber nie `fanOutPost` auf — Video war abspielbar, landete aber nie im Feed. Kein Replica-Problem, aber durch den Live-Test nach dem Rollout entdeckt. Fix: `fanOutPost` auch im Self-Heal-Zweig aufrufen (Commit `faf8ee9`). diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0565c96..4d2d72e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,6 +40,15 @@ x-pawfeed-common: &pawfeed-common networks: - default - nginx_default + # Used by docker/rolling-deploy.sh to know when a replica is ready to + # take traffic again before moving on to the next one. wget is the only + # HTTP client available in the node:22-alpine runner image. + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:3000/api/health"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 15s services: redis: diff --git a/docker/rolling-deploy.sh b/docker/rolling-deploy.sh new file mode 100644 index 0000000..ac07b3e --- /dev/null +++ b/docker/rolling-deploy.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Zero-downtime rolling deploy for the pawfeed-1/2/3 replicas. +# +# Builds the shared image once, then restarts each replica one at a time, +# waiting for the Docker healthcheck (GET /api/health, see +# docker-compose.yml) to report "healthy" before moving to the next one. +# At every point at least 2 of 3 replicas are serving traffic, so NPM's +# least_conn upstream (docker/npm-upstream.conf) never has fewer than 2 +# healthy backends. `docker compose up -d --build` (start.sh) still works +# for a first-time deploy or when downtime doesn't matter — use this +# script for routine deploys against a live site instead. +set -e +cd "$(dirname "$0")" + +if [ ! -f .env ]; then + echo "ERROR: docker/.env not found." + echo "Copy .env.example to docker/.env and fill in your values." + exit 1 +fi + +export SENTRY_RELEASE="$(git -C .. rev-parse --short HEAD)" + +REPLICAS="pawfeed-1 pawfeed-2 pawfeed-3" +TIMEOUT_SECS=60 + +echo "Building shared image..." +docker compose build pawfeed-1 + +wait_healthy() { + replica="$1" + elapsed=0 + while [ "$elapsed" -lt "$TIMEOUT_SECS" ]; do + status="$(docker inspect --format='{{.State.Health.Status}}' "$replica" 2>/dev/null || echo "starting")" + if [ "$status" = "healthy" ]; then + return 0 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done + return 1 +} + +for replica in $REPLICAS; do + echo "--- Redeploying $replica ---" + docker compose up -d --no-deps "$replica" + + echo "Waiting for $replica to report healthy (up to ${TIMEOUT_SECS}s)..." + if wait_healthy "$replica"; then + echo "$replica is healthy." + else + echo "ERROR: $replica did not become healthy within ${TIMEOUT_SECS}s." + echo "Check: docker logs $replica" + echo "Aborting before touching the remaining replicas — the ones already" + echo "redeployed keep running, the rest are untouched and still on the old image." + exit 1 + fi +done + +echo "Rolling deploy complete — all 3 replicas updated with zero downtime." diff --git a/src/app/api/health/route.ts b/src/app/api/health/route.ts new file mode 100644 index 0000000..54d45b3 --- /dev/null +++ b/src/app/api/health/route.ts @@ -0,0 +1,14 @@ +import { NextResponse } from "next/server"; + +/** + * GET /api/health + * + * Pure liveness check — no DB/Redis calls — used by the Docker HEALTHCHECK + * in docker/docker-compose.yml and docker/rolling-deploy.sh to confirm a + * replica has finished starting up before moving on to the next one. + * Deliberately does not check dependencies: a DB/Redis outage must not make + * every replica report unhealthy and get pulled from rotation. + */ +export async function GET() { + return NextResponse.json({ status: "ok" }); +} diff --git a/src/proxy.ts b/src/proxy.ts index c1bc70d..11cccff 100644 --- a/src/proxy.ts +++ b/src/proxy.ts @@ -31,6 +31,7 @@ const isPublicRoute = createRouteMatcher([ "/api/auth/(.*)", "/api/cron/(.*)", "/api/csp-report", + "/api/health", // tRPC handles its own auth per-procedure (protectedProcedure vs. // publicProcedure) and returns a proper JSON UNAUTHORIZED error — letting // Clerk's page-level auth.protect() intercept it first produced a 307 HTML