diff --git a/backend/package.json b/backend/package.json index 3c97c1d..f272c68 100644 --- a/backend/package.json +++ b/backend/package.json @@ -30,6 +30,7 @@ "pg": "^8.23.1", "pino": "^9.14.0", "ssh2-sftp-client": "^12.1.1", + "undici": "^7.30.0", "zod": "^3.25.76" } } diff --git a/backend/src/config.ts b/backend/src/config.ts index c6a12ee..9cd6453 100644 --- a/backend/src/config.ts +++ b/backend/src/config.ts @@ -27,10 +27,24 @@ const envSchema = z.object({ LOG_LEVEL: z.string().default('info'), }); +export interface ProxyConfig { + httpProxy: string | undefined; + httpsProxy: string | undefined; + noProxy: string | undefined; +} + export type Config = z.infer & { sftp: { host: string; port: number; username: string; password: string; root: string } | null; + /** Outbound HTTP(S) proxy from the usual environment variables (upper or lower case); null when unset. */ + proxy: ProxyConfig | null; }; +function proxyFromEnv(env: Record): ProxyConfig | null { + const pick = (name: string) => env[name] || env[name.toLowerCase()] || undefined; + const proxy = { httpProxy: pick('HTTP_PROXY'), httpsProxy: pick('HTTPS_PROXY'), noProxy: pick('NO_PROXY') }; + return proxy.httpProxy || proxy.httpsProxy ? proxy : null; +} + export function loadConfig(env: Record = process.env): Config { const r = envSchema.safeParse(env); if (!r.success) { @@ -42,5 +56,5 @@ export function loadConfig(env: Record = process.env c.SFTP_HOST && c.SFTP_USER && c.SFTP_PASSWORD ? { host: c.SFTP_HOST, port: c.SFTP_PORT, username: c.SFTP_USER, password: c.SFTP_PASSWORD, root: c.SFTP_ROOT } : null; - return { ...c, sftp }; + return { ...c, sftp, proxy: proxyFromEnv(env) }; } diff --git a/backend/src/net/http.ts b/backend/src/net/http.ts new file mode 100644 index 0000000..7954ebd --- /dev/null +++ b/backend/src/net/http.ts @@ -0,0 +1,18 @@ +import { EnvHttpProxyAgent, fetch as undiciFetch, type Dispatcher } from 'undici'; +import type { ProxyConfig } from '../config.ts'; + +export type HttpFetch = (url: string, init?: { signal?: AbortSignal }) => Promise; + +/** + * fetch for outbound internet requests. Node's built-in fetch ignores HTTP(S)_PROXY, so behind a forward + * proxy (e.g. Squid in the DMZ) requests go through undici's EnvHttpProxyAgent, which also honours NO_PROXY. + */ +export function createHttpFetch(proxy: ProxyConfig | null): HttpFetch { + if (!proxy) return (url, init) => fetch(url, init); + const dispatcher: Dispatcher = new EnvHttpProxyAgent({ + httpProxy: proxy.httpProxy ?? proxy.httpsProxy, + httpsProxy: proxy.httpsProxy ?? proxy.httpProxy, + noProxy: proxy.noProxy, + }); + return (url, init) => undiciFetch(url, { ...init, dispatcher }) as unknown as Promise; +} diff --git a/backend/src/routes/public/avatar.ts b/backend/src/routes/public/avatar.ts index 9d40459..296177f 100644 --- a/backend/src/routes/public/avatar.ts +++ b/backend/src/routes/public/avatar.ts @@ -1,4 +1,5 @@ import type { FastifyInstance } from 'fastify'; +import type { HttpFetch } from '../../net/http.ts'; import { UUID_RE } from './shared.ts'; const TTL_MS = 24 * 3600_000; @@ -15,7 +16,7 @@ const PLACEHOLDER = Buffer.from( * Player heads are fetched server-side and cached, so visitors' browsers never contact third parties * (no visitor IPs leave the portal). */ -export async function avatarRoutes(app: FastifyInstance, { source }: { source: string }) { +export async function avatarRoutes(app: FastifyInstance, { source, httpFetch }: { source: string; httpFetch: HttpFetch }) { const cache = new Map(); app.get<{ Params: { uuid: string } }>('/api/avatar/:uuid', async (req, reply) => { @@ -26,7 +27,7 @@ export async function avatarRoutes(app: FastifyInstance, { source }: { source: s const hit = cache.get(uuid); if (hit && Date.now() - hit.at < TTL_MS) return reply.send(hit.png); try { - const res = await fetch(source.replace('{uuid}', uuid), { signal: AbortSignal.timeout(TIMEOUT_MS) }); + const res = await httpFetch(source.replace('{uuid}', uuid), { signal: AbortSignal.timeout(TIMEOUT_MS) }); const type = res.headers.get('content-type') ?? ''; if (!res.ok || !type.startsWith('image/png')) throw new Error(`avatar ${res.status}`); const png = Buffer.from(await res.arrayBuffer()); diff --git a/backend/src/routes/public/index.ts b/backend/src/routes/public/index.ts index 9530d4a..bf38241 100644 --- a/backend/src/routes/public/index.ts +++ b/backend/src/routes/public/index.ts @@ -1,6 +1,7 @@ import type { FastifyInstance } from 'fastify'; import type { Pool } from 'pg'; import type { Config } from '../../config.ts'; +import { createHttpFetch } from '../../net/http.ts'; import { avatarRoutes } from './avatar.ts'; import { leaderboardRoutes } from './leaderboards.ts'; import { overviewRoutes } from './overview.ts'; @@ -16,5 +17,5 @@ export async function publicRoutes(app: FastifyInstance, { pool, config }: { poo await leaderboardRoutes(app, { pool }); await playerRoutes(app, { pool, timeZone: config.TIMEZONE }); await serverRoutes(app, { pool, config }); - await avatarRoutes(app, { source: config.AVATAR_SOURCE }); + await avatarRoutes(app, { source: config.AVATAR_SOURCE, httpFetch: createHttpFetch(config.proxy) }); } diff --git a/backend/test/proxy.test.ts b/backend/test/proxy.test.ts new file mode 100644 index 0000000..2dc4ca4 --- /dev/null +++ b/backend/test/proxy.test.ts @@ -0,0 +1,75 @@ +import { createServer, type Server } from 'node:http'; +import { connect, type AddressInfo, type Socket } from 'node:net'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { buildApp } from '../src/app.ts'; +import { loadConfig } from '../src/config.ts'; +import { getTestPool } from './helpers/db.ts'; +import { TEST_SECRET } from './helpers/sign.ts'; + +const PNG = Buffer.from('89504e470d0a1a0a0000000d4948445200000001000000010806000000', 'hex'); +const UUID = '308a693c-2bd8-4d6b-8790-670aef5be601'; + +let upstream: Server; +let proxy: Server; +const proxied: string[] = []; +const port = (s: Server) => (s.address() as AddressInfo).port; +const listen = (s: Server) => new Promise((r) => s.listen(0, '127.0.0.1', () => r())); + +beforeAll(async () => { + upstream = createServer((_req, res) => res.writeHead(200, { 'content-type': 'image/png' }).end(PNG)); + // Minimal forward proxy like Squid: undici tunnels every request with CONNECT host:port. + // "upstream.test" only exists for the proxy, which relays to the local upstream server. + proxy = createServer((_req, res) => res.writeHead(405).end()); + proxy.on('connect', (req, client: Socket, head) => { + proxied.push(req.url ?? ''); + const upstreamSocket = connect(Number(new URL(`http://${req.url}`).port), '127.0.0.1', () => { + client.write('HTTP/1.1 200 Connection Established\r\n\r\n'); + upstreamSocket.write(head); + upstreamSocket.pipe(client); + client.pipe(upstreamSocket); + }); + }); + await listen(upstream); + await listen(proxy); +}); +afterAll(() => { + upstream.close(); + proxy.close(); +}); + +async function avatarVia(env: Record) { + const config = loadConfig({ + NODE_ENV: 'test', + DATABASE_URL: process.env.TEST_DATABASE_URL, + INGEST_HMAC_SECRET: TEST_SECRET, + LOG_LEVEL: 'silent', + AVATAR_SOURCE: `http://upstream.test:${port(upstream)}/avatar/{uuid}`, + ...env, + }); + const app = await buildApp({ pool: await getTestPool(), config }); + try { + return await app.inject({ method: 'GET', url: `/api/avatar/${UUID}` }); + } finally { + await app.close(); + } +} + +describe('outbound proxy', () => { + it('fetches player heads through HTTP_PROXY', async () => { + proxied.length = 0; + const res = await avatarVia({ HTTP_PROXY: `http://127.0.0.1:${port(proxy)}` }); + expect(res.statusCode).toBe(200); + expect(res.rawPayload.equals(PNG)).toBe(true); + expect(proxied).toEqual([`upstream.test:${port(upstream)}`]); + }); + + it('accepts lower-case proxy variables', () => { + const c = loadConfig({ DATABASE_URL: 'postgres://x', INGEST_HMAC_SECRET: 'x'.repeat(32), https_proxy: 'http://p:3128', no_proxy: '.local' }); + expect(c.proxy).toEqual({ httpProxy: undefined, httpsProxy: 'http://p:3128', noProxy: '.local' }); + }); + + it('uses no proxy when none is configured', () => { + const c = loadConfig({ DATABASE_URL: 'postgres://x', INGEST_HMAC_SECRET: 'x'.repeat(32) }); + expect(c.proxy).toBeNull(); + }); +}); diff --git a/deploy/.env.example b/deploy/.env.example index 4f8406a..7765c0d 100644 --- a/deploy/.env.example +++ b/deploy/.env.example @@ -7,6 +7,13 @@ TRAEFIK_CERTRESOLVER=letsencrypt # Öffentliche IP(s) des Mine-Hoster-Gameservers, kommagetrennt (CIDR). Nur sie dürfen /api/ingest erreichen. INGEST_ALLOWED_IPS=203.0.113.10/32 +# ---- Ausgehender Proxy (optional) ------------------------------------------ +# Für Build (npm, Google Fonts) und Spielerköpfe. Leer lassen, wenn der Server direkt ins Internet darf. +# Die Proxy-Einstellung des Docker-Daemons (systemd) gilt nur für Image-Pulls und reicht dafür nicht. +HTTP_PROXY= +HTTPS_PROXY= +NO_PROXY=localhost,127.0.0.1,.local + # ---- Datenbank ------------------------------------------------------------- POSTGRES_PASSWORD=bitte-ein-langes-zufaelliges-passwort BACKUP_KEEP_DAYS=14 diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1bb44b9..21ff4eb 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -7,15 +7,31 @@ name: atm10-stats +# Outbound proxy (e.g. Squid in the DMZ). The Docker daemon's own proxy setting only covers image pulls; +# RUN steps in the build (npm, Google Fonts) and the running API need it passed explicitly. +x-proxy-build-args: &proxy-build-args + HTTP_PROXY: ${HTTP_PROXY:-} + HTTPS_PROXY: ${HTTPS_PROXY:-} + NO_PROXY: ${NO_PROXY:-} + http_proxy: ${HTTP_PROXY:-} + https_proxy: ${HTTPS_PROXY:-} + no_proxy: ${NO_PROXY:-} + x-backend: &backend build: context: .. dockerfile: backend/Dockerfile + args: *proxy-build-args image: atm10-stats/backend:latest restart: unless-stopped env_file: ${ENV_FILE:-.env} environment: DATABASE_URL: postgres://atm:${POSTGRES_PASSWORD}@db:5432/atm + # Player heads (mc-heads.net) go through the proxy; Compose-internal names and the database never do. + # SFTP to the game server is a direct SSH connection and does not use these variables. + HTTP_PROXY: ${HTTP_PROXY:-} + HTTPS_PROXY: ${HTTPS_PROXY:-} + NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1},api,db,web depends_on: db: condition: service_healthy @@ -71,6 +87,7 @@ services: build: context: .. dockerfile: web/Dockerfile + args: *proxy-build-args image: atm10-stats/web:latest restart: unless-stopped environment: diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 223f59f..36b7e65 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -14,6 +14,34 @@ bei Mine-Hoster, laufender Betrieb und Fehlerbehebung. > Traefik **v2**: Die Middleware heißt dort `ipwhitelist` statt `ipallowlist`. In `deploy/docker-compose.yml` > die Zeile `traefik.http.middlewares.atm-ingest-allow.ipallowlist.sourcerange` entsprechend umbenennen. +### Ausgehender Proxy (z. B. Squid in der DMZ) + +Muss der Server über einen Proxy ins Internet, trage ihn in `deploy/.env` ein: + +```bash +HTTP_PROXY=http://squid-proxy.example.local:3128 +HTTPS_PROXY=http://squid-proxy.example.local:3128 +NO_PROXY=localhost,127.0.0.1,.local +``` + +Compose reicht die Werte als Build-Argumente an beide Images und zur Laufzeit an api und worker weiter. Die +Container-Namen `api`, `db` und `web` hängt es automatisch an `NO_PROXY` an, sodass interner Verkehr nie über den +Proxy läuft. Eine Proxy-Einstellung für den Docker-Daemon (`/etc/systemd/system/docker.service.d/proxy.conf`) gilt +nur für Image-Pulls. Für `npm ci` im Build reicht sie **nicht**; ohne die Einträge oben bricht der Build mit +„npm error Exit handler never called!“ ab. + +Whitelist für den Proxy (alle Verbindungen sind `CONNECT …:443`): + +| Domain | Wofür | Wann | +|---|---|---| +| `registry.npmjs.org` | npm-Pakete | Build, Pflicht | +| `fonts.googleapis.com`, `fonts.gstatic.com` | Schriften des Dashboards | Build, Pflicht | +| `registry-1.docker.io`, `auth.docker.io`, `production.cloudflare.docker.com` | Basis-Images (über die Daemon-Einstellung) | Pull, Pflicht | +| `mc-heads.net` | Spielerköpfe | Laufzeit, optional (ohne: graue Platzhalter) | + +Der SFTP-Abgleich zum Gameserver ist eine direkte SSH-Verbindung und läuft nicht über den Proxy; dafür braucht +der DMZ-Server eine eigene Firewall-Freigabe (siehe OPNsense). + ## 2. Portal installieren ```bash @@ -166,6 +194,9 @@ Das Volume `backups` liegt auf demselben Host. Für echte Ausfallsicherheit die | Mod: `outbox limit reached, dropped …` | Backend länger nicht erreichbar | Verbindung reparieren; ggf. `maxOutboxMb` erhöhen | | `/sources`: SFTP „Fehler beim letzten Abgleich“ | Zugangsdaten, Port oder OPNsense-Regel | Fehlertext unter `/admin` → *Letzte Fehler* | | Live-Ticker bleibt leer, Seite sonst ok | Proxy puffert SSE | keine Kompressions-/Buffering-Middleware auf `/api/live` | +| Build: `npm error Exit handler never called!` nach ~2 min | `npm ci` erreicht `registry.npmjs.org` nicht (ausgehender Proxy/Firewall) | Proxy in `deploy/.env` eintragen und Domains freischalten (Abschnitt 1, „Ausgehender Proxy“) | +| Build: Fehler beim Laden von Google Fonts | `fonts.googleapis.com`/`fonts.gstatic.com` nicht freigeschaltet | in der Proxy-Whitelist ergänzen | +| Nur graue Spielerköpfe | `mc-heads.net` nicht erreichbar | in der Proxy-Whitelist ergänzen (optional) | | Login unter `/admin` klappt nicht ohne HTTPS | Cookie ist `Secure` | Portal über HTTPS aufrufen (lokal: `NODE_ENV=development`) | Spieler, die nicht im Portal erscheinen möchten, lassen sich unter `/admin` ausblenden: Sie verschwinden aus allen diff --git a/package-lock.json b/package-lock.json index e2e2e53..fa87ca9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -28,6 +28,7 @@ "pg": "^8.23.1", "pino": "^9.14.0", "ssh2-sftp-client": "^12.1.1", + "undici": "^7.30.0", "zod": "^3.25.76" }, "devDependencies": { @@ -10719,6 +10720,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici": { + "version": "7.30.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.0.tgz", + "integrity": "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",