feat(deploy): ausgehender HTTP(S)-Proxy für Build und Laufzeit
- Compose reicht HTTP_PROXY/HTTPS_PROXY/NO_PROXY als Build-Argumente an beide Images weiter (npm ci, Google Fonts) und zur Laufzeit an api und worker; api, db, web stehen immer in NO_PROXY - Spielerköpfe werden über undici EnvHttpProxyAgent geladen (Node-fetch ignoriert Proxy-Variablen) - Doku: Proxy-Abschnitt mit Whitelist und Fehlerbildern - geprüft mit lokalem Squid: Build nur über registry.npmjs.org, fonts.googleapis.com, fonts.gstatic.com; zur Laufzeit nur mc-heads.net Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lucashahmannandClaude Opus 5.5 committed
1 parent
e66f89f49e
commit
d45d1821df
10 files changed
+179
-4
No files matched your search
@@ -30,6 +30,7 @@
|
||||
"pg": "^8.23.1",
|
||||
"pino": "^9.14.0",
|
||||
"ssh2-sftp-client": "^12.1.1",
|
||||
"undici": "^7.30.0",
|
||||
"zod": "^3.25.76"
|
||||
}
|
||||
}
|
||||
+15
-1
@@ -27,10 +27,24 @@ const envSchema = z.object({
|
||||
LOG_LEVEL: z.string().default('info'),
|
||||
});
|
||||
|
||||
export interface ProxyConfig {
|
||||
httpProxy: string | undefined;
|
||||
httpsProxy: string | undefined;
|
||||
noProxy: string | undefined;
|
||||
}
|
||||
|
||||
export type Config = z.infer<typeof envSchema> & {
|
||||
sftp: { host: string; port: number; username: string; password: string; root: string } | null;
|
||||
/** Outbound HTTP(S) proxy from the usual environment variables (upper or lower case); null when unset. */
|
||||
proxy: ProxyConfig | null;
|
||||
};
|
||||
|
||||
function proxyFromEnv(env: Record<string, string | undefined>): ProxyConfig | null {
|
||||
const pick = (name: string) => env[name] || env[name.toLowerCase()] || undefined;
|
||||
const proxy = { httpProxy: pick('HTTP_PROXY'), httpsProxy: pick('HTTPS_PROXY'), noProxy: pick('NO_PROXY') };
|
||||
return proxy.httpProxy || proxy.httpsProxy ? proxy : null;
|
||||
}
|
||||
|
||||
export function loadConfig(env: Record<string, string | undefined> = process.env): Config {
|
||||
const r = envSchema.safeParse(env);
|
||||
if (!r.success) {
|
||||
@@ -42,5 +56,5 @@ export function loadConfig(env: Record<string, string | undefined> = process.env
|
||||
c.SFTP_HOST && c.SFTP_USER && c.SFTP_PASSWORD
|
||||
? { host: c.SFTP_HOST, port: c.SFTP_PORT, username: c.SFTP_USER, password: c.SFTP_PASSWORD, root: c.SFTP_ROOT }
|
||||
: null;
|
||||
return { ...c, sftp };
|
||||
return { ...c, sftp, proxy: proxyFromEnv(env) };
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
import { EnvHttpProxyAgent, fetch as undiciFetch, type Dispatcher } from 'undici';
|
||||
import type { ProxyConfig } from '../config.ts';
|
||||
|
||||
export type HttpFetch = (url: string, init?: { signal?: AbortSignal }) => Promise<Response>;
|
||||
|
||||
/**
|
||||
* fetch for outbound internet requests. Node's built-in fetch ignores HTTP(S)_PROXY, so behind a forward
|
||||
* proxy (e.g. Squid in the DMZ) requests go through undici's EnvHttpProxyAgent, which also honours NO_PROXY.
|
||||
*/
|
||||
export function createHttpFetch(proxy: ProxyConfig | null): HttpFetch {
|
||||
if (!proxy) return (url, init) => fetch(url, init);
|
||||
const dispatcher: Dispatcher = new EnvHttpProxyAgent({
|
||||
httpProxy: proxy.httpProxy ?? proxy.httpsProxy,
|
||||
httpsProxy: proxy.httpsProxy ?? proxy.httpProxy,
|
||||
noProxy: proxy.noProxy,
|
||||
});
|
||||
return (url, init) => undiciFetch(url, { ...init, dispatcher }) as unknown as Promise<Response>;
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import type { HttpFetch } from '../../net/http.ts';
|
||||
import { UUID_RE } from './shared.ts';
|
||||
|
||||
const TTL_MS = 24 * 3600_000;
|
||||
@@ -15,7 +16,7 @@ const PLACEHOLDER = Buffer.from(
|
||||
* Player heads are fetched server-side and cached, so visitors' browsers never contact third parties
|
||||
* (no visitor IPs leave the portal).
|
||||
*/
|
||||
export async function avatarRoutes(app: FastifyInstance, { source }: { source: string }) {
|
||||
export async function avatarRoutes(app: FastifyInstance, { source, httpFetch }: { source: string; httpFetch: HttpFetch }) {
|
||||
const cache = new Map<string, { png: Buffer; at: number }>();
|
||||
|
||||
app.get<{ Params: { uuid: string } }>('/api/avatar/:uuid', async (req, reply) => {
|
||||
@@ -26,7 +27,7 @@ export async function avatarRoutes(app: FastifyInstance, { source }: { source: s
|
||||
const hit = cache.get(uuid);
|
||||
if (hit && Date.now() - hit.at < TTL_MS) return reply.send(hit.png);
|
||||
try {
|
||||
const res = await fetch(source.replace('{uuid}', uuid), { signal: AbortSignal.timeout(TIMEOUT_MS) });
|
||||
const res = await httpFetch(source.replace('{uuid}', uuid), { signal: AbortSignal.timeout(TIMEOUT_MS) });
|
||||
const type = res.headers.get('content-type') ?? '';
|
||||
if (!res.ok || !type.startsWith('image/png')) throw new Error(`avatar ${res.status}`);
|
||||
const png = Buffer.from(await res.arrayBuffer());
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
import type { Pool } from 'pg';
|
||||
import type { Config } from '../../config.ts';
|
||||
import { createHttpFetch } from '../../net/http.ts';
|
||||
import { avatarRoutes } from './avatar.ts';
|
||||
import { leaderboardRoutes } from './leaderboards.ts';
|
||||
import { overviewRoutes } from './overview.ts';
|
||||
@@ -16,5 +17,5 @@ export async function publicRoutes(app: FastifyInstance, { pool, config }: { poo
|
||||
await leaderboardRoutes(app, { pool });
|
||||
await playerRoutes(app, { pool, timeZone: config.TIMEZONE });
|
||||
await serverRoutes(app, { pool, config });
|
||||
await avatarRoutes(app, { source: config.AVATAR_SOURCE });
|
||||
await avatarRoutes(app, { source: config.AVATAR_SOURCE, httpFetch: createHttpFetch(config.proxy) });
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
import { createServer, type Server } from 'node:http';
|
||||
import { connect, type AddressInfo, type Socket } from 'node:net';
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { buildApp } from '../src/app.ts';
|
||||
import { loadConfig } from '../src/config.ts';
|
||||
import { getTestPool } from './helpers/db.ts';
|
||||
import { TEST_SECRET } from './helpers/sign.ts';
|
||||
|
||||
const PNG = Buffer.from('89504e470d0a1a0a0000000d4948445200000001000000010806000000', 'hex');
|
||||
const UUID = '308a693c-2bd8-4d6b-8790-670aef5be601';
|
||||
|
||||
let upstream: Server;
|
||||
let proxy: Server;
|
||||
const proxied: string[] = [];
|
||||
const port = (s: Server) => (s.address() as AddressInfo).port;
|
||||
const listen = (s: Server) => new Promise<void>((r) => s.listen(0, '127.0.0.1', () => r()));
|
||||
|
||||
beforeAll(async () => {
|
||||
upstream = createServer((_req, res) => res.writeHead(200, { 'content-type': 'image/png' }).end(PNG));
|
||||
// Minimal forward proxy like Squid: undici tunnels every request with CONNECT host:port.
|
||||
// "upstream.test" only exists for the proxy, which relays to the local upstream server.
|
||||
proxy = createServer((_req, res) => res.writeHead(405).end());
|
||||
proxy.on('connect', (req, client: Socket, head) => {
|
||||
proxied.push(req.url ?? '');
|
||||
const upstreamSocket = connect(Number(new URL(`http://${req.url}`).port), '127.0.0.1', () => {
|
||||
client.write('HTTP/1.1 200 Connection Established\r\n\r\n');
|
||||
upstreamSocket.write(head);
|
||||
upstreamSocket.pipe(client);
|
||||
client.pipe(upstreamSocket);
|
||||
});
|
||||
});
|
||||
await listen(upstream);
|
||||
await listen(proxy);
|
||||
});
|
||||
afterAll(() => {
|
||||
upstream.close();
|
||||
proxy.close();
|
||||
});
|
||||
|
||||
async function avatarVia(env: Record<string, string>) {
|
||||
const config = loadConfig({
|
||||
NODE_ENV: 'test',
|
||||
DATABASE_URL: process.env.TEST_DATABASE_URL,
|
||||
INGEST_HMAC_SECRET: TEST_SECRET,
|
||||
LOG_LEVEL: 'silent',
|
||||
AVATAR_SOURCE: `http://upstream.test:${port(upstream)}/avatar/{uuid}`,
|
||||
...env,
|
||||
});
|
||||
const app = await buildApp({ pool: await getTestPool(), config });
|
||||
try {
|
||||
return await app.inject({ method: 'GET', url: `/api/avatar/${UUID}` });
|
||||
} finally {
|
||||
await app.close();
|
||||
}
|
||||
}
|
||||
|
||||
describe('outbound proxy', () => {
|
||||
it('fetches player heads through HTTP_PROXY', async () => {
|
||||
proxied.length = 0;
|
||||
const res = await avatarVia({ HTTP_PROXY: `http://127.0.0.1:${port(proxy)}` });
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.rawPayload.equals(PNG)).toBe(true);
|
||||
expect(proxied).toEqual([`upstream.test:${port(upstream)}`]);
|
||||
});
|
||||
|
||||
it('accepts lower-case proxy variables', () => {
|
||||
const c = loadConfig({ DATABASE_URL: 'postgres://x', INGEST_HMAC_SECRET: 'x'.repeat(32), https_proxy: 'http://p:3128', no_proxy: '.local' });
|
||||
expect(c.proxy).toEqual({ httpProxy: undefined, httpsProxy: 'http://p:3128', noProxy: '.local' });
|
||||
});
|
||||
|
||||
it('uses no proxy when none is configured', () => {
|
||||
const c = loadConfig({ DATABASE_URL: 'postgres://x', INGEST_HMAC_SECRET: 'x'.repeat(32) });
|
||||
expect(c.proxy).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,13 @@ TRAEFIK_CERTRESOLVER=letsencrypt
|
||||
# Öffentliche IP(s) des Mine-Hoster-Gameservers, kommagetrennt (CIDR). Nur sie dürfen /api/ingest erreichen.
|
||||
INGEST_ALLOWED_IPS=203.0.113.10/32
|
||||
|
||||
# ---- Ausgehender Proxy (optional) ------------------------------------------
|
||||
# Für Build (npm, Google Fonts) und Spielerköpfe. Leer lassen, wenn der Server direkt ins Internet darf.
|
||||
# Die Proxy-Einstellung des Docker-Daemons (systemd) gilt nur für Image-Pulls und reicht dafür nicht.
|
||||
HTTP_PROXY=
|
||||
HTTPS_PROXY=
|
||||
NO_PROXY=localhost,127.0.0.1,.local
|
||||
|
||||
# ---- Datenbank -------------------------------------------------------------
|
||||
POSTGRES_PASSWORD=bitte-ein-langes-zufaelliges-passwort
|
||||
BACKUP_KEEP_DAYS=14
|
||||
|
||||
@@ -7,15 +7,31 @@
|
||||
|
||||
name: atm10-stats
|
||||
|
||||
# Outbound proxy (e.g. Squid in the DMZ). The Docker daemon's own proxy setting only covers image pulls;
|
||||
# RUN steps in the build (npm, Google Fonts) and the running API need it passed explicitly.
|
||||
x-proxy-build-args: &proxy-build-args
|
||||
HTTP_PROXY: ${HTTP_PROXY:-}
|
||||
HTTPS_PROXY: ${HTTPS_PROXY:-}
|
||||
NO_PROXY: ${NO_PROXY:-}
|
||||
http_proxy: ${HTTP_PROXY:-}
|
||||
https_proxy: ${HTTPS_PROXY:-}
|
||||
no_proxy: ${NO_PROXY:-}
|
||||
|
||||
x-backend: &backend
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: backend/Dockerfile
|
||||
args: *proxy-build-args
|
||||
image: atm10-stats/backend:latest
|
||||
restart: unless-stopped
|
||||
env_file: ${ENV_FILE:-.env}
|
||||
environment:
|
||||
DATABASE_URL: postgres://atm:${POSTGRES_PASSWORD}@db:5432/atm
|
||||
# Player heads (mc-heads.net) go through the proxy; Compose-internal names and the database never do.
|
||||
# SFTP to the game server is a direct SSH connection and does not use these variables.
|
||||
HTTP_PROXY: ${HTTP_PROXY:-}
|
||||
HTTPS_PROXY: ${HTTPS_PROXY:-}
|
||||
NO_PROXY: ${NO_PROXY:-localhost,127.0.0.1},api,db,web
|
||||
depends_on:
|
||||
db:
|
||||
condition: service_healthy
|
||||
@@ -71,6 +87,7 @@ services:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: web/Dockerfile
|
||||
args: *proxy-build-args
|
||||
image: atm10-stats/web:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
|
||||
@@ -14,6 +14,34 @@ bei Mine-Hoster, laufender Betrieb und Fehlerbehebung.
|
||||
> Traefik **v2**: Die Middleware heißt dort `ipwhitelist` statt `ipallowlist`. In `deploy/docker-compose.yml`
|
||||
> die Zeile `traefik.http.middlewares.atm-ingest-allow.ipallowlist.sourcerange` entsprechend umbenennen.
|
||||
|
||||
### Ausgehender Proxy (z. B. Squid in der DMZ)
|
||||
|
||||
Muss der Server über einen Proxy ins Internet, trage ihn in `deploy/.env` ein:
|
||||
|
||||
```bash
|
||||
HTTP_PROXY=http://squid-proxy.example.local:3128
|
||||
HTTPS_PROXY=http://squid-proxy.example.local:3128
|
||||
NO_PROXY=localhost,127.0.0.1,.local
|
||||
```
|
||||
|
||||
Compose reicht die Werte als Build-Argumente an beide Images und zur Laufzeit an api und worker weiter. Die
|
||||
Container-Namen `api`, `db` und `web` hängt es automatisch an `NO_PROXY` an, sodass interner Verkehr nie über den
|
||||
Proxy läuft. Eine Proxy-Einstellung für den Docker-Daemon (`/etc/systemd/system/docker.service.d/proxy.conf`) gilt
|
||||
nur für Image-Pulls. Für `npm ci` im Build reicht sie **nicht**; ohne die Einträge oben bricht der Build mit
|
||||
„npm error Exit handler never called!“ ab.
|
||||
|
||||
Whitelist für den Proxy (alle Verbindungen sind `CONNECT …:443`):
|
||||
|
||||
| Domain | Wofür | Wann |
|
||||
|---|---|---|
|
||||
| `registry.npmjs.org` | npm-Pakete | Build, Pflicht |
|
||||
| `fonts.googleapis.com`, `fonts.gstatic.com` | Schriften des Dashboards | Build, Pflicht |
|
||||
| `registry-1.docker.io`, `auth.docker.io`, `production.cloudflare.docker.com` | Basis-Images (über die Daemon-Einstellung) | Pull, Pflicht |
|
||||
| `mc-heads.net` | Spielerköpfe | Laufzeit, optional (ohne: graue Platzhalter) |
|
||||
|
||||
Der SFTP-Abgleich zum Gameserver ist eine direkte SSH-Verbindung und läuft nicht über den Proxy; dafür braucht
|
||||
der DMZ-Server eine eigene Firewall-Freigabe (siehe OPNsense).
|
||||
|
||||
## 2. Portal installieren
|
||||
|
||||
```bash
|
||||
@@ -166,6 +194,9 @@ Das Volume `backups` liegt auf demselben Host. Für echte Ausfallsicherheit die
|
||||
| Mod: `outbox limit reached, dropped …` | Backend länger nicht erreichbar | Verbindung reparieren; ggf. `maxOutboxMb` erhöhen |
|
||||
| `/sources`: SFTP „Fehler beim letzten Abgleich“ | Zugangsdaten, Port oder OPNsense-Regel | Fehlertext unter `/admin` → *Letzte Fehler* |
|
||||
| Live-Ticker bleibt leer, Seite sonst ok | Proxy puffert SSE | keine Kompressions-/Buffering-Middleware auf `/api/live` |
|
||||
| Build: `npm error Exit handler never called!` nach ~2 min | `npm ci` erreicht `registry.npmjs.org` nicht (ausgehender Proxy/Firewall) | Proxy in `deploy/.env` eintragen und Domains freischalten (Abschnitt 1, „Ausgehender Proxy“) |
|
||||
| Build: Fehler beim Laden von Google Fonts | `fonts.googleapis.com`/`fonts.gstatic.com` nicht freigeschaltet | in der Proxy-Whitelist ergänzen |
|
||||
| Nur graue Spielerköpfe | `mc-heads.net` nicht erreichbar | in der Proxy-Whitelist ergänzen (optional) |
|
||||
| Login unter `/admin` klappt nicht ohne HTTPS | Cookie ist `Secure` | Portal über HTTPS aufrufen (lokal: `NODE_ENV=development`) |
|
||||
|
||||
Spieler, die nicht im Portal erscheinen möchten, lassen sich unter `/admin` ausblenden: Sie verschwinden aus allen
|
||||
|
||||
Generated
+10
@@ -28,6 +28,7 @@
|
||||
"pg": "^8.23.1",
|
||||
"pino": "^9.14.0",
|
||||
"ssh2-sftp-client": "^12.1.1",
|
||||
"undici": "^7.30.0",
|
||||
"zod": "^3.25.76"
|
||||
},
|
||||
"devDependencies": {
|
||||
@@ -10719,6 +10720,15 @@
|
||||
"url": "https://github.com/sponsors/ljharb"
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.30.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.30.0.tgz",
|
||||
"integrity": "sha512-dkrQXeHSaoamnItlYbmzG0wFYrM0ZwDxCIg0A7aKjTyyhh9svRzCNFEzV+Vm05/yehjCzjDZ31KXfGEjYSztDQ==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.18.1"
|
||||
}
|
||||
},
|
||||
"node_modules/undici-types": {
|
||||
"version": "6.21.0",
|
||||
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
|
||||
|
||||
Reference in new issue
Block a user