diff --git a/services/game-stream/kustomization.yaml b/services/game-stream/kustomization.yaml index 0972637e..145381df 100644 --- a/services/game-stream/kustomization.yaml +++ b/services/game-stream/kustomization.yaml @@ -4,9 +4,15 @@ kind: Kustomization resources: - namespace.yaml - vault-serviceaccount.yaml + - wolf-api-proxy-configmap.yaml + - wolf-gatekeeper-configmap.yaml - wolf-service.yaml + - wolf-api-service.yaml + - wolf-gatekeeper-service.yaml + - wolfmanager-service.yaml - wolf-moonlight-service.yaml - wolf-statefulset.yaml + - wolf-gatekeeper-daemonset.yaml - oauth2-proxy-wolf.yaml - certificate.yaml - ingress.yaml diff --git a/services/game-stream/oauth2-proxy-wolf.yaml b/services/game-stream/oauth2-proxy-wolf.yaml index 2c29ac2a..f3c0e46a 100644 --- a/services/game-stream/oauth2-proxy-wolf.yaml +++ b/services/game-stream/oauth2-proxy-wolf.yaml @@ -77,8 +77,6 @@ spec: - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas - --scope=openid profile email groups - --email-domain=* - - --allowed-group=game-stream-users - - --allowed-group=/game-stream-users - --allowed-group=admin - --allowed-group=/admin - --set-xauthrequest=true @@ -90,7 +88,7 @@ spec: - --cookie-refresh=20m - --cookie-expire=24h - --insecure-oidc-allow-unverified-email=true - - --upstream=http://ariadne.maintenance.svc.cluster.local + - --upstream=http://wolfmanager.game-stream.svc.cluster.local:8080 - --http-address=0.0.0.0:4180 - --skip-provider-button=true - --approval-prompt=auto diff --git a/services/game-stream/wolf-api-proxy-configmap.yaml b/services/game-stream/wolf-api-proxy-configmap.yaml new file mode 100644 index 00000000..2d919a7e --- /dev/null +++ b/services/game-stream/wolf-api-proxy-configmap.yaml @@ -0,0 +1,87 @@ +# services/game-stream/wolf-api-proxy-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: wolf-api-proxy + namespace: game-stream +data: + wolf_api_proxy.py: | + #!/usr/bin/env python3 + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + import json + import os + import socket + + SOCKET_PATH = os.environ.get("WOLF_SOCKET_PATH", "/run/user/wolf/wolf.sock") + LISTEN = ("0.0.0.0", int(os.environ.get("WOLF_API_PROXY_PORT", "8088"))) + + + def _response(handler, status, payload): + body = json.dumps(payload).encode("utf-8") + handler.send_response(status) + handler.send_header("content-type", "application/json") + handler.send_header("content-length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + + def _forward(method, path, body): + if not path.startswith("/api/v1/"): + return 404, {"success": False, "error": "unsupported path"} + if method not in {"GET", "POST"}: + return 405, {"success": False, "error": "unsupported method"} + if not os.path.exists(SOCKET_PATH): + return 503, {"success": False, "error": "wolf socket is not ready"} + + payload = body or b"" + request = [ + f"{method} {path} HTTP/1.1", + "Host: wolf.local", + "Connection: close", + f"Content-Length: {len(payload)}", + ] + if payload: + request.append("Content-Type: application/json") + request_bytes = ("\r\n".join(request) + "\r\n\r\n").encode("utf-8") + payload + + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as sock: + sock.settimeout(5) + sock.connect(SOCKET_PATH) + sock.sendall(request_bytes) + chunks = [] + while True: + chunk = sock.recv(65536) + if not chunk: + break + chunks.append(chunk) + + raw = b"".join(chunks) + header_bytes, _, body_bytes = raw.partition(b"\r\n\r\n") + status_line = header_bytes.splitlines()[0].decode("utf-8", "replace") if header_bytes else "" + try: + status = int(status_line.split()[1]) + except Exception: + status = 502 + try: + return status, json.loads(body_bytes.decode("utf-8") or "{}") + except Exception: + return status, {"success": False, "error": body_bytes.decode("utf-8", "replace")} + + + class Handler(BaseHTTPRequestHandler): + def log_message(self, _format, *args): + return + + def do_GET(self): + status, payload = _forward("GET", self.path, b"") + _response(self, status, payload) + + def do_POST(self): + length = int(self.headers.get("content-length") or "0") + body = self.rfile.read(length) if length else b"" + status, payload = _forward("POST", self.path, body) + _response(self, status, payload) + + + if __name__ == "__main__": + ThreadingHTTPServer(LISTEN, Handler).serve_forever() diff --git a/services/game-stream/wolf-api-service.yaml b/services/game-stream/wolf-api-service.yaml new file mode 100644 index 00000000..4ea6b439 --- /dev/null +++ b/services/game-stream/wolf-api-service.yaml @@ -0,0 +1,17 @@ +# services/game-stream/wolf-api-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: wolf-api + namespace: game-stream + labels: + app: wolf +spec: + type: ClusterIP + selector: + app: wolf + ports: + - name: http + port: 8088 + targetPort: 8088 + protocol: TCP diff --git a/services/game-stream/wolf-gatekeeper-configmap.yaml b/services/game-stream/wolf-gatekeeper-configmap.yaml new file mode 100644 index 00000000..3c0d6bfa --- /dev/null +++ b/services/game-stream/wolf-gatekeeper-configmap.yaml @@ -0,0 +1,176 @@ +# services/game-stream/wolf-gatekeeper-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: wolf-gatekeeper + namespace: game-stream +data: + wolf_gatekeeper.py: | + #!/usr/bin/env python3 + from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + import ipaddress + import json + import os + import re + import subprocess + + LISTEN = ("0.0.0.0", int(os.environ.get("WOLF_GATEKEEPER_PORT", "8087"))) + HOST_ROOT = os.environ.get("HOST_ROOT", "/host") + NFT_PATH = os.environ.get("NFT_PATH", "/sbin/nft") + MAX_TTL_SECONDS = int(os.environ.get("MAX_TTL_SECONDS", "28800")) + TCP_PORTS = ["47984", "47989", "48010"] + UDP_PORTS = ["47999", "48100", "48200"] + PRIVATE_V4 = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "127.0.0.0/8"] + PRIVATE_V6 = ["::1/128", "fc00::/7", "fe80::/10"] + + + def _json(handler, status, payload): + body = json.dumps(payload).encode("utf-8") + handler.send_response(status) + handler.send_header("content-type", "application/json") + handler.send_header("content-length", str(len(body))) + handler.end_headers() + handler.wfile.write(body) + + + def _nft(args, check=True): + command = ["chroot", HOST_ROOT, NFT_PATH, *args] + proc = subprocess.run(command, check=False, capture_output=True, text=True) + if check and proc.returncode != 0: + raise RuntimeError(proc.stderr.strip() or proc.stdout.strip() or "nft failed") + return proc + + + def _port_set(ports): + return ["{", *[f"{port}," for port in ports[:-1]], ports[-1], "}"] + + + def _ensure_rules(): + _nft(["add", "table", "inet", "wolf_gatekeeper"], check=False) + _nft(["add", "set", "inet", "wolf_gatekeeper", "allowed_v4", "{", "type", "ipv4_addr;", "flags", "timeout;", "}"], check=False) + _nft(["add", "set", "inet", "wolf_gatekeeper", "allowed_v6", "{", "type", "ipv6_addr;", "flags", "timeout;", "}"], check=False) + _nft( + [ + "add", + "chain", + "inet", + "wolf_gatekeeper", + "input", + "{", + "type", + "filter", + "hook", + "input", + "priority", + "-90;", + "policy", + "accept;", + "}", + ], + check=False, + ) + _nft(["flush", "chain", "inet", "wolf_gatekeeper", "input"], check=False) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "iifname", "lo", "accept"]) + for cidr in PRIVATE_V4: + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "ip", "saddr", cidr, "accept"]) + for cidr in PRIVATE_V6: + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "ip6", "saddr", cidr, "accept"]) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "ip", "saddr", "@allowed_v4", "tcp", "dport", *_port_set(TCP_PORTS), "accept"]) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "ip", "saddr", "@allowed_v4", "udp", "dport", *_port_set(UDP_PORTS), "accept"]) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "ip6", "saddr", "@allowed_v6", "tcp", "dport", *_port_set(TCP_PORTS), "accept"]) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "ip6", "saddr", "@allowed_v6", "udp", "dport", *_port_set(UDP_PORTS), "accept"]) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "tcp", "dport", *_port_set(TCP_PORTS), "drop"]) + _nft(["add", "rule", "inet", "wolf_gatekeeper", "input", "udp", "dport", *_port_set(UDP_PORTS), "drop"]) + + + def _validate_ip(value): + ip = ipaddress.ip_address(str(value or "").strip()) + return str(ip), ip.version + + + def _set_name(version): + return "allowed_v4" if version == 4 else "allowed_v6" + + + def _unlock(value, ttl_seconds): + ip, version = _validate_ip(value) + ttl = max(60, min(int(ttl_seconds or MAX_TTL_SECONDS), MAX_TTL_SECONDS)) + set_name = _set_name(version) + _nft(["delete", "element", "inet", "wolf_gatekeeper", set_name, "{", ip, "}"], check=False) + _nft(["add", "element", "inet", "wolf_gatekeeper", set_name, "{", ip, "timeout", f"{ttl}s", "}"]) + return {"ip": ip, "ttl_seconds": ttl} + + + def _revoke(value): + ip, version = _validate_ip(value) + _nft(["delete", "element", "inet", "wolf_gatekeeper", _set_name(version), "{", ip, "}"], check=False) + return {"ip": ip} + + + def _entries(set_name, pattern): + proc = _nft(["list", "set", "inet", "wolf_gatekeeper", set_name], check=False) + if proc.returncode != 0: + return [] + values = [] + for match in re.finditer(pattern, proc.stdout): + value = match.group(0) + if value not in values: + values.append(value) + return values + + + def _status(): + _ensure_rules() + v4 = _entries("allowed_v4", r"\b(?:\d{1,3}\.){3}\d{1,3}\b") + v6 = _entries("allowed_v6", r"\b[0-9a-fA-F:]{2,}\b") + return {"success": True, "active_unlocks": [{"ip": ip} for ip in [*v4, *v6]], "tcp_ports": TCP_PORTS, "udp_ports": UDP_PORTS} + + + class Handler(BaseHTTPRequestHandler): + def log_message(self, _format, *args): + return + + def _payload(self): + length = int(self.headers.get("content-length") or "0") + if not length: + return {} + try: + data = json.loads(self.rfile.read(length).decode("utf-8")) + return data if isinstance(data, dict) else {} + except Exception: + return {} + + def do_GET(self): + try: + if self.path == "/healthz": + _ensure_rules() + _json(self, 200, {"ok": True}) + elif self.path == "/status": + _json(self, 200, _status()) + else: + _json(self, 404, {"success": False, "error": "not found"}) + except Exception as exc: + _json(self, 500, {"success": False, "error": str(exc)}) + + def do_POST(self): + try: + payload = self._payload() + if self.path == "/unlock": + _ensure_rules() + result = _unlock(payload.get("ip"), payload.get("ttl_seconds")) + result.update({"success": True, "actor": payload.get("actor") or "", "target_user": payload.get("target_user") or ""}) + _json(self, 200, result) + elif self.path == "/revoke": + _ensure_rules() + result = _revoke(payload.get("ip")) + result.update({"success": True}) + _json(self, 200, result) + else: + _json(self, 404, {"success": False, "error": "not found"}) + except Exception as exc: + _json(self, 400, {"success": False, "error": str(exc)}) + + + if __name__ == "__main__": + _ensure_rules() + ThreadingHTTPServer(LISTEN, Handler).serve_forever() diff --git a/services/game-stream/wolf-gatekeeper-daemonset.yaml b/services/game-stream/wolf-gatekeeper-daemonset.yaml new file mode 100644 index 00000000..81ecc64a --- /dev/null +++ b/services/game-stream/wolf-gatekeeper-daemonset.yaml @@ -0,0 +1,77 @@ +# services/game-stream/wolf-gatekeeper-daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: wolf-gatekeeper + namespace: game-stream + labels: + app: wolf-gatekeeper +spec: + selector: + matchLabels: + app: wolf-gatekeeper + template: + metadata: + labels: + app: wolf-gatekeeper + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/hostname: titan-24 + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + containers: + - name: gatekeeper + image: ghcr.io/games-on-whales/wolf:stable + imagePullPolicy: IfNotPresent + command: ["/usr/bin/python3", "/opt/wolf-gatekeeper/wolf_gatekeeper.py"] + env: + - name: HOST_ROOT + value: /host + - name: NFT_PATH + value: /sbin/nft + - name: MAX_TTL_SECONDS + value: "28800" + ports: + - name: http + containerPort: 8087 + securityContext: + privileged: true + readinessProbe: + httpGet: + path: /healthz + port: 8087 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /healthz + port: 8087 + initialDelaySeconds: 20 + periodSeconds: 20 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: script + mountPath: /opt/wolf-gatekeeper + readOnly: true + - name: host-root + mountPath: /host + readOnly: true + volumes: + - name: script + configMap: + name: wolf-gatekeeper + defaultMode: 0555 + - name: host-root + hostPath: + path: / + type: Directory diff --git a/services/game-stream/wolf-gatekeeper-service.yaml b/services/game-stream/wolf-gatekeeper-service.yaml new file mode 100644 index 00000000..67cb948e --- /dev/null +++ b/services/game-stream/wolf-gatekeeper-service.yaml @@ -0,0 +1,17 @@ +# services/game-stream/wolf-gatekeeper-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: wolf-gatekeeper + namespace: game-stream + labels: + app: wolf-gatekeeper +spec: + type: ClusterIP + selector: + app: wolf-gatekeeper + ports: + - name: http + port: 8087 + targetPort: 8087 + protocol: TCP diff --git a/services/game-stream/wolf-statefulset.yaml b/services/game-stream/wolf-statefulset.yaml index ec0f254e..58088579 100644 --- a/services/game-stream/wolf-statefulset.yaml +++ b/services/game-stream/wolf-statefulset.yaml @@ -21,6 +21,15 @@ spec: hostNetwork: true dnsPolicy: ClusterFirstWithHostNet runtimeClassName: nvidia + securityContext: + fsGroup: 1000 + initContainers: + - name: wolfmanager-data-permissions + image: busybox:1.36 + command: ["sh", "-c", "mkdir -p /app/data && chown -R 1000:1000 /app/data"] + volumeMounts: + - name: wolfmanager-data + mountPath: /app/data nodeSelector: kubernetes.io/hostname: titan-24 tolerations: @@ -54,17 +63,114 @@ spec: volumeMounts: - name: wolf-state mountPath: /etc/wolf + - name: wolf-runtime + mountPath: /run/user/wolf - name: docker-socket mountPath: /var/run/docker.sock - name: dev mountPath: /dev - name: udev mountPath: /run/udev + - name: wolf-api-proxy + image: ghcr.io/games-on-whales/wolf:stable + imagePullPolicy: IfNotPresent + command: ["/usr/bin/python3", "/opt/wolf-api-proxy/wolf_api_proxy.py"] + ports: + - name: api-proxy + containerPort: 8088 + resources: + requests: + cpu: 25m + memory: 64Mi + limits: + cpu: 250m + memory: 256Mi + volumeMounts: + - name: wolf-runtime + mountPath: /run/user/wolf + - name: wolf-api-proxy + mountPath: /opt/wolf-api-proxy + readOnly: true + - name: wolfmanager + image: ghcr.io/salty2011/wolfmanager:latest + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-ec"] + args: + - | + umask 077 + mkdir -p /app/data + if [ ! -s /app/data/jwt_secret ]; then + head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' > /app/data/jwt_secret + fi + if [ ! -s /app/data/admin_password ]; then + printf 'Wm%s1a\n' "$(head -c 18 /dev/urandom | od -An -tx1 | tr -d ' \n')" > /app/data/admin_password + fi + export Jwt__SecretKey="$(cat /app/data/jwt_secret)" + export Admin__Password="$(cat /app/data/admin_password)" + exec dotnet WolfManager.Api.dll + env: + - name: ASPNETCORE_URLS + value: http://+:8080 + - name: ASPNETCORE_ENVIRONMENT + value: Production + - name: ConnectionStrings__DefaultConnection + value: Data Source=/app/data/wolfmanager.db + - name: Jobs__Storage + value: Memory + - name: Jobs__DashboardEnabled + value: "true" + - name: Wolf__UseUnixSocket + value: "true" + - name: Wolf__UnixSocketPath + value: /run/user/wolf/wolf.sock + - name: OpenTelemetry__ServiceName + value: WolfManager + - name: OpenTelemetry__ConsoleExporter + value: "false" + - name: OpenTelemetry__OtlpExporter + value: "false" + ports: + - name: wolfmanager + containerPort: 8080 + readinessProbe: + httpGet: + path: /health/ready + port: 8080 + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health/live + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 20 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: "1" + memory: 1Gi + volumeMounts: + - name: wolf-runtime + mountPath: /run/user/wolf + - name: wolfmanager-data + mountPath: /app/data volumes: - name: wolf-state hostPath: path: /etc/wolf type: DirectoryOrCreate + - name: wolf-runtime + emptyDir: {} + - name: wolf-api-proxy + configMap: + name: wolf-api-proxy + defaultMode: 0555 + - name: wolfmanager-data + hostPath: + path: /etc/wolfmanager + type: DirectoryOrCreate - name: docker-socket hostPath: path: /var/run/docker.sock diff --git a/services/game-stream/wolfmanager-service.yaml b/services/game-stream/wolfmanager-service.yaml new file mode 100644 index 00000000..43b45bfe --- /dev/null +++ b/services/game-stream/wolfmanager-service.yaml @@ -0,0 +1,17 @@ +# services/game-stream/wolfmanager-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: wolfmanager + namespace: game-stream + labels: + app: wolf +spec: + type: ClusterIP + selector: + app: wolf + ports: + - name: http + port: 8080 + targetPort: 8080 + protocol: TCP diff --git a/services/maintenance/ariadne-deployment.yaml b/services/maintenance/ariadne-deployment.yaml index 950779c9..1f394378 100644 --- a/services/maintenance/ariadne-deployment.yaml +++ b/services/maintenance/ariadne-deployment.yaml @@ -295,6 +295,14 @@ spec: value: game-stream/wolf-oidc - name: ARIADNE_SCHEDULE_WOLF_OIDC value: "17 */6 * * *" + - name: WOLF_API_URL + value: http://wolf-api.game-stream.svc.cluster.local:8088 + - name: WOLF_GATEKEEPER_URL + value: http://wolf-gatekeeper.game-stream.svc.cluster.local:8087 + - name: GAME_STREAM_FIREWALL_UNLOCK_TTL_SEC + value: "28800" + - name: GAME_STREAM_MOONLIGHT_HOST + value: moonlight.bstein.dev - name: GAME_STREAM_USER_GROUP value: game-stream-users - name: GAME_STREAM_ADMIN_GROUP