From 271f3e8c32c3782a7ee5cf85b806c098f077e122 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 23 Aug 2026 10:37:01 -0300 Subject: [PATCH] ops: spread saturated node workloads --- .../applications/kustomization.yaml | 1 - infrastructure/core/coredns-deployment.yaml | 15 ++++++- .../longhorn/core/kustomization.yaml | 1 - .../base/priorityclass/kustomization.yaml | 1 - .../base/storageclass/kustomization.yaml | 3 -- .../chat-ai-gateway-deployment.yaml | 4 ++ .../crypto/wallet-monero-temp/deployment.yaml | 6 ++- .../finance/actual-budget-deployment.yaml | 2 +- services/game-stream/oauth2-proxy-wolf.yaml | 4 ++ services/hermes/agent-deployment.yaml | 28 ++++++++----- services/hermes/chat-statefulset.yaml | 7 +++- services/hermes/deployment.yaml | 6 ++- services/hermes/scripts/cli_lane_dispatch.py | 3 +- services/hermes/scripts/cli_lane_metrics.py | 13 +++++- services/jenkins/deployment.yaml | 42 +++++-------------- services/monitoring/helmrelease.yaml | 37 ++++++---------- services/openldap/statefulset.yaml | 4 ++ services/outline/deployment.yaml | 6 ++- services/planka/deployment.yaml | 6 ++- services/quality/oauth2-proxy-sonarqube.yaml | 4 ++ .../sonarqube-exporter-deployment.yaml | 4 ++ services/vaultwarden/deployment.yaml | 4 ++ testing/tests/test_hermes_cli_lane_metrics.py | 15 +++++++ 23 files changed, 132 insertions(+), 84 deletions(-) diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index 794f0869..faac8c41 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -40,7 +40,6 @@ resources: - cassandra-auth/kustomization.yaml - cassandra/kustomization.yaml - cassandra/image-automation.yaml - - veles/kustomization.yaml - typhon/kustomization.yaml - nextcloud/kustomization.yaml - nextcloud-mail-sync/kustomization.yaml diff --git a/infrastructure/core/coredns-deployment.yaml b/infrastructure/core/coredns-deployment.yaml index 0d8ff02b..0e8dafdb 100644 --- a/infrastructure/core/coredns-deployment.yaml +++ b/infrastructure/core/coredns-deployment.yaml @@ -9,7 +9,7 @@ metadata: kubernetes.io/name: CoreDNS spec: progressDeadlineSeconds: 600 - replicas: 2 + replicas: 3 revisionHistoryLimit: 0 selector: matchLabels: @@ -95,6 +95,19 @@ spec: operator: In values: - "true" + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/storage-backbone + operator: DoesNotExist + - weight: 80 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 dnsPolicy: Default nodeSelector: kubernetes.io/os: linux diff --git a/infrastructure/longhorn/core/kustomization.yaml b/infrastructure/longhorn/core/kustomization.yaml index 8f610d1b..c7675a55 100644 --- a/infrastructure/longhorn/core/kustomization.yaml +++ b/infrastructure/longhorn/core/kustomization.yaml @@ -8,7 +8,6 @@ resources: - vault-sync-deployment.yaml - helmrelease.yaml - cassandra-recurring-jobs.yaml - - veles-recurring-jobs.yaml - longhorn-settings-ensure-job.yaml - longhorn-csi-toleration-ensure-job.yaml - longhorn-disk-tags-ensure-job.yaml diff --git a/infrastructure/modules/base/priorityclass/kustomization.yaml b/infrastructure/modules/base/priorityclass/kustomization.yaml index d83776b7..a6962aab 100644 --- a/infrastructure/modules/base/priorityclass/kustomization.yaml +++ b/infrastructure/modules/base/priorityclass/kustomization.yaml @@ -5,4 +5,3 @@ resources: - scavenger.yaml - cassandra.yaml - media.yaml - - veles.yaml diff --git a/infrastructure/modules/base/storageclass/kustomization.yaml b/infrastructure/modules/base/storageclass/kustomization.yaml index 1898deb7..7eae3822 100644 --- a/infrastructure/modules/base/storageclass/kustomization.yaml +++ b/infrastructure/modules/base/storageclass/kustomization.yaml @@ -8,6 +8,3 @@ resources: - cassandra-db.yaml - cassandra-artifacts.yaml - cassandra-storage-policy.yaml - - veles-oceanus-db.yaml - - veles-oceanus-artifacts.yaml - - veles-oceanus-policy.yaml diff --git a/services/bstein-dev-home/chat-ai-gateway-deployment.yaml b/services/bstein-dev-home/chat-ai-gateway-deployment.yaml index 50d4be4f..7f718e8f 100644 --- a/services/bstein-dev-home/chat-ai-gateway-deployment.yaml +++ b/services/bstein-dev-home/chat-ai-gateway-deployment.yaml @@ -16,6 +16,10 @@ spec: app: chat-ai-gateway annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "bstein-dev-home" vault.hashicorp.com/agent-inject-secret-portal-env.sh: "kv/data/atlas/portal/atlas-portal-db" vault.hashicorp.com/agent-inject-template-portal-env.sh: | diff --git a/services/crypto/wallet-monero-temp/deployment.yaml b/services/crypto/wallet-monero-temp/deployment.yaml index 4b73f64c..07fcf6b1 100644 --- a/services/crypto/wallet-monero-temp/deployment.yaml +++ b/services/crypto/wallet-monero-temp/deployment.yaml @@ -19,6 +19,10 @@ spec: app: wallet-monero-temp annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "crypto" vault.hashicorp.com/agent-inject-secret-wallet-rpc-env.sh: "kv/data/atlas/crypto/wallet-monero-temp-rpc-auth" vault.hashicorp.com/agent-inject-template-wallet-rpc-env.sh: | @@ -68,7 +72,7 @@ spec: name: rpc resources: requests: - cpu: 100m + cpu: 50m memory: 128Mi limits: cpu: "1" diff --git a/services/finance/actual-budget-deployment.yaml b/services/finance/actual-budget-deployment.yaml index 637e9ae1..77ec6a0f 100644 --- a/services/finance/actual-budget-deployment.yaml +++ b/services/finance/actual-budget-deployment.yaml @@ -165,7 +165,7 @@ spec: failureThreshold: 6 resources: requests: - cpu: 200m + cpu: 50m memory: 512Mi limits: cpu: "1" diff --git a/services/game-stream/oauth2-proxy-wolf.yaml b/services/game-stream/oauth2-proxy-wolf.yaml index 5d2f92f2..bb340fca 100644 --- a/services/game-stream/oauth2-proxy-wolf.yaml +++ b/services/game-stream/oauth2-proxy-wolf.yaml @@ -33,6 +33,10 @@ spec: app: oauth2-proxy-wolf annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "game-stream" vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/game-stream/wolf-oidc" vault.hashicorp.com/agent-inject-template-oidc-config: | diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 8ba2972b..e1814d56 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -24,8 +24,8 @@ spec: ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback - ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260823-dual-provider-quota-health" + ai.bstein.dev/placement: titan-22 spare capacity; Jellyfin retains preemption priority + ai.bstein.dev/config-rev: "20260823-media-safe-load-spread" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" @@ -92,6 +92,7 @@ spec: vault.hashicorp.com/agent-limits-mem: 128Mi spec: serviceAccountName: hermes-agent + priorityClassName: scavenger enableServiceLinks: false automountServiceAccountToken: true securityContext: @@ -104,20 +105,25 @@ spec: - matchExpressions: - key: kubernetes.io/arch operator: In - values: [arm64] + values: [amd64] - key: node-role.kubernetes.io/worker operator: In values: ["true"] - key: kubernetes.io/hostname - operator: NotIn - values: [titan-04, titan-06, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19] + operator: In + values: [titan-22] preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: - - key: hardware + - key: atlas.bstein.dev/media-primary operator: In - values: [rpi5] + values: ["true"] + tolerations: + - key: atlas.bstein.dev/media-primary + operator: Equal + value: "true" + effect: PreferNoSchedule initContainers: - name: init-config image: busybox:1.37 @@ -921,10 +927,9 @@ spec: - {name: scm-boundary, mountPath: /opt/scm, readOnly: true} - {name: tmp, mountPath: /tmp} readinessProbe: - exec: - command: - - /opt/hermes/.venv/bin/python - - /opt/coordinator/cli_lane_capabilities.py + httpGet: + path: /healthz + port: lane-metrics initialDelaySeconds: 2 periodSeconds: 5 timeoutSeconds: 5 @@ -1188,6 +1193,7 @@ spec: - {name: HERMES_CLAUDE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key} - {name: HERMES_CLAUDE_BROKER_READ_TIMEOUT, value: "1800"} - {name: HERMES_CLAUDE_BROKER_CONCURRENCY, value: "2"} + - {name: HERMES_CLAUDE_HEALTH_POLL_SECONDS, value: "300"} - {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json} - {name: HERMES_ROUTING_CATALOG_PATH, value: /routing-catalog/catalog.json} readinessProbe: diff --git a/services/hermes/chat-statefulset.yaml b/services/hermes/chat-statefulset.yaml index 38756cf7..b71b2f54 100644 --- a/services/hermes/chat-statefulset.yaml +++ b/services/hermes/chat-statefulset.yaml @@ -64,8 +64,13 @@ spec: values: ["true"] - key: kubernetes.io/hostname operator: NotIn - values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19] + values: [titan-05, titan-13, titan-14, titan-17, titan-18, titan-19] preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/storage-backbone + operator: DoesNotExist - weight: 100 preference: matchExpressions: diff --git a/services/hermes/deployment.yaml b/services/hermes/deployment.yaml index 15c55b71..308a75ed 100644 --- a/services/hermes/deployment.yaml +++ b/services/hermes/deployment.yaml @@ -66,12 +66,16 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: - - titan-08 - titan-13 - titan-14 - titan-17 - titan-18 preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: node-role.kubernetes.io/storage-backbone + operator: DoesNotExist - weight: 80 preference: matchExpressions: diff --git a/services/hermes/scripts/cli_lane_dispatch.py b/services/hermes/scripts/cli_lane_dispatch.py index 2630f8d7..94724585 100644 --- a/services/hermes/scripts/cli_lane_dispatch.py +++ b/services/hermes/scripts/cli_lane_dispatch.py @@ -14,6 +14,7 @@ from cli_lane_board import _external, _record_board_access_error, _task_value from cli_lane_capabilities import ( initialize_kanban_capabilities, kanban_capabilities, + readiness_issue, refresh_kanban_capabilities, ) from cli_lane_config import ( @@ -162,7 +163,7 @@ def main() -> int: RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True) atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644) - start_metrics_server() + start_metrics_server(health_check=lambda: readiness_issue() is None) capabilities = initialize_kanban_capabilities(kanban_db) recover_orphans() workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8)) diff --git a/services/hermes/scripts/cli_lane_metrics.py b/services/hermes/scripts/cli_lane_metrics.py index e3192998..8551b25f 100644 --- a/services/hermes/scripts/cli_lane_metrics.py +++ b/services/hermes/scripts/cli_lane_metrics.py @@ -7,6 +7,7 @@ import os import sys import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Callable METRIC_HELP = { @@ -149,7 +150,11 @@ def record_soft_exclusion(provider: str, excluded: bool) -> None: ) -def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None: +def start_metrics_server( + port: int | None = None, + *, + health_check: Callable[[], bool] | None = None, +) -> ThreadingHTTPServer | None: """Serve lane metrics on the pod-local scrape port without blocking dispatch.""" selected_port = ( int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011")) @@ -167,7 +172,11 @@ def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None: self.end_headers() self.wfile.write(payload) elif self.path == "/healthz": - self.send_response(200) + try: + healthy = health_check is None or health_check() + except Exception: + healthy = False + self.send_response(200 if healthy else 503) self.end_headers() else: self.send_error(404) diff --git a/services/jenkins/deployment.yaml b/services/jenkins/deployment.yaml index 46109da7..258725c0 100644 --- a/services/jenkins/deployment.yaml +++ b/services/jenkins/deployment.yaml @@ -71,8 +71,9 @@ spec: bstein.dev/restarted-at: "2026-05-20T09:40:31Z" spec: serviceAccountName: jenkins + priorityClassName: scavenger nodeSelector: - kubernetes.io/arch: arm64 + kubernetes.io/arch: amd64 node-role.kubernetes.io/worker: "true" affinity: nodeAffinity: @@ -80,43 +81,20 @@ spec: nodeSelectorTerms: - matchExpressions: - key: kubernetes.io/hostname - operator: NotIn - values: - - titan-13 - - titan-15 - - titan-17 - - titan-19 - - key: hardware operator: In - values: ["rpi5"] + values: ["titan-22"] preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: - - key: atlas.bstein.dev/spillover - operator: DoesNotExist - - weight: 95 - preference: - matchExpressions: - - key: kubernetes.io/hostname - operator: NotIn - values: - - titan-13 - - titan-15 - - titan-17 - - titan-19 - - weight: 90 - preference: - matchExpressions: - - key: hardware + - key: atlas.bstein.dev/media-primary operator: In - values: ["rpi5"] - - weight: 50 - preference: - matchExpressions: - - key: hardware - operator: In - values: ["rpi4"] + values: ["true"] + tolerations: + - key: atlas.bstein.dev/media-primary + operator: Equal + value: "true" + effect: PreferNoSchedule hostAliases: - ip: 38.28.125.112 hostnames: diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index a938d8e2..760b4da8 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -70,6 +70,7 @@ metadata: spec: interval: 15m upgrade: + force: true disableWait: true chart: spec: @@ -81,6 +82,14 @@ spec: namespace: flux-system values: server: + # This single-node database is allowed to consume titan-22's spare + # capacity, but Jellyfin can preempt it whenever media capacity needs it. + priorityClassName: scavenger + tolerations: + - key: atlas.bstein.dev/media-primary + operator: Equal + value: "true" + effect: PreferNoSchedule # keep 1 year; supports "d", "y" extraArgs: retentionPeriod: "1y" # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11} @@ -114,38 +123,16 @@ spec: values: - "true" - key: kubernetes.io/hostname - operator: NotIn + operator: In values: - # Longhorn engine-image probes repeatedly fail on these - # attachment hosts and can surface volume I/O errors. - - titan-14 - - titan-18 - # Keep the metrics database off this saturated Pi 4. - - titan-12 - - titan-20 - - titan-21 - titan-22 - - titan-24 preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: - - key: hardware + - key: atlas.bstein.dev/media-primary operator: In - values: - - rpi5 - - weight: 50 - preference: - matchExpressions: - - key: hardware - operator: In - values: - - rpi4 - - weight: 25 - preference: - matchExpressions: - - key: node-role.kubernetes.io/storage-backbone - operator: Exists + values: ["true"] # Enable built-in Kubernetes scraping scrape: diff --git a/services/openldap/statefulset.yaml b/services/openldap/statefulset.yaml index bbf6686f..d22599b3 100644 --- a/services/openldap/statefulset.yaml +++ b/services/openldap/statefulset.yaml @@ -18,6 +18,10 @@ spec: app: openldap annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "sso" vault.hashicorp.com/agent-inject-secret-openldap-env: "kv/data/atlas/sso/openldap-admin" vault.hashicorp.com/agent-inject-template-openldap-env: | diff --git a/services/outline/deployment.yaml b/services/outline/deployment.yaml index 93798881..2553ab63 100644 --- a/services/outline/deployment.yaml +++ b/services/outline/deployment.yaml @@ -22,6 +22,10 @@ spec: app: outline annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "outline" vault.hashicorp.com/agent-inject-secret-outline-env.sh: "kv/data/atlas/outline/outline-db" vault.hashicorp.com/agent-inject-template-outline-env.sh: | @@ -152,7 +156,7 @@ spec: failureThreshold: 6 resources: requests: - cpu: 200m + cpu: 50m memory: 512Mi limits: cpu: "1" diff --git a/services/planka/deployment.yaml b/services/planka/deployment.yaml index 84465b3a..b753f2c2 100644 --- a/services/planka/deployment.yaml +++ b/services/planka/deployment.yaml @@ -22,6 +22,10 @@ spec: app: planka annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "planka" vault.hashicorp.com/agent-inject-secret-planka-env.sh: "kv/data/atlas/planka/planka-db" vault.hashicorp.com/agent-inject-template-planka-env.sh: | @@ -167,7 +171,7 @@ spec: failureThreshold: 6 resources: requests: - cpu: 200m + cpu: 50m memory: 512Mi limits: cpu: "1" diff --git a/services/quality/oauth2-proxy-sonarqube.yaml b/services/quality/oauth2-proxy-sonarqube.yaml index d780393d..4a62c390 100644 --- a/services/quality/oauth2-proxy-sonarqube.yaml +++ b/services/quality/oauth2-proxy-sonarqube.yaml @@ -34,6 +34,10 @@ spec: app: oauth2-proxy-sonarqube annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "quality" vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/quality/sonarqube-oidc" vault.hashicorp.com/agent-inject-template-oidc-config: | diff --git a/services/quality/sonarqube-exporter-deployment.yaml b/services/quality/sonarqube-exporter-deployment.yaml index 43dcd7f2..0fdbadd8 100644 --- a/services/quality/sonarqube-exporter-deployment.yaml +++ b/services/quality/sonarqube-exporter-deployment.yaml @@ -17,6 +17,10 @@ spec: app: sonarqube-exporter annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "quality" vault.hashicorp.com/agent-inject-secret-sonarqube-exporter-env.sh: "kv/data/atlas/quality/sonarqube-oidc" vault.hashicorp.com/agent-inject-template-sonarqube-exporter-env.sh: | diff --git a/services/vaultwarden/deployment.yaml b/services/vaultwarden/deployment.yaml index 5d1c5446..0bae1a6b 100644 --- a/services/vaultwarden/deployment.yaml +++ b/services/vaultwarden/deployment.yaml @@ -20,6 +20,10 @@ spec: app: vaultwarden annotations: vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-requests-cpu: 25m + vault.hashicorp.com/agent-requests-mem: 64Mi + vault.hashicorp.com/agent-limits-cpu: 100m + vault.hashicorp.com/agent-limits-mem: 128Mi vault.hashicorp.com/role: "vaultwarden" vault.hashicorp.com/agent-inject-secret-vaultwarden-env.sh: "kv/data/atlas/vaultwarden/vaultwarden-db-url" vault.hashicorp.com/agent-inject-template-vaultwarden-env.sh: | diff --git a/testing/tests/test_hermes_cli_lane_metrics.py b/testing/tests/test_hermes_cli_lane_metrics.py index ff832065..0608bbe1 100644 --- a/testing/tests/test_hermes_cli_lane_metrics.py +++ b/testing/tests/test_hermes_cli_lane_metrics.py @@ -104,6 +104,21 @@ def test_metrics_server_reads_port_from_environment(monkeypatch): server.server_close() +def test_metrics_server_reports_in_process_readiness_failure(): + server = lanes.start_metrics_server(0, health_check=lambda: False) + assert server is not None + try: + port = server.server_address[1] + try: + urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz") + raise AssertionError("failed lane readiness must return 503") + except urllib.error.HTTPError as error: + assert error.code == 503 + finally: + server.shutdown() + server.server_close() + + def test_metrics_server_declines_conflicting_port(capsys): first = lanes.start_metrics_server(0) assert first is not None