ops: spread saturated node workloads

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -40,7 +40,6 @@ resources:
   - cassandra-auth/kustomization.yaml
   - cassandra/kustomization.yaml
   - cassandra/image-automation.yaml
-  - veles/kustomization.yaml
   - typhon/kustomization.yaml
   - nextcloud/kustomization.yaml
   - nextcloud-mail-sync/kustomization.yaml

infrastructure/core/coredns-deployment.yaml

@@ -9,7 +9,7 @@ metadata:
     kubernetes.io/name: CoreDNS
 spec:
   progressDeadlineSeconds: 600
-  replicas: 2
+  replicas: 3
   revisionHistoryLimit: 0
   selector:
     matchLabels:
@@ -95,6 +95,19 @@ spec:
                     operator: In
                     values:
                       - "true"
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: node-role.kubernetes.io/storage-backbone
+                    operator: DoesNotExist
+            - weight: 80
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values:
+                      - rpi5
       dnsPolicy: Default
       nodeSelector:
         kubernetes.io/os: linux

infrastructure/longhorn/core/kustomization.yaml

@@ -8,7 +8,6 @@ resources:
   - vault-sync-deployment.yaml
   - helmrelease.yaml
   - cassandra-recurring-jobs.yaml
-  - veles-recurring-jobs.yaml
   - longhorn-settings-ensure-job.yaml
   - longhorn-csi-toleration-ensure-job.yaml
   - longhorn-disk-tags-ensure-job.yaml

infrastructure/modules/base/priorityclass/kustomization.yaml

@@ -5,4 +5,3 @@ resources:
   - scavenger.yaml
   - cassandra.yaml
   - media.yaml
-  - veles.yaml

infrastructure/modules/base/storageclass/kustomization.yaml

@@ -8,6 +8,3 @@ resources:
   - cassandra-db.yaml
   - cassandra-artifacts.yaml
   - cassandra-storage-policy.yaml
-  - veles-oceanus-db.yaml
-  - veles-oceanus-artifacts.yaml
-  - veles-oceanus-policy.yaml

services/bstein-dev-home/chat-ai-gateway-deployment.yaml

@@ -16,6 +16,10 @@ spec:
         app: chat-ai-gateway
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "bstein-dev-home"
         vault.hashicorp.com/agent-inject-secret-portal-env.sh: "kv/data/atlas/portal/atlas-portal-db"
         vault.hashicorp.com/agent-inject-template-portal-env.sh: |

services/crypto/wallet-monero-temp/deployment.yaml

@@ -19,6 +19,10 @@ spec:
         app: wallet-monero-temp
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "crypto"
         vault.hashicorp.com/agent-inject-secret-wallet-rpc-env.sh: "kv/data/atlas/crypto/wallet-monero-temp-rpc-auth"
         vault.hashicorp.com/agent-inject-template-wallet-rpc-env.sh: |
@@ -68,7 +72,7 @@ spec:
               name: rpc
           resources:
             requests:
-              cpu: 100m
+              cpu: 50m
               memory: 128Mi
             limits:
               cpu: "1"

services/finance/actual-budget-deployment.yaml

@@ -165,7 +165,7 @@ spec:
             failureThreshold: 6
           resources:
             requests:
-              cpu: 200m
+              cpu: 50m
               memory: 512Mi
             limits:
               cpu: "1"

services/game-stream/oauth2-proxy-wolf.yaml

@@ -33,6 +33,10 @@ spec:
         app: oauth2-proxy-wolf
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "game-stream"
         vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/game-stream/wolf-oidc"
         vault.hashicorp.com/agent-inject-template-oidc-config: |

services/hermes/agent-deployment.yaml

@@ -24,8 +24,8 @@ spec:
         ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
         ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
         ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
-        ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
+        ai.bstein.dev/placement: titan-22 spare capacity; Jellyfin retains preemption priority
-        ai.bstein.dev/config-rev: "20260823-dual-provider-quota-health"
+        ai.bstein.dev/config-rev: "20260823-media-safe-load-spread"
         prometheus.io/scrape: "true"
         prometheus.io/path: /metrics
         prometheus.io/port: "9010"
@@ -92,6 +92,7 @@ spec:
         vault.hashicorp.com/agent-limits-mem: 128Mi
     spec:
       serviceAccountName: hermes-agent
+      priorityClassName: scavenger
       enableServiceLinks: false
       automountServiceAccountToken: true
       securityContext:
@@ -104,20 +105,25 @@ spec:
               - matchExpressions:
                   - key: kubernetes.io/arch
                     operator: In
-                    values: [arm64]
+                    values: [amd64]
                   - key: node-role.kubernetes.io/worker
                     operator: In
                     values: ["true"]
                   - key: kubernetes.io/hostname
-                    operator: NotIn
+                    operator: In
-                    values: [titan-04, titan-06, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+                    values: [titan-22]
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
                 matchExpressions:
-                  - key: hardware
+                  - key: atlas.bstein.dev/media-primary
                     operator: In
-                    values: [rpi5]
+                    values: ["true"]
+      tolerations:
+        - key: atlas.bstein.dev/media-primary
+          operator: Equal
+          value: "true"
+          effect: PreferNoSchedule
       initContainers:
         - name: init-config
           image: busybox:1.37
@@ -921,10 +927,9 @@ spec:
             - {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
             - {name: tmp, mountPath: /tmp}
           readinessProbe:
-            exec:
+            httpGet:
-              command:
+              path: /healthz
-                - /opt/hermes/.venv/bin/python
+              port: lane-metrics
-                - /opt/coordinator/cli_lane_capabilities.py
             initialDelaySeconds: 2
             periodSeconds: 5
             timeoutSeconds: 5
@@ -1188,6 +1193,7 @@ spec:
             - {name: HERMES_CLAUDE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key}
             - {name: HERMES_CLAUDE_BROKER_READ_TIMEOUT, value: "1800"}
             - {name: HERMES_CLAUDE_BROKER_CONCURRENCY, value: "2"}
+            - {name: HERMES_CLAUDE_HEALTH_POLL_SECONDS, value: "300"}
             - {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json}
             - {name: HERMES_ROUTING_CATALOG_PATH, value: /routing-catalog/catalog.json}
           readinessProbe:

services/hermes/chat-statefulset.yaml

@@ -64,8 +64,13 @@ spec:
                     values: ["true"]
                   - key: kubernetes.io/hostname
                     operator: NotIn
-                    values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+                    values: [titan-05, titan-13, titan-14, titan-17, titan-18, titan-19]
           preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: node-role.kubernetes.io/storage-backbone
+                    operator: DoesNotExist
             - weight: 100
               preference:
                 matchExpressions:

services/hermes/deployment.yaml

@@ -66,12 +66,16 @@ spec:
                   - key: kubernetes.io/hostname
                     operator: NotIn
                     values:
-                      - titan-08
                       - titan-13
                       - titan-14
                       - titan-17
                       - titan-18
           preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: node-role.kubernetes.io/storage-backbone
+                    operator: DoesNotExist
             - weight: 80
               preference:
                 matchExpressions:

services/hermes/scripts/cli_lane_dispatch.py

@@ -14,6 +14,7 @@ from cli_lane_board import _external, _record_board_access_error, _task_value
 from cli_lane_capabilities import (
     initialize_kanban_capabilities,
     kanban_capabilities,
+    readiness_issue,
     refresh_kanban_capabilities,
 )
 from cli_lane_config import (
@@ -162,7 +163,7 @@ def main() -> int:
 
     RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
     atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
-    start_metrics_server()
+    start_metrics_server(health_check=lambda: readiness_issue() is None)
     capabilities = initialize_kanban_capabilities(kanban_db)
     recover_orphans()
     workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))

services/hermes/scripts/cli_lane_metrics.py

@@ -7,6 +7,7 @@ import os
 import sys
 import threading
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from typing import Callable
 
 
 METRIC_HELP = {
@@ -149,7 +150,11 @@ def record_soft_exclusion(provider: str, excluded: bool) -> None:
     )
 
 
-def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
+def start_metrics_server(
+    port: int | None = None,
+    *,
+    health_check: Callable[[], bool] | None = None,
+) -> ThreadingHTTPServer | None:
     """Serve lane metrics on the pod-local scrape port without blocking dispatch."""
     selected_port = (
         int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
@@ -167,7 +172,11 @@ def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
                 self.end_headers()
                 self.wfile.write(payload)
             elif self.path == "/healthz":
-                self.send_response(200)
+                try:
+                    healthy = health_check is None or health_check()
+                except Exception:
+                    healthy = False
+                self.send_response(200 if healthy else 503)
                 self.end_headers()
             else:
                 self.send_error(404)

services/jenkins/deployment.yaml

@@ -71,8 +71,9 @@ spec:
         bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
     spec:
       serviceAccountName: jenkins
+      priorityClassName: scavenger
       nodeSelector:
-        kubernetes.io/arch: arm64
+        kubernetes.io/arch: amd64
         node-role.kubernetes.io/worker: "true"
       affinity:
         nodeAffinity:
@@ -80,43 +81,20 @@ spec:
             nodeSelectorTerms:
               - matchExpressions:
                   - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-13
-                      - titan-15
-                      - titan-17
-                      - titan-19
-                  - key: hardware
                     operator: In
-                    values: ["rpi5"]
+                    values: ["titan-22"]
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
                 matchExpressions:
-                  - key: atlas.bstein.dev/spillover
+                  - key: atlas.bstein.dev/media-primary
-                    operator: DoesNotExist
-            - weight: 95
-              preference:
-                matchExpressions:
-                  - key: kubernetes.io/hostname
-                    operator: NotIn
-                    values:
-                      - titan-13
-                      - titan-15
-                      - titan-17
-                      - titan-19
-            - weight: 90
-              preference:
-                matchExpressions:
-                  - key: hardware
                     operator: In
-                    values: ["rpi5"]
+                    values: ["true"]
-            - weight: 50
+      tolerations:
-              preference:
+        - key: atlas.bstein.dev/media-primary
-                matchExpressions:
+          operator: Equal
-                  - key: hardware
+          value: "true"
-                    operator: In
+          effect: PreferNoSchedule
-                    values: ["rpi4"]
       hostAliases:
         - ip: 38.28.125.112
           hostnames:

services/monitoring/helmrelease.yaml

@@ -70,6 +70,7 @@ metadata:
 spec:
   interval: 15m
   upgrade:
+    force: true
     disableWait: true
   chart:
     spec:
@@ -81,6 +82,14 @@ spec:
         namespace: flux-system
   values:
     server:
+      # This single-node database is allowed to consume titan-22's spare
+      # capacity, but Jellyfin can preempt it whenever media capacity needs it.
+      priorityClassName: scavenger
+      tolerations:
+        - key: atlas.bstein.dev/media-primary
+          operator: Equal
+          value: "true"
+          effect: PreferNoSchedule
       # keep 1 year; supports "d", "y"
       extraArgs:
         retentionPeriod: "1y"   # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11}
@@ -114,38 +123,16 @@ spec:
                     values:
                       - "true"
                   - key: kubernetes.io/hostname
-                    operator: NotIn
+                    operator: In
                     values:
-                      # Longhorn engine-image probes repeatedly fail on these
-                      # attachment hosts and can surface volume I/O errors.
-                      - titan-14
-                      - titan-18
-                      # Keep the metrics database off this saturated Pi 4.
-                      - titan-12
-                      - titan-20
-                      - titan-21
                       - titan-22
-                      - titan-24
           preferredDuringSchedulingIgnoredDuringExecution:
             - weight: 100
               preference:
                 matchExpressions:
-                  - key: hardware
+                  - key: atlas.bstein.dev/media-primary
                     operator: In
-                    values:
+                    values: ["true"]
-                      - rpi5
-            - weight: 50
-              preference:
-                matchExpressions:
-                  - key: hardware
-                    operator: In
-                    values:
-                      - rpi4
-            - weight: 25
-              preference:
-                matchExpressions:
-                  - key: node-role.kubernetes.io/storage-backbone
-                    operator: Exists
 
       # Enable built-in Kubernetes scraping
       scrape:

services/openldap/statefulset.yaml

@@ -18,6 +18,10 @@ spec:
         app: openldap
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "sso"
         vault.hashicorp.com/agent-inject-secret-openldap-env: "kv/data/atlas/sso/openldap-admin"
         vault.hashicorp.com/agent-inject-template-openldap-env: |

services/outline/deployment.yaml

@@ -22,6 +22,10 @@ spec:
         app: outline
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "outline"
         vault.hashicorp.com/agent-inject-secret-outline-env.sh: "kv/data/atlas/outline/outline-db"
         vault.hashicorp.com/agent-inject-template-outline-env.sh: |
@@ -152,7 +156,7 @@ spec:
             failureThreshold: 6
           resources:
             requests:
-              cpu: 200m
+              cpu: 50m
               memory: 512Mi
             limits:
               cpu: "1"

services/planka/deployment.yaml

@@ -22,6 +22,10 @@ spec:
         app: planka
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "planka"
         vault.hashicorp.com/agent-inject-secret-planka-env.sh: "kv/data/atlas/planka/planka-db"
         vault.hashicorp.com/agent-inject-template-planka-env.sh: |
@@ -167,7 +171,7 @@ spec:
             failureThreshold: 6
           resources:
             requests:
-              cpu: 200m
+              cpu: 50m
               memory: 512Mi
             limits:
               cpu: "1"

services/quality/oauth2-proxy-sonarqube.yaml

@@ -34,6 +34,10 @@ spec:
         app: oauth2-proxy-sonarqube
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "quality"
         vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/quality/sonarqube-oidc"
         vault.hashicorp.com/agent-inject-template-oidc-config: |

services/quality/sonarqube-exporter-deployment.yaml

@@ -17,6 +17,10 @@ spec:
         app: sonarqube-exporter
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "quality"
         vault.hashicorp.com/agent-inject-secret-sonarqube-exporter-env.sh: "kv/data/atlas/quality/sonarqube-oidc"
         vault.hashicorp.com/agent-inject-template-sonarqube-exporter-env.sh: |

services/vaultwarden/deployment.yaml

@@ -20,6 +20,10 @@ spec:
         app: vaultwarden
       annotations:
         vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-requests-cpu: 25m
+        vault.hashicorp.com/agent-requests-mem: 64Mi
+        vault.hashicorp.com/agent-limits-cpu: 100m
+        vault.hashicorp.com/agent-limits-mem: 128Mi
         vault.hashicorp.com/role: "vaultwarden"
         vault.hashicorp.com/agent-inject-secret-vaultwarden-env.sh: "kv/data/atlas/vaultwarden/vaultwarden-db-url"
         vault.hashicorp.com/agent-inject-template-vaultwarden-env.sh: |

testing/tests/test_hermes_cli_lane_metrics.py

@@ -104,6 +104,21 @@ def test_metrics_server_reads_port_from_environment(monkeypatch):
     server.server_close()
 
 
+def test_metrics_server_reports_in_process_readiness_failure():
+    server = lanes.start_metrics_server(0, health_check=lambda: False)
+    assert server is not None
+    try:
+        port = server.server_address[1]
+        try:
+            urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz")
+            raise AssertionError("failed lane readiness must return 503")
+        except urllib.error.HTTPError as error:
+            assert error.code == 503
+    finally:
+        server.shutdown()
+        server.server_close()
+
+
 def test_metrics_server_declines_conflicting_port(capsys):
     first = lanes.start_metrics_server(0)
     assert first is not None