titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity

ops: spread saturated node workloads

Browse Source
This commit is contained in:
jenkins 2026-08-23 10:37:01 -03:00
parent a019ecd556
commit 271f3e8c32
23 changed files with 132 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
clusters/atlas/flux-system/applications/kustomization.yaml
View File

@ -40,7 +40,6 @@ resources:
- cassandra-auth/kustomization.yaml
- cassandra/kustomization.yaml
- cassandra/image-automation.yaml
- veles/kustomization.yaml
- typhon/kustomization.yaml
- nextcloud/kustomization.yaml
- nextcloud-mail-sync/kustomization.yaml

15
infrastructure/core/coredns-deployment.yaml
View File

@ -9,7 +9,7 @@ metadata:
kubernetes.io/name: CoreDNS
spec:
progressDeadlineSeconds: 600
replicas: 2
replicas: 3
revisionHistoryLimit: 0
selector:
matchLabels:
@ -95,6 +95,19 @@ spec:
operator: In
values:
- "true"
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: node-role.kubernetes.io/storage-backbone
operator: DoesNotExist
- weight: 80
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi5
dnsPolicy: Default
nodeSelector:
kubernetes.io/os: linux

1
infrastructure/longhorn/core/kustomization.yaml
View File

@ -8,7 +8,6 @@ resources:
- vault-sync-deployment.yaml
- helmrelease.yaml
- cassandra-recurring-jobs.yaml
- veles-recurring-jobs.yaml
- longhorn-settings-ensure-job.yaml
- longhorn-csi-toleration-ensure-job.yaml
- longhorn-disk-tags-ensure-job.yaml

1
infrastructure/modules/base/priorityclass/kustomization.yaml
View File

@ -5,4 +5,3 @@ resources:
- scavenger.yaml
- cassandra.yaml
- media.yaml
- veles.yaml

3
infrastructure/modules/base/storageclass/kustomization.yaml
View File

@ -8,6 +8,3 @@ resources:
- cassandra-db.yaml
- cassandra-artifacts.yaml
- cassandra-storage-policy.yaml
- veles-oceanus-db.yaml
- veles-oceanus-artifacts.yaml
- veles-oceanus-policy.yaml

4
services/bstein-dev-home/chat-ai-gateway-deployment.yaml
View File

@ -16,6 +16,10 @@ spec:
app: chat-ai-gateway
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "bstein-dev-home"
vault.hashicorp.com/agent-inject-secret-portal-env.sh: "kv/data/atlas/portal/atlas-portal-db"
vault.hashicorp.com/agent-inject-template-portal-env.sh: |

6
services/crypto/wallet-monero-temp/deployment.yaml
View File

@ -19,6 +19,10 @@ spec:
app: wallet-monero-temp
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "crypto"
vault.hashicorp.com/agent-inject-secret-wallet-rpc-env.sh: "kv/data/atlas/crypto/wallet-monero-temp-rpc-auth"
vault.hashicorp.com/agent-inject-template-wallet-rpc-env.sh: |
@ -68,7 +72,7 @@ spec:
name: rpc
resources:
requests:
cpu: 100m
cpu: 50m
memory: 128Mi
limits:
cpu: "1"

2
services/finance/actual-budget-deployment.yaml
View File

@ -165,7 +165,7 @@ spec:
failureThreshold: 6
resources:
requests:
cpu: 200m
cpu: 50m
memory: 512Mi
limits:
cpu: "1"

4
services/game-stream/oauth2-proxy-wolf.yaml
View File

@ -33,6 +33,10 @@ spec:
app: oauth2-proxy-wolf
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "game-stream"
vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/game-stream/wolf-oidc"
vault.hashicorp.com/agent-inject-template-oidc-config: |

28
services/hermes/agent-deployment.yaml
View File

@ -24,8 +24,8 @@ spec:
ai.bstein.dev/router-wire-contract: ollama-numeric-keepalive
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
ai.bstein.dev/config-rev: "20260823-dual-provider-quota-health"
ai.bstein.dev/placement: titan-22 spare capacity; Jellyfin retains preemption priority
ai.bstein.dev/config-rev: "20260823-media-safe-load-spread"
prometheus.io/scrape: "true"
prometheus.io/path: /metrics
prometheus.io/port: "9010"
@ -92,6 +92,7 @@ spec:
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-agent
priorityClassName: scavenger
enableServiceLinks: false
automountServiceAccountToken: true
securityContext:
@ -104,20 +105,25 @@ spec:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values: [arm64]
values: [amd64]
- key: node-role.kubernetes.io/worker
operator: In
values: ["true"]
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-04, titan-06, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
operator: In
values: [titan-22]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
- key: atlas.bstein.dev/media-primary
operator: In
values: [rpi5]
values: ["true"]
tolerations:
- key: atlas.bstein.dev/media-primary
operator: Equal
value: "true"
effect: PreferNoSchedule
initContainers:
- name: init-config
image: busybox:1.37
@ -921,10 +927,9 @@ spec:
- {name: scm-boundary, mountPath: /opt/scm, readOnly: true}
- {name: tmp, mountPath: /tmp}
readinessProbe:
exec:
command:
- /opt/hermes/.venv/bin/python
- /opt/coordinator/cli_lane_capabilities.py
httpGet:
path: /healthz
port: lane-metrics
initialDelaySeconds: 2
periodSeconds: 5
timeoutSeconds: 5
@ -1188,6 +1193,7 @@ spec:
- {name: HERMES_CLAUDE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key}
- {name: HERMES_CLAUDE_BROKER_READ_TIMEOUT, value: "1800"}
- {name: HERMES_CLAUDE_BROKER_CONCURRENCY, value: "2"}
- {name: HERMES_CLAUDE_HEALTH_POLL_SECONDS, value: "300"}
- {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json}
- {name: HERMES_ROUTING_CATALOG_PATH, value: /routing-catalog/catalog.json}
readinessProbe:

7
services/hermes/chat-statefulset.yaml
View File

@ -64,8 +64,13 @@ spec:
values: ["true"]
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
values: [titan-05, titan-13, titan-14, titan-17, titan-18, titan-19]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: node-role.kubernetes.io/storage-backbone
operator: DoesNotExist
- weight: 100
preference:
matchExpressions:

6
services/hermes/deployment.yaml
View File

@ -66,12 +66,16 @@ spec:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-08
- titan-13
- titan-14
- titan-17
- titan-18
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: node-role.kubernetes.io/storage-backbone
operator: DoesNotExist
- weight: 80
preference:
matchExpressions:

3
services/hermes/scripts/cli_lane_dispatch.py
View File

@ -14,6 +14,7 @@ from cli_lane_board import _external, _record_board_access_error, _task_value
from cli_lane_capabilities import (
initialize_kanban_capabilities,
kanban_capabilities,
readiness_issue,
refresh_kanban_capabilities,
)
from cli_lane_config import (
@ -162,7 +163,7 @@ def main() -> int:
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
atomic_json(RESULT_SCHEMA_PATH, RESULT_SCHEMA, 0o644)
start_metrics_server()
start_metrics_server(health_check=lambda: readiness_issue() is None)
capabilities = initialize_kanban_capabilities(kanban_db)
recover_orphans()
workers = max(1, min(int(os.environ.get("HERMES_CLI_LANE_CONCURRENCY", "4")), 8))

13
services/hermes/scripts/cli_lane_metrics.py
View File

@ -7,6 +7,7 @@ import os
import sys
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Callable
METRIC_HELP = {
@ -149,7 +150,11 @@ def record_soft_exclusion(provider: str, excluded: bool) -> None:
)
def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
def start_metrics_server(
port: int | None = None,
*,
health_check: Callable[[], bool] | None = None,
) -> ThreadingHTTPServer | None:
"""Serve lane metrics on the pod-local scrape port without blocking dispatch."""
selected_port = (
int(os.environ.get("HERMES_CLI_LANE_METRICS_PORT", "9011"))
@ -167,7 +172,11 @@ def start_metrics_server(port: int | None = None) -> ThreadingHTTPServer | None:
self.end_headers()
self.wfile.write(payload)
elif self.path == "/healthz":
self.send_response(200)
try:
healthy = health_check is None or health_check()
except Exception:
healthy = False
self.send_response(200 if healthy else 503)
self.end_headers()
else:
self.send_error(404)

42
services/jenkins/deployment.yaml
View File

@ -71,8 +71,9 @@ spec:
bstein.dev/restarted-at: "2026-05-20T09:40:31Z"
spec:
serviceAccountName: jenkins
priorityClassName: scavenger
nodeSelector:
kubernetes.io/arch: arm64
kubernetes.io/arch: amd64
node-role.kubernetes.io/worker: "true"
affinity:
nodeAffinity:
@ -80,43 +81,20 @@ spec:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
- key: hardware
operator: In
values: ["rpi5"]
values: ["titan-22"]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: atlas.bstein.dev/spillover
operator: DoesNotExist
- weight: 95
preference:
matchExpressions:
- key: kubernetes.io/hostname
operator: NotIn
values:
- titan-13
- titan-15
- titan-17
- titan-19
- weight: 90
preference:
matchExpressions:
- key: hardware
- key: atlas.bstein.dev/media-primary
operator: In
values: ["rpi5"]
- weight: 50
preference:
matchExpressions:
- key: hardware
operator: In
values: ["rpi4"]
values: ["true"]
tolerations:
- key: atlas.bstein.dev/media-primary
operator: Equal
value: "true"
effect: PreferNoSchedule
hostAliases:
- ip: 38.28.125.112
hostnames:

37
services/monitoring/helmrelease.yaml
View File

@ -70,6 +70,7 @@ metadata:
spec:
interval: 15m
upgrade:
force: true
disableWait: true
chart:
spec:
@ -81,6 +82,14 @@ spec:
namespace: flux-system
values:
server:
# This single-node database is allowed to consume titan-22's spare
# capacity, but Jellyfin can preempt it whenever media capacity needs it.
priorityClassName: scavenger
tolerations:
- key: atlas.bstein.dev/media-primary
operator: Equal
value: "true"
effect: PreferNoSchedule
# keep 1 year; supports "d", "y"
extraArgs:
retentionPeriod: "1y" # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11}
@ -114,38 +123,16 @@ spec:
values:
- "true"
- key: kubernetes.io/hostname
operator: NotIn
operator: In
values:
# Longhorn engine-image probes repeatedly fail on these
# attachment hosts and can surface volume I/O errors.
- titan-14
- titan-18
# Keep the metrics database off this saturated Pi 4.
- titan-12
- titan-20
- titan-21
- titan-22
- titan-24
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: hardware
- key: atlas.bstein.dev/media-primary
operator: In
values:
- rpi5
- weight: 50
preference:
matchExpressions:
- key: hardware
operator: In
values:
- rpi4
- weight: 25
preference:
matchExpressions:
- key: node-role.kubernetes.io/storage-backbone
operator: Exists
values: ["true"]
# Enable built-in Kubernetes scraping
scrape:

4
services/openldap/statefulset.yaml
View File

@ -18,6 +18,10 @@ spec:
app: openldap
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "sso"
vault.hashicorp.com/agent-inject-secret-openldap-env: "kv/data/atlas/sso/openldap-admin"
vault.hashicorp.com/agent-inject-template-openldap-env: |

6
services/outline/deployment.yaml
View File

@ -22,6 +22,10 @@ spec:
app: outline
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "outline"
vault.hashicorp.com/agent-inject-secret-outline-env.sh: "kv/data/atlas/outline/outline-db"
vault.hashicorp.com/agent-inject-template-outline-env.sh: |
@ -152,7 +156,7 @@ spec:
failureThreshold: 6
resources:
requests:
cpu: 200m
cpu: 50m
memory: 512Mi
limits:
cpu: "1"

6
services/planka/deployment.yaml
View File

@ -22,6 +22,10 @@ spec:
app: planka
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "planka"
vault.hashicorp.com/agent-inject-secret-planka-env.sh: "kv/data/atlas/planka/planka-db"
vault.hashicorp.com/agent-inject-template-planka-env.sh: |
@ -167,7 +171,7 @@ spec:
failureThreshold: 6
resources:
requests:
cpu: 200m
cpu: 50m
memory: 512Mi
limits:
cpu: "1"

4
services/quality/oauth2-proxy-sonarqube.yaml
View File

@ -34,6 +34,10 @@ spec:
app: oauth2-proxy-sonarqube
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "quality"
vault.hashicorp.com/agent-inject-secret-oidc-config: "kv/data/atlas/quality/sonarqube-oidc"
vault.hashicorp.com/agent-inject-template-oidc-config: |

4
services/quality/sonarqube-exporter-deployment.yaml
View File

@ -17,6 +17,10 @@ spec:
app: sonarqube-exporter
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "quality"
vault.hashicorp.com/agent-inject-secret-sonarqube-exporter-env.sh: "kv/data/atlas/quality/sonarqube-oidc"
vault.hashicorp.com/agent-inject-template-sonarqube-exporter-env.sh: |

4
services/vaultwarden/deployment.yaml
View File

@ -20,6 +20,10 @@ spec:
app: vaultwarden
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-requests-cpu: 25m
vault.hashicorp.com/agent-requests-mem: 64Mi
vault.hashicorp.com/agent-limits-cpu: 100m
vault.hashicorp.com/agent-limits-mem: 128Mi
vault.hashicorp.com/role: "vaultwarden"
vault.hashicorp.com/agent-inject-secret-vaultwarden-env.sh: "kv/data/atlas/vaultwarden/vaultwarden-db-url"
vault.hashicorp.com/agent-inject-template-vaultwarden-env.sh: |

15
testing/tests/test_hermes_cli_lane_metrics.py
View File

@ -104,6 +104,21 @@ def test_metrics_server_reads_port_from_environment(monkeypatch):
server.server_close()
def test_metrics_server_reports_in_process_readiness_failure():
server = lanes.start_metrics_server(0, health_check=lambda: False)
assert server is not None
try:
port = server.server_address[1]
try:
urllib.request.urlopen(f"http://127.0.0.1:{port}/healthz")
raise AssertionError("failed lane readiness must return 503")
except urllib.error.HTTPError as error:
assert error.code == 503
finally:
server.shutdown()
server.server_close()
def test_metrics_server_declines_conflicting_port(capsys):
first = lanes.start_metrics_server(0)
assert first is not None