From 441977449e92d2cb1df729127a40085f905dc46a Mon Sep 17 00:00:00 2001 From: jenkins Date: Fri, 14 Aug 2026 23:54:35 -0300 Subject: [PATCH] gitea: move Atlas repositories to organization --- Jenkinsfile | 2 +- README.md | 2 +- ci/Jenkinsfile.titan-iac | 2 +- clusters/atlas/flux-system/gotk-sync.yaml | 2 +- knowledge/catalog/atlas-summary.json | 6 +- knowledge/catalog/atlas.json | 306 ++++++++++++++---- knowledge/catalog/atlas.yaml | 267 ++++++++++----- knowledge/catalog/runbooks.json | 2 +- knowledge/diagrams/atlas-http.mmd | 6 +- knowledge/hermes-automated-triage-24h-plan.md | 4 +- knowledge/runbooks/cluster-power-recovery.md | 4 +- scripts/bootstrap/recovery-config.env | 2 +- scripts/ops/cluster_power_recovery.sh | 2 +- scripts/ops/hermes_triage_monitor.py | 2 +- scripts/sync/comms_sync_kb.sh | 3 +- .../knowledge/catalog/atlas-summary.json | 6 +- services/comms/knowledge/catalog/atlas.json | 306 ++++++++++++++---- services/comms/knowledge/catalog/atlas.yaml | 267 ++++++++++----- .../comms/knowledge/catalog/runbooks.json | 2 +- .../comms/knowledge/diagrams/atlas-http.mmd | 6 +- .../hermes-automated-triage-24h-plan.md | 4 +- .../runbooks/cluster-power-recovery.md | 4 +- .../gitea/atlas-identity-bootstrap-job.yaml | 6 +- .../scripts/gitea_atlas_identity_ensure.sh | 75 ++++- services/hermes/scripts/hermes_coordinator.py | 62 +++- services/jenkins/configmap-jcasc.yaml | 20 +- .../maintenance/apps/ariadne-deployment.yaml | 4 +- testing/tests/test_hermes_chat_quality.py | 6 +- testing/tests/test_hermes_coordinator.py | 98 +++++- 29 files changed, 1141 insertions(+), 337 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f84c6982..3b324fa0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -480,7 +480,7 @@ PY set +x git config user.email "jenkins@bstein.dev" git config user.name "jenkins" - git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git + git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/atlas/titan-iac.git git push origin HEAD:${FLUX_BRANCH} ''' } diff --git a/README.md b/README.md index 9ec9bd4c..8154c6f2 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ Flux-managed Kubernetes desired-state config for `bstein.dev`. Canonical source URL: -- `ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` +- `ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git` ## Scope diff --git a/ci/Jenkinsfile.titan-iac b/ci/Jenkinsfile.titan-iac index d4564884..c2283e08 100644 --- a/ci/Jenkinsfile.titan-iac +++ b/ci/Jenkinsfile.titan-iac @@ -479,7 +479,7 @@ PY set +x git config user.email "jenkins@bstein.dev" git config user.name "jenkins" - git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/bstein/titan-iac.git + git remote set-url origin https://${GIT_USER}:${GIT_TOKEN}@scm.bstein.dev/atlas/titan-iac.git git push origin HEAD:${FLUX_BRANCH} ''' } diff --git a/clusters/atlas/flux-system/gotk-sync.yaml b/clusters/atlas/flux-system/gotk-sync.yaml index b9d38c2c..30a1a7f9 100644 --- a/clusters/atlas/flux-system/gotk-sync.yaml +++ b/clusters/atlas/flux-system/gotk-sync.yaml @@ -12,7 +12,7 @@ spec: branch: main secretRef: name: flux-system-gitea - url: ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git + url: ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git --- apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization diff --git a/knowledge/catalog/atlas-summary.json b/knowledge/catalog/atlas-summary.json index ac8f8e56..004af8ca 100644 --- a/knowledge/catalog/atlas-summary.json +++ b/knowledge/catalog/atlas-summary.json @@ -1,8 +1,8 @@ { "counts": { "helmrelease_host_hints": 23, - "http_endpoints": 60, - "services": 85, - "workloads": 119 + "http_endpoints": 61, + "services": 91, + "workloads": 124 } } diff --git a/knowledge/catalog/atlas.json b/knowledge/catalog/atlas.json index c2de82ba..d88e6d2b 100644 --- a/knowledge/catalog/atlas.json +++ b/knowledge/catalog/atlas.json @@ -304,7 +304,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-454" + "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464" ] }, { @@ -320,7 +320,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-454" + "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464" ] }, { @@ -365,7 +365,7 @@ "kubernetes.io/arch": "amd64" }, "images": [ - "registry.bstein.dev/cassandra/cassandra-backend:0.9.63" + "registry.bstein.dev/cassandra/cassandra-backend:0.9.65" ] }, { @@ -378,7 +378,7 @@ "serviceAccountName": "cassandra-frontend", "nodeSelector": {}, "images": [ - "registry.bstein.dev/cassandra/cassandra-frontend:0.9.63" + "registry.bstein.dev/cassandra/cassandra-frontend:0.9.65" ] }, { @@ -902,8 +902,8 @@ "serviceAccountName": "hermes-triage", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f", - "registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d" + "registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1", + "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" ] }, { @@ -916,8 +916,8 @@ "serviceAccountName": "hermes-agent", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5", - "registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d" + "quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561", + "registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1" ] }, { @@ -930,7 +930,7 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-chat-router@sha256:882f9c3af7268618a19424765536f4cf6c37e8d380e515780baf4104e5b2931e" + "registry.bstein.dev/bstein/hermes-chat-router@sha256:e3794fb8b9ee76d699893ba303d7080f54698c0a06064dc32293e99e988cabff" ] }, { @@ -989,6 +989,75 @@ "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-4", + "labels": { + "ai.bstein.dev/tenant-ordinal": "4", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-5", + "labels": { + "ai.bstein.dev/tenant-ordinal": "5", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-6", + "labels": { + "ai.bstein.dev/tenant-ordinal": "6", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-7", + "labels": { + "ai.bstein.dev/tenant-ordinal": "7", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-local-image", + "labels": { + "app": "hermes-local-image" + }, + "serviceAccountName": "hermes-gpu-runtime", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-local-image@sha256:d6257f49a60244fb1e30733848a187fa932acebfdb82a2560d4e760fa018b645" + ] + }, { "kind": "Deployment", "namespace": "hermes", @@ -1005,14 +1074,14 @@ { "kind": "Deployment", "namespace": "hermes", - "name": "hermes-ollama", + "name": "hermes-oauth-sessions", "labels": { - "app": "hermes-ollama" + "app": "hermes-oauth-sessions" }, "serviceAccountName": null, "nodeSelector": {}, "images": [ - "ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d" + "redis:7.4.1-alpine@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275" ] }, { @@ -1030,6 +1099,20 @@ "registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-switchyard", + "labels": { + "app": "hermes-switchyard" + }, + "serviceAccountName": "hermes-switchyard", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-switchyard-brokers@sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083", + "registry.bstein.dev/bstein/hermes-switchyard@sha256:3d952d528a4e4cb8afdf84f8995272f5d6107dc127b68cc20234b1da4aff43eb" + ] + }, { "kind": "Deployment", "namespace": "hermes", @@ -1039,25 +1122,12 @@ }, "serviceAccountName": null, "nodeSelector": { - "kubernetes.io/hostname": "titan-20" + "kubernetes.io/hostname": "titan-21" }, "images": [ "registry.bstein.dev/bstein/hermes-jetson-tts@sha256:5cb9e57faab46365bff606c559af57b9505892be2909aea1ac523568d478b2cc" ] }, - { - "kind": "Deployment", - "namespace": "hermes", - "name": "oauth2-proxy-hermes-agent", - "labels": { - "app": "oauth2-proxy-hermes-agent" - }, - "serviceAccountName": "hermes-agent", - "nodeSelector": {}, - "images": [ - "quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561" - ] - }, { "kind": "Deployment", "namespace": "hermes", @@ -1094,8 +1164,8 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:10522c69676e250b5d0014d811ed9eb706d0b7c16f35cf66650f480e63ab3ab5", - "registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d" + "registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1", + "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" ] }, { @@ -1445,7 +1515,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-amd64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-292-amd64" ] }, { @@ -1461,7 +1531,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-arm64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-291-arm64" ] }, { @@ -1550,7 +1620,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/ariadne:0.1.0-448" + "registry.bstein.dev/bstein/ariadne:0.1.0-454" ] }, { @@ -1578,7 +1648,7 @@ "serviceAccountName": "metis", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/metis:0.1.0-283-arm64" + "registry.bstein.dev/bstein/metis:0.1.0-291-arm64" ] }, { @@ -3020,28 +3090,6 @@ } ] }, - { - "namespace": "hermes", - "name": "hermes-agent", - "type": "ClusterIP", - "selector": { - "app": "hermes-agent" - }, - "ports": [ - { - "name": "dashboard", - "port": 8787, - "targetPort": "dashboard", - "protocol": "TCP" - }, - { - "name": "terminal", - "port": 7681, - "targetPort": "terminal", - "protocol": "TCP" - } - ] - }, { "namespace": "hermes", "name": "hermes-chat-router", @@ -3096,6 +3144,86 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-claude-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9006, + "targetPort": "claude-broker", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-codex-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-gpu-handoff", + "type": "ClusterIP", + "selector": { + "app": "hermes-model-gate" + }, + "ports": [ + { + "name": "http", + "port": 11434, + "targetPort": "handoff", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-image-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9002, + "targetPort": "image-broker", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-local-image", + "type": "ClusterIP", + "selector": { + "app": "hermes-local-image" + }, + "ports": [ + { + "name": "http", + "port": 9004, + "targetPort": "local-image", + "protocol": "TCP" + } + ] + }, { "namespace": "hermes", "name": "hermes-model-gate", @@ -3114,16 +3242,16 @@ }, { "namespace": "hermes", - "name": "hermes-ollama", + "name": "hermes-oauth-sessions", "type": "ClusterIP", "selector": { - "app": "hermes-ollama" + "app": "hermes-oauth-sessions" }, "ports": [ { - "name": "http", - "port": 11434, - "targetPort": "http", + "name": "redis", + "port": 6379, + "targetPort": "redis", "protocol": "TCP" } ] @@ -3144,6 +3272,22 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-switchyard", + "type": "ClusterIP", + "selector": { + "app": "hermes-switchyard" + }, + "ports": [ + { + "name": "http", + "port": 9005, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "hermes", "name": "hermes-triage", @@ -3187,13 +3331,13 @@ "name": "oauth2-proxy-hermes-agent", "type": "ClusterIP", "selector": { - "app": "oauth2-proxy-hermes-agent" + "app": "hermes-agent" }, "ports": [ { "name": "http", "port": 80, - "targetPort": "http", + "targetPort": "auth-http", "protocol": "TCP" } ] @@ -3492,6 +3636,22 @@ } ] }, + { + "namespace": "monitoring", + "name": "nvidia-process-exporter-local", + "type": "ClusterIP", + "selector": { + "app": "nvidia-process-exporter" + }, + "ports": [ + { + "name": "metrics", + "port": 9401, + "targetPort": "metrics", + "protocol": "TCP" + } + ] + }, { "namespace": "monitoring", "name": "platform-quality-gateway", @@ -3934,13 +4094,33 @@ "workloads": [ { "kind": "Deployment", - "name": "oauth2-proxy-hermes-agent" + "name": "hermes-agent" } ] }, "via": { "kind": "Ingress", - "name": "hermes-sites", + "name": "hermes-agent-dashboard", + "source": "hermes" + } + }, + { + "host": "agent.hermes.bstein.dev", + "path": "/terminal", + "backend": { + "namespace": "hermes", + "service": "oauth2-proxy-hermes-agent", + "port": "http", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes-agent" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "hermes-agent-terminal", "source": "hermes" } }, diff --git a/knowledge/catalog/atlas.yaml b/knowledge/catalog/atlas.yaml index f7e9e0e3..434335af 100644 --- a/knowledge/catalog/atlas.yaml +++ b/knowledge/catalog/atlas.yaml @@ -187,7 +187,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-454 + - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-frontend @@ -198,7 +198,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-454 + - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-vault-sync @@ -229,7 +229,7 @@ workloads: cassandra.bstein.dev/node-pool: oceanus kubernetes.io/arch: amd64 images: - - registry.bstein.dev/cassandra/cassandra-backend:0.9.63 + - registry.bstein.dev/cassandra/cassandra-backend:0.9.65 - kind: Deployment namespace: cassandra name: cassandra-frontend @@ -238,7 +238,7 @@ workloads: serviceAccountName: cassandra-frontend nodeSelector: {} images: - - registry.bstein.dev/cassandra/cassandra-frontend:0.9.63 + - registry.bstein.dev/cassandra/cassandra-frontend:0.9.65 - kind: Deployment namespace: cassandra name: cassandra-vault-sync @@ -604,8 +604,8 @@ workloads: serviceAccountName: hermes-triage nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f - - registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1 + - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 - kind: Deployment namespace: hermes name: hermes-agent @@ -614,8 +614,8 @@ workloads: serviceAccountName: hermes-agent nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 - - registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 + - registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1 - kind: Deployment namespace: hermes name: hermes-chat-router @@ -624,7 +624,7 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-chat-router@sha256:882f9c3af7268618a19424765536f4cf6c37e8d380e515780baf4104e5b2931e + - registry.bstein.dev/bstein/hermes-chat-router@sha256:e3794fb8b9ee76d699893ba303d7080f54698c0a06064dc32293e99e988cabff - kind: Deployment namespace: hermes name: hermes-chat-sandbox-0 @@ -665,6 +665,55 @@ workloads: nodeSelector: {} images: - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-4 + labels: + ai.bstein.dev/tenant-ordinal: '4' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-5 + labels: + ai.bstein.dev/tenant-ordinal: '5' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-6 + labels: + ai.bstein.dev/tenant-ordinal: '6' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-7 + labels: + ai.bstein.dev/tenant-ordinal: '7' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-local-image + labels: + app: hermes-local-image + serviceAccountName: hermes-gpu-runtime + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-local-image@sha256:d6257f49a60244fb1e30733848a187fa932acebfdb82a2560d4e760fa018b645 - kind: Deployment namespace: hermes name: hermes-model-gate @@ -676,13 +725,13 @@ workloads: - python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df - kind: Deployment namespace: hermes - name: hermes-ollama + name: hermes-oauth-sessions labels: - app: hermes-ollama + app: hermes-oauth-sessions serviceAccountName: null nodeSelector: {} images: - - ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d + - redis:7.4.1-alpine@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275 - kind: Deployment namespace: hermes name: hermes-stt @@ -693,6 +742,16 @@ workloads: kubernetes.io/hostname: titan-21 images: - registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f +- kind: Deployment + namespace: hermes + name: hermes-switchyard + labels: + app: hermes-switchyard + serviceAccountName: hermes-switchyard + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-switchyard-brokers@sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083 + - registry.bstein.dev/bstein/hermes-switchyard@sha256:3d952d528a4e4cb8afdf84f8995272f5d6107dc127b68cc20234b1da4aff43eb - kind: Deployment namespace: hermes name: hermes-tts @@ -700,18 +759,9 @@ workloads: app: hermes-tts serviceAccountName: null nodeSelector: - kubernetes.io/hostname: titan-20 + kubernetes.io/hostname: titan-21 images: - registry.bstein.dev/bstein/hermes-jetson-tts@sha256:5cb9e57faab46365bff606c559af57b9505892be2909aea1ac523568d478b2cc -- kind: Deployment - namespace: hermes - name: oauth2-proxy-hermes-agent - labels: - app: oauth2-proxy-hermes-agent - serviceAccountName: hermes-agent - nodeSelector: {} - images: - - quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 - kind: Deployment namespace: hermes name: oauth2-proxy-hermes-chat @@ -738,8 +788,8 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:10522c69676e250b5d0014d811ed9eb706d0b7c16f35cf66650f480e63ab3ab5 - - registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1 + - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 - kind: Deployment namespace: jellyfin name: jellyfin @@ -978,7 +1028,7 @@ workloads: kubernetes.io/arch: amd64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-amd64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-292-amd64 - kind: DaemonSet namespace: maintenance name: metis-sentinel-arm64 @@ -989,7 +1039,7 @@ workloads: kubernetes.io/arch: arm64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-arm64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-291-arm64 - kind: DaemonSet namespace: maintenance name: node-image-sweeper @@ -1049,7 +1099,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/ariadne:0.1.0-448 + - registry.bstein.dev/bstein/ariadne:0.1.0-454 - kind: Deployment namespace: maintenance name: maintenance-vault-sync @@ -1068,7 +1118,7 @@ workloads: serviceAccountName: metis nodeSelector: {} images: - - registry.bstein.dev/bstein/metis:0.1.0-283-arm64 + - registry.bstein.dev/bstein/metis:0.1.0-291-arm64 - kind: Deployment namespace: maintenance name: oauth2-proxy-metis @@ -2013,20 +2063,6 @@ services: port: 9119 targetPort: dashboard protocol: TCP -- namespace: hermes - name: hermes-agent - type: ClusterIP - selector: - app: hermes-agent - ports: - - name: dashboard - port: 8787 - targetPort: dashboard - protocol: TCP - - name: terminal - port: 7681 - targetPort: terminal - protocol: TCP - namespace: hermes name: hermes-chat-router type: ClusterIP @@ -2061,6 +2097,56 @@ services: port: 8642 targetPort: api protocol: TCP +- namespace: hermes + name: hermes-claude-broker + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9006 + targetPort: claude-broker + protocol: TCP +- namespace: hermes + name: hermes-codex-broker + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9003 + targetPort: codex-broker + protocol: TCP +- namespace: hermes + name: hermes-gpu-handoff + type: ClusterIP + selector: + app: hermes-model-gate + ports: + - name: http + port: 11434 + targetPort: handoff + protocol: TCP +- namespace: hermes + name: hermes-image-broker + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9002 + targetPort: image-broker + protocol: TCP +- namespace: hermes + name: hermes-local-image + type: ClusterIP + selector: + app: hermes-local-image + ports: + - name: http + port: 9004 + targetPort: local-image + protocol: TCP - namespace: hermes name: hermes-model-gate type: ClusterIP @@ -2072,14 +2158,14 @@ services: targetPort: http protocol: TCP - namespace: hermes - name: hermes-ollama + name: hermes-oauth-sessions type: ClusterIP selector: - app: hermes-ollama + app: hermes-oauth-sessions ports: - - name: http - port: 11434 - targetPort: http + - name: redis + port: 6379 + targetPort: redis protocol: TCP - namespace: hermes name: hermes-stt @@ -2091,6 +2177,16 @@ services: port: 9000 targetPort: http protocol: TCP +- namespace: hermes + name: hermes-switchyard + type: ClusterIP + selector: + app: hermes-switchyard + ports: + - name: http + port: 9005 + targetPort: http + protocol: TCP - namespace: hermes name: hermes-triage type: ClusterIP @@ -2119,11 +2215,11 @@ services: name: oauth2-proxy-hermes-agent type: ClusterIP selector: - app: oauth2-proxy-hermes-agent + app: hermes-agent ports: - name: http port: 80 - targetPort: http + targetPort: auth-http protocol: TCP - namespace: hermes name: oauth2-proxy-hermes-chat @@ -2311,6 +2407,16 @@ services: port: 9401 targetPort: metrics protocol: TCP +- namespace: monitoring + name: nvidia-process-exporter-local + type: ClusterIP + selector: + app: nvidia-process-exporter + ports: + - name: metrics + port: 9401 + targetPort: metrics + protocol: TCP - namespace: monitoring name: platform-quality-gateway type: ClusterIP @@ -2588,12 +2694,23 @@ http_endpoints: namespace: hermes service: oauth2-proxy-hermes-agent port: http - workloads: + workloads: &id001 - kind: Deployment - name: oauth2-proxy-hermes-agent + name: hermes-agent via: kind: Ingress - name: hermes-sites + name: hermes-agent-dashboard + source: hermes +- host: agent.hermes.bstein.dev + path: /terminal + backend: + namespace: hermes + service: oauth2-proxy-hermes-agent + port: http + workloads: *id001 + via: + kind: Ingress + name: hermes-agent-terminal source: hermes - host: auth.bstein.dev path: / @@ -2640,7 +2757,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: &id001 + workloads: &id002 - kind: Deployment name: matrix-wellknown via: @@ -2653,7 +2770,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown-bstein-dev @@ -2716,7 +2833,7 @@ http_endpoints: namespace: cassandra service: cassandra-backend port: 80 - workloads: &id002 + workloads: &id003 - kind: Deployment name: cassandra-backend via: @@ -2729,7 +2846,7 @@ http_endpoints: namespace: cassandra service: cassandra-backend port: 80 - workloads: *id002 + workloads: *id003 via: kind: Ingress name: cassandra @@ -2740,7 +2857,7 @@ http_endpoints: namespace: cassandra service: cassandra-backend port: 80 - workloads: *id002 + workloads: *id003 via: kind: Ingress name: cassandra @@ -2842,7 +2959,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown @@ -2853,7 +2970,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown @@ -2875,7 +2992,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: &id004 + workloads: &id005 - kind: Deployment name: matrix-guest-register via: @@ -2888,7 +3005,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: &id003 + workloads: &id004 - kind: Deployment name: matrix-authentication-service via: @@ -2901,7 +3018,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -2912,7 +3029,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -2923,7 +3040,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: *id004 + workloads: *id005 via: kind: Ingress name: matrix-routing @@ -2971,7 +3088,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -2982,7 +3099,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown-matrix-live @@ -2993,7 +3110,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown-matrix-live @@ -3015,7 +3132,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: *id004 + workloads: *id005 via: kind: Ingress name: matrix-routing @@ -3026,7 +3143,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -3037,7 +3154,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -3048,7 +3165,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -3059,7 +3176,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: *id004 + workloads: *id005 via: kind: Ingress name: matrix-routing @@ -3276,7 +3393,7 @@ http_endpoints: namespace: veles service: veles-backend port: 80 - workloads: &id005 + workloads: &id006 - kind: Deployment name: veles-backend via: @@ -3289,7 +3406,7 @@ http_endpoints: namespace: veles service: veles-backend port: 80 - workloads: *id005 + workloads: *id006 via: kind: Ingress name: veles @@ -3300,7 +3417,7 @@ http_endpoints: namespace: veles service: veles-backend port: 80 - workloads: *id005 + workloads: *id006 via: kind: Ingress name: veles diff --git a/knowledge/catalog/runbooks.json b/knowledge/catalog/runbooks.json index bef71b94..f0d7306f 100644 --- a/knowledge/catalog/runbooks.json +++ b/knowledge/catalog/runbooks.json @@ -26,7 +26,7 @@ "tags": [], "entrypoints": [], "source_paths": [], - "body": "Atlas Cluster Power Recovery (Graceful Shutdown/Startup)\n\nPurpose\n- Provide a safe operator flow for planned power events and cold-boot recovery.\n- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path.\n- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle.\n- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery.\n\nBootstrapping risk to remember\n- Flux source is Git over SSH to `scm.bstein.dev` (Gitea).\n- Gitea itself is a Flux-managed workload and depends on storage + database.\n- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images.\n- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall.\n- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.\n\nScript\n- `scripts/ops/cluster_power_recovery.sh`\n- `scripts/ops/cluster_power_console.sh`\n- Modes:\n - `prepare`\n - `shutdown`\n - `harbor-seed`\n - `startup`\n - `status`\n- Default is dry-run. Add `--execute` to actually perform actions.\n\nDry-run examples\n- Shutdown preview:\n - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`\n- Startup preview:\n - `scripts/ops/cluster_power_recovery.sh startup`\n- Harbor seed preview:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed`\n\nExecute examples\n- Prepare helper image on every node:\n - `scripts/ops/cluster_power_recovery.sh prepare --execute`\n- Seed Harbor runtime images onto `titan-05` from the control-host bundle:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`\n- Planned shutdown:\n - `scripts/ops/cluster_power_recovery.sh shutdown --execute`\n- Planned startup (canonical branch):\n - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`\n\nManual remote console examples\n- Canonical operator hosts:\n - `titan-db`\n - `tethys` (`titan-24`)\n- Both hosts now have:\n - `~/ananke-tools/cluster_power_recovery.sh`\n - `~/ananke-tools/cluster_power_console.sh`\n - `~/ananke-tools/bootstrap/recovery-config.env`\n - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt`\n - `~/ananke-tools/kubeconfig`\n - `~/ananke-cluster-power`\n - `~/bin/ananke-cluster-power`\n - `~/ananke-repo/{infrastructure,services,scripts}`\n- Both hosts also keep the Harbor bootstrap bundle at:\n - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n- Remote usage:\n - `ssh titan-db`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n - `ssh tethys`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n\nUseful options\n- `--shutdown-mode host-poweroff|cluster-only`\n- `--expected-flux-branch main`\n- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-branch main`\n- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only)\n- `--skip-local-bootstrap` (not recommended for cold-start recovery)\n- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred)\n- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node)\n- `--skip-helper-prewarm`\n- `--min-startup-battery 35`\n- `--ups-host pyrphoros@localhost`\n- `--require-ups-battery`\n- `--drain-timeout 180`\n- `--emergency-drain-timeout 45`\n- `--flux-ready-timeout 1200`\n- `--startup-checklist-timeout 900`\n- `--startup-stability-window 180`\n- `--startup-stability-timeout 900`\n- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state`\n- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n\nControlled drill checklist (recommended)\n- Operator host: use `titan-db` as canonical control host for the drill.\n- On-site coordination:\n - Have on-site operator ready before shutdown starts.\n - Confirm they will manually power cluster nodes back on after shutdown completes.\n - Confirm who will announce \"all nodes powered on\" to resume startup.\n- Preflight on `titan-db`:\n - `mkdir -p ~/ananke-logs`\n - `~/ananke-cluster-power status` and verify:\n - `ups_host=pyrphoros@localhost`\n - `ups_battery` is numeric\n - `flux_source_ready=True`\n- Warm helper image just before shutdown:\n - `~/ananke-cluster-power prepare --execute`\n- Run in a persistent shell and capture logs:\n - `tmux new -s ananke-drill`\n - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log`\n- Execute controlled shutdown with telemetry enforcement:\n - `~/ananke-cluster-power shutdown --execute --require-ups-battery`\n- After on-site power-on confirmation, execute startup:\n - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery`\n- Post-check:\n - `~/ananke-cluster-power status`\n - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures.\n\nOperational notes\n- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn.\n- Shutdown behavior is explicit:\n - `host-poweroff` schedules host poweroff after service stop.\n - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off.\n- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted.\n- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`).\n- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo.\n- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`.\n- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.\n- Harbor is reconciled after the first critical stateful services.\n- Harbor bootstrap is now designed around a control-host bundle:\n - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.\n - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.\n - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.\n- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.\n- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls.\n- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap.\n- Startup completion is strict now:\n - all non-optional Flux kustomizations must be `Ready=True`\n - external service checklist must pass (defaults include Gitea, Grafana, Harbor)\n - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`)\n - stability soak must pass with no crashloop/pull-failure churn\n- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile.\n- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster.\n- Dry-run mode no longer mutates outage recovery state.\n- `harbor-seed --execute` was validated by:\n - prewarming the helper image across all nodes\n - streaming the Harbor bootstrap bundle to `titan-05`\n - importing Harbor runtime images into host `containerd`\n - successfully running a Harbor-backed canary pod (`harbor-canary-ok`)\n- After bootstrap, Flux resources are resumed and reconciled.\n- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`." + "body": "Atlas Cluster Power Recovery (Graceful Shutdown/Startup)\n\nPurpose\n- Provide a safe operator flow for planned power events and cold-boot recovery.\n- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path.\n- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle.\n- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery.\n\nBootstrapping risk to remember\n- Flux source is Git over SSH to `scm.bstein.dev` (Gitea).\n- Gitea itself is a Flux-managed workload and depends on storage + database.\n- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images.\n- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall.\n- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.\n\nScript\n- `scripts/ops/cluster_power_recovery.sh`\n- `scripts/ops/cluster_power_console.sh`\n- Modes:\n - `prepare`\n - `shutdown`\n - `harbor-seed`\n - `startup`\n - `status`\n- Default is dry-run. Add `--execute` to actually perform actions.\n\nDry-run examples\n- Shutdown preview:\n - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`\n- Startup preview:\n - `scripts/ops/cluster_power_recovery.sh startup`\n- Harbor seed preview:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed`\n\nExecute examples\n- Prepare helper image on every node:\n - `scripts/ops/cluster_power_recovery.sh prepare --execute`\n- Seed Harbor runtime images onto `titan-05` from the control-host bundle:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`\n- Planned shutdown:\n - `scripts/ops/cluster_power_recovery.sh shutdown --execute`\n- Planned startup (canonical branch):\n - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`\n\nManual remote console examples\n- Canonical operator hosts:\n - `titan-db`\n - `tethys` (`titan-24`)\n- Both hosts now have:\n - `~/ananke-tools/cluster_power_recovery.sh`\n - `~/ananke-tools/cluster_power_console.sh`\n - `~/ananke-tools/bootstrap/recovery-config.env`\n - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt`\n - `~/ananke-tools/kubeconfig`\n - `~/ananke-cluster-power`\n - `~/bin/ananke-cluster-power`\n - `~/ananke-repo/{infrastructure,services,scripts}`\n- Both hosts also keep the Harbor bootstrap bundle at:\n - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n- Remote usage:\n - `ssh titan-db`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n - `ssh tethys`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n\nUseful options\n- `--shutdown-mode host-poweroff|cluster-only`\n- `--expected-flux-branch main`\n- `--expected-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git`\n- `--force-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git`\n- `--force-flux-branch main`\n- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only)\n- `--skip-local-bootstrap` (not recommended for cold-start recovery)\n- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred)\n- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node)\n- `--skip-helper-prewarm`\n- `--min-startup-battery 35`\n- `--ups-host pyrphoros@localhost`\n- `--require-ups-battery`\n- `--drain-timeout 180`\n- `--emergency-drain-timeout 45`\n- `--flux-ready-timeout 1200`\n- `--startup-checklist-timeout 900`\n- `--startup-stability-window 180`\n- `--startup-stability-timeout 900`\n- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state`\n- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n\nControlled drill checklist (recommended)\n- Operator host: use `titan-db` as canonical control host for the drill.\n- On-site coordination:\n - Have on-site operator ready before shutdown starts.\n - Confirm they will manually power cluster nodes back on after shutdown completes.\n - Confirm who will announce \"all nodes powered on\" to resume startup.\n- Preflight on `titan-db`:\n - `mkdir -p ~/ananke-logs`\n - `~/ananke-cluster-power status` and verify:\n - `ups_host=pyrphoros@localhost`\n - `ups_battery` is numeric\n - `flux_source_ready=True`\n- Warm helper image just before shutdown:\n - `~/ananke-cluster-power prepare --execute`\n- Run in a persistent shell and capture logs:\n - `tmux new -s ananke-drill`\n - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log`\n- Execute controlled shutdown with telemetry enforcement:\n - `~/ananke-cluster-power shutdown --execute --require-ups-battery`\n- After on-site power-on confirmation, execute startup:\n - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery`\n- Post-check:\n - `~/ananke-cluster-power status`\n - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures.\n\nOperational notes\n- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn.\n- Shutdown behavior is explicit:\n - `host-poweroff` schedules host poweroff after service stop.\n - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off.\n- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted.\n- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`).\n- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo.\n- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`.\n- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.\n- Harbor is reconciled after the first critical stateful services.\n- Harbor bootstrap is now designed around a control-host bundle:\n - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.\n - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.\n - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.\n- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.\n- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls.\n- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap.\n- Startup completion is strict now:\n - all non-optional Flux kustomizations must be `Ready=True`\n - external service checklist must pass (defaults include Gitea, Grafana, Harbor)\n - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`)\n - stability soak must pass with no crashloop/pull-failure churn\n- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile.\n- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster.\n- Dry-run mode no longer mutates outage recovery state.\n- `harbor-seed --execute` was validated by:\n - prewarming the helper image across all nodes\n - streaming the Harbor bootstrap bundle to `titan-05`\n - importing Harbor runtime images into host `containerd`\n - successfully running a Harbor-backed canary pod (`harbor-canary-ok`)\n- After bootstrap, Flux resources are resumed and reconciled.\n- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`." }, { "path": "runbooks/comms-verify.md", diff --git a/knowledge/diagrams/atlas-http.mmd b/knowledge/diagrams/atlas-http.mmd index 8c0eb353..11b55c8a 100644 --- a/knowledge/diagrams/atlas-http.mmd +++ b/knowledge/diagrams/atlas-http.mmd @@ -2,8 +2,8 @@ flowchart LR host_agent_hermes_bstein_dev["agent.hermes.bstein.dev"] svc_hermes_oauth2_proxy_hermes_agent["hermes/oauth2-proxy-hermes-agent (Service)"] host_agent_hermes_bstein_dev --> svc_hermes_oauth2_proxy_hermes_agent - wl_hermes_oauth2_proxy_hermes_agent["hermes/oauth2-proxy-hermes-agent (Deployment)"] - svc_hermes_oauth2_proxy_hermes_agent --> wl_hermes_oauth2_proxy_hermes_agent + wl_hermes_hermes_agent["hermes/hermes-agent (Deployment)"] + svc_hermes_oauth2_proxy_hermes_agent --> wl_hermes_hermes_agent host_auth_bstein_dev["auth.bstein.dev"] svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"] host_auth_bstein_dev --> svc_sso_oauth2_proxy @@ -248,7 +248,7 @@ flowchart LR end subgraph hermes[hermes] svc_hermes_oauth2_proxy_hermes_agent - wl_hermes_oauth2_proxy_hermes_agent + wl_hermes_hermes_agent svc_hermes_oauth2_proxy_hermes_chat wl_hermes_oauth2_proxy_hermes_chat svc_hermes_oauth2_proxy_hermes_triage diff --git a/knowledge/hermes-automated-triage-24h-plan.md b/knowledge/hermes-automated-triage-24h-plan.md index bf798a0b..7228e89e 100644 --- a/knowledge/hermes-automated-triage-24h-plan.md +++ b/knowledge/hermes-automated-triage-24h-plan.md @@ -1005,7 +1005,7 @@ The handoff must state plainly whether the full Hermes Agent or the Ollama fallb | Bounded source-patch proposal → branch → pull request | live (demo repo) | | Human-required metric, vmalert rule, Alertmanager | live | | Gitea issue filing for human-required incidents | live, proven on 4 real repos | -| Candidate-file selection for real service repos | live, PR opened on `bstein/ariadne` | +| Candidate-file selection for real service repos | live, PR opened on `atlas/ariadne` | | Narrowed alerting (repair failed, or unattended 6h) | live | ## What the fixture and code demos actually prove @@ -1055,7 +1055,7 @@ Code proposal: red build 22:34:59 -> pull request open 22:35:32 (33s). A one-line defect was seeded in `ariadne/utils/errors.py` (dropping the `errorMessage` key from the HTTP payload lookup) and pushed to master. Build 409 went red, and Ariadne opened -[bstein/ariadne PR #3](https://scm.bstein.dev/bstein/ariadne/pulls/3) from +[atlas/ariadne PR #3](https://scm.bstein.dev/atlas/ariadne/pulls/3) from branch `hermes-repair/409` with the correct one-line fix: ```diff diff --git a/knowledge/runbooks/cluster-power-recovery.md b/knowledge/runbooks/cluster-power-recovery.md index 86436d4c..ee80c101 100644 --- a/knowledge/runbooks/cluster-power-recovery.md +++ b/knowledge/runbooks/cluster-power-recovery.md @@ -72,8 +72,8 @@ Manual remote console examples Useful options - `--shutdown-mode host-poweroff|cluster-only` - `--expected-flux-branch main` -- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` -- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` +- `--expected-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git` +- `--force-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git` - `--force-flux-branch main` - `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only) - `--skip-local-bootstrap` (not recommended for cold-start recovery) diff --git a/scripts/bootstrap/recovery-config.env b/scripts/bootstrap/recovery-config.env index ee1dcab7..548bc7e7 100644 --- a/scripts/bootstrap/recovery-config.env +++ b/scripts/bootstrap/recovery-config.env @@ -1,6 +1,6 @@ CANONICAL_CONTROL_HOST="titan-db" DEFAULT_FLUX_BRANCH="main" -EXPECTED_FLUX_URL="ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git" +EXPECTED_FLUX_URL="ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git" SHUTDOWN_MODE="host-poweroff" STATE_SUBDIR=".local/share/ananke" HARBOR_BUNDLE_BASENAME="harbor-bootstrap-v2.14.1-arm64.tar.zst" diff --git a/scripts/ops/cluster_power_recovery.sh b/scripts/ops/cluster_power_recovery.sh index 157512ce..8bcf8874 100755 --- a/scripts/ops/cluster_power_recovery.sh +++ b/scripts/ops/cluster_power_recovery.sh @@ -107,7 +107,7 @@ esac EXECUTE=0 SHUTDOWN_MODE="${SHUTDOWN_MODE:-host-poweroff}" EXPECTED_FLUX_BRANCH="${DEFAULT_FLUX_BRANCH:-main}" -EXPECTED_FLUX_URL="${EXPECTED_FLUX_URL:-ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git}" +EXPECTED_FLUX_URL="${EXPECTED_FLUX_URL:-ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git}" ALLOW_FLUX_SOURCE_MUTATION=0 FORCE_FLUX_URL="" FORCE_FLUX_BRANCH="" diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index 303f182a..24ea9714 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -409,7 +409,7 @@ def evidence_for(key: str, incident: str = "") -> None: f" whole Ariadne response box, not out of one branch. Every path ends" f" here: an executed action, an escalation, or a pull request all record" f" the same audit events and metrics.{RESET}") - print(f" {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n") + print(f" {YELLOW}issues filed by triage: {GITEA}/atlas/ariadne/issues{RESET}\n") elif key == "verify": if IS_CODE_JOB: print(f" {DIM}the branch build validates the proposal. Nothing merges" diff --git a/scripts/sync/comms_sync_kb.sh b/scripts/sync/comms_sync_kb.sh index 894f06b8..46667f67 100755 --- a/scripts/sync/comms_sync_kb.sh +++ b/scripts/sync/comms_sync_kb.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash set -euo pipefail -python scripts/render/knowledge_render_atlas.py --write -python scripts/render/knowledge_render_atlas.py --write --out services/comms/knowledge +python scripts/render/knowledge_render_atlas.py --write --sync-comms diff --git a/services/comms/knowledge/catalog/atlas-summary.json b/services/comms/knowledge/catalog/atlas-summary.json index ac8f8e56..004af8ca 100644 --- a/services/comms/knowledge/catalog/atlas-summary.json +++ b/services/comms/knowledge/catalog/atlas-summary.json @@ -1,8 +1,8 @@ { "counts": { "helmrelease_host_hints": 23, - "http_endpoints": 60, - "services": 85, - "workloads": 119 + "http_endpoints": 61, + "services": 91, + "workloads": 124 } } diff --git a/services/comms/knowledge/catalog/atlas.json b/services/comms/knowledge/catalog/atlas.json index c2de82ba..d88e6d2b 100644 --- a/services/comms/knowledge/catalog/atlas.json +++ b/services/comms/knowledge/catalog/atlas.json @@ -304,7 +304,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-454" + "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464" ] }, { @@ -320,7 +320,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-454" + "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464" ] }, { @@ -365,7 +365,7 @@ "kubernetes.io/arch": "amd64" }, "images": [ - "registry.bstein.dev/cassandra/cassandra-backend:0.9.63" + "registry.bstein.dev/cassandra/cassandra-backend:0.9.65" ] }, { @@ -378,7 +378,7 @@ "serviceAccountName": "cassandra-frontend", "nodeSelector": {}, "images": [ - "registry.bstein.dev/cassandra/cassandra-frontend:0.9.63" + "registry.bstein.dev/cassandra/cassandra-frontend:0.9.65" ] }, { @@ -902,8 +902,8 @@ "serviceAccountName": "hermes-triage", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f", - "registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d" + "registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1", + "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" ] }, { @@ -916,8 +916,8 @@ "serviceAccountName": "hermes-agent", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5", - "registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d" + "quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561", + "registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1" ] }, { @@ -930,7 +930,7 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-chat-router@sha256:882f9c3af7268618a19424765536f4cf6c37e8d380e515780baf4104e5b2931e" + "registry.bstein.dev/bstein/hermes-chat-router@sha256:e3794fb8b9ee76d699893ba303d7080f54698c0a06064dc32293e99e988cabff" ] }, { @@ -989,6 +989,75 @@ "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-4", + "labels": { + "ai.bstein.dev/tenant-ordinal": "4", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-5", + "labels": { + "ai.bstein.dev/tenant-ordinal": "5", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-6", + "labels": { + "ai.bstein.dev/tenant-ordinal": "6", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-chat-sandbox-7", + "labels": { + "ai.bstein.dev/tenant-ordinal": "7", + "app": "hermes-chat-sandbox" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-local-image", + "labels": { + "app": "hermes-local-image" + }, + "serviceAccountName": "hermes-gpu-runtime", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-local-image@sha256:d6257f49a60244fb1e30733848a187fa932acebfdb82a2560d4e760fa018b645" + ] + }, { "kind": "Deployment", "namespace": "hermes", @@ -1005,14 +1074,14 @@ { "kind": "Deployment", "namespace": "hermes", - "name": "hermes-ollama", + "name": "hermes-oauth-sessions", "labels": { - "app": "hermes-ollama" + "app": "hermes-oauth-sessions" }, "serviceAccountName": null, "nodeSelector": {}, "images": [ - "ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d" + "redis:7.4.1-alpine@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275" ] }, { @@ -1030,6 +1099,20 @@ "registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-switchyard", + "labels": { + "app": "hermes-switchyard" + }, + "serviceAccountName": "hermes-switchyard", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-switchyard-brokers@sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083", + "registry.bstein.dev/bstein/hermes-switchyard@sha256:3d952d528a4e4cb8afdf84f8995272f5d6107dc127b68cc20234b1da4aff43eb" + ] + }, { "kind": "Deployment", "namespace": "hermes", @@ -1039,25 +1122,12 @@ }, "serviceAccountName": null, "nodeSelector": { - "kubernetes.io/hostname": "titan-20" + "kubernetes.io/hostname": "titan-21" }, "images": [ "registry.bstein.dev/bstein/hermes-jetson-tts@sha256:5cb9e57faab46365bff606c559af57b9505892be2909aea1ac523568d478b2cc" ] }, - { - "kind": "Deployment", - "namespace": "hermes", - "name": "oauth2-proxy-hermes-agent", - "labels": { - "app": "oauth2-proxy-hermes-agent" - }, - "serviceAccountName": "hermes-agent", - "nodeSelector": {}, - "images": [ - "quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561" - ] - }, { "kind": "Deployment", "namespace": "hermes", @@ -1094,8 +1164,8 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:10522c69676e250b5d0014d811ed9eb706d0b7c16f35cf66650f480e63ab3ab5", - "registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d" + "registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1", + "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" ] }, { @@ -1445,7 +1515,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-amd64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-292-amd64" ] }, { @@ -1461,7 +1531,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-arm64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-291-arm64" ] }, { @@ -1550,7 +1620,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/ariadne:0.1.0-448" + "registry.bstein.dev/bstein/ariadne:0.1.0-454" ] }, { @@ -1578,7 +1648,7 @@ "serviceAccountName": "metis", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/metis:0.1.0-283-arm64" + "registry.bstein.dev/bstein/metis:0.1.0-291-arm64" ] }, { @@ -3020,28 +3090,6 @@ } ] }, - { - "namespace": "hermes", - "name": "hermes-agent", - "type": "ClusterIP", - "selector": { - "app": "hermes-agent" - }, - "ports": [ - { - "name": "dashboard", - "port": 8787, - "targetPort": "dashboard", - "protocol": "TCP" - }, - { - "name": "terminal", - "port": 7681, - "targetPort": "terminal", - "protocol": "TCP" - } - ] - }, { "namespace": "hermes", "name": "hermes-chat-router", @@ -3096,6 +3144,86 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-claude-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9006, + "targetPort": "claude-broker", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-codex-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9003, + "targetPort": "codex-broker", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-gpu-handoff", + "type": "ClusterIP", + "selector": { + "app": "hermes-model-gate" + }, + "ports": [ + { + "name": "http", + "port": 11434, + "targetPort": "handoff", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-image-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9002, + "targetPort": "image-broker", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-local-image", + "type": "ClusterIP", + "selector": { + "app": "hermes-local-image" + }, + "ports": [ + { + "name": "http", + "port": 9004, + "targetPort": "local-image", + "protocol": "TCP" + } + ] + }, { "namespace": "hermes", "name": "hermes-model-gate", @@ -3114,16 +3242,16 @@ }, { "namespace": "hermes", - "name": "hermes-ollama", + "name": "hermes-oauth-sessions", "type": "ClusterIP", "selector": { - "app": "hermes-ollama" + "app": "hermes-oauth-sessions" }, "ports": [ { - "name": "http", - "port": 11434, - "targetPort": "http", + "name": "redis", + "port": 6379, + "targetPort": "redis", "protocol": "TCP" } ] @@ -3144,6 +3272,22 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-switchyard", + "type": "ClusterIP", + "selector": { + "app": "hermes-switchyard" + }, + "ports": [ + { + "name": "http", + "port": 9005, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "hermes", "name": "hermes-triage", @@ -3187,13 +3331,13 @@ "name": "oauth2-proxy-hermes-agent", "type": "ClusterIP", "selector": { - "app": "oauth2-proxy-hermes-agent" + "app": "hermes-agent" }, "ports": [ { "name": "http", "port": 80, - "targetPort": "http", + "targetPort": "auth-http", "protocol": "TCP" } ] @@ -3492,6 +3636,22 @@ } ] }, + { + "namespace": "monitoring", + "name": "nvidia-process-exporter-local", + "type": "ClusterIP", + "selector": { + "app": "nvidia-process-exporter" + }, + "ports": [ + { + "name": "metrics", + "port": 9401, + "targetPort": "metrics", + "protocol": "TCP" + } + ] + }, { "namespace": "monitoring", "name": "platform-quality-gateway", @@ -3934,13 +4094,33 @@ "workloads": [ { "kind": "Deployment", - "name": "oauth2-proxy-hermes-agent" + "name": "hermes-agent" } ] }, "via": { "kind": "Ingress", - "name": "hermes-sites", + "name": "hermes-agent-dashboard", + "source": "hermes" + } + }, + { + "host": "agent.hermes.bstein.dev", + "path": "/terminal", + "backend": { + "namespace": "hermes", + "service": "oauth2-proxy-hermes-agent", + "port": "http", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes-agent" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "hermes-agent-terminal", "source": "hermes" } }, diff --git a/services/comms/knowledge/catalog/atlas.yaml b/services/comms/knowledge/catalog/atlas.yaml index f7e9e0e3..434335af 100644 --- a/services/comms/knowledge/catalog/atlas.yaml +++ b/services/comms/knowledge/catalog/atlas.yaml @@ -187,7 +187,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-454 + - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-frontend @@ -198,7 +198,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-454 + - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-vault-sync @@ -229,7 +229,7 @@ workloads: cassandra.bstein.dev/node-pool: oceanus kubernetes.io/arch: amd64 images: - - registry.bstein.dev/cassandra/cassandra-backend:0.9.63 + - registry.bstein.dev/cassandra/cassandra-backend:0.9.65 - kind: Deployment namespace: cassandra name: cassandra-frontend @@ -238,7 +238,7 @@ workloads: serviceAccountName: cassandra-frontend nodeSelector: {} images: - - registry.bstein.dev/cassandra/cassandra-frontend:0.9.63 + - registry.bstein.dev/cassandra/cassandra-frontend:0.9.65 - kind: Deployment namespace: cassandra name: cassandra-vault-sync @@ -604,8 +604,8 @@ workloads: serviceAccountName: hermes-triage nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:15c5c538c0b58686af2e54e10bc870b23284789d485a609349df24ed3053622f - - registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1 + - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 - kind: Deployment namespace: hermes name: hermes-agent @@ -614,8 +614,8 @@ workloads: serviceAccountName: hermes-agent nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:46de11122226a68adad6d902e81ef9b4b53deb0433dcc61f57fa43fae2e3a3a5 - - registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 + - registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1 - kind: Deployment namespace: hermes name: hermes-chat-router @@ -624,7 +624,7 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-chat-router@sha256:882f9c3af7268618a19424765536f4cf6c37e8d380e515780baf4104e5b2931e + - registry.bstein.dev/bstein/hermes-chat-router@sha256:e3794fb8b9ee76d699893ba303d7080f54698c0a06064dc32293e99e988cabff - kind: Deployment namespace: hermes name: hermes-chat-sandbox-0 @@ -665,6 +665,55 @@ workloads: nodeSelector: {} images: - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-4 + labels: + ai.bstein.dev/tenant-ordinal: '4' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-5 + labels: + ai.bstein.dev/tenant-ordinal: '5' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-6 + labels: + ai.bstein.dev/tenant-ordinal: '6' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-chat-sandbox-7 + labels: + ai.bstein.dev/tenant-ordinal: '7' + app: hermes-chat-sandbox + serviceAccountName: null + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-local-image + labels: + app: hermes-local-image + serviceAccountName: hermes-gpu-runtime + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-local-image@sha256:d6257f49a60244fb1e30733848a187fa932acebfdb82a2560d4e760fa018b645 - kind: Deployment namespace: hermes name: hermes-model-gate @@ -676,13 +725,13 @@ workloads: - python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df - kind: Deployment namespace: hermes - name: hermes-ollama + name: hermes-oauth-sessions labels: - app: hermes-ollama + app: hermes-oauth-sessions serviceAccountName: null nodeSelector: {} images: - - ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d + - redis:7.4.1-alpine@sha256:c1e88455c85225310bbea54816e9c3f4b5295815e6dbf80c34d40afc6df28275 - kind: Deployment namespace: hermes name: hermes-stt @@ -693,6 +742,16 @@ workloads: kubernetes.io/hostname: titan-21 images: - registry.bstein.dev/bstein/hermes-jetson-stt@sha256:dfb0b0788dcf63747d8c761c6d3ea6a7459bcd8d0a5c92bcd4f6f434a760249f +- kind: Deployment + namespace: hermes + name: hermes-switchyard + labels: + app: hermes-switchyard + serviceAccountName: hermes-switchyard + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-switchyard-brokers@sha256:ee7e95e060ef8083da505162d7e9030daba15fdd828cc047bbcbe6aa409d2083 + - registry.bstein.dev/bstein/hermes-switchyard@sha256:3d952d528a4e4cb8afdf84f8995272f5d6107dc127b68cc20234b1da4aff43eb - kind: Deployment namespace: hermes name: hermes-tts @@ -700,18 +759,9 @@ workloads: app: hermes-tts serviceAccountName: null nodeSelector: - kubernetes.io/hostname: titan-20 + kubernetes.io/hostname: titan-21 images: - registry.bstein.dev/bstein/hermes-jetson-tts@sha256:5cb9e57faab46365bff606c559af57b9505892be2909aea1ac523568d478b2cc -- kind: Deployment - namespace: hermes - name: oauth2-proxy-hermes-agent - labels: - app: oauth2-proxy-hermes-agent - serviceAccountName: hermes-agent - nodeSelector: {} - images: - - quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 - kind: Deployment namespace: hermes name: oauth2-proxy-hermes-chat @@ -738,8 +788,8 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:10522c69676e250b5d0014d811ed9eb706d0b7c16f35cf66650f480e63ab3ab5 - - registry.bstein.dev/bstein/hermes-webui@sha256:c109e6faec1d6b86859a182bc845a2e35d64260459dda3892c0510db1dc7272d + - registry.bstein.dev/bstein/hermes-agent@sha256:3154e1df5f6941583f4a369fcc9581bbf9cb9d502676e91b557e90cc576923e1 + - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 - kind: Deployment namespace: jellyfin name: jellyfin @@ -978,7 +1028,7 @@ workloads: kubernetes.io/arch: amd64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-amd64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-292-amd64 - kind: DaemonSet namespace: maintenance name: metis-sentinel-arm64 @@ -989,7 +1039,7 @@ workloads: kubernetes.io/arch: arm64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-283-arm64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-291-arm64 - kind: DaemonSet namespace: maintenance name: node-image-sweeper @@ -1049,7 +1099,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/ariadne:0.1.0-448 + - registry.bstein.dev/bstein/ariadne:0.1.0-454 - kind: Deployment namespace: maintenance name: maintenance-vault-sync @@ -1068,7 +1118,7 @@ workloads: serviceAccountName: metis nodeSelector: {} images: - - registry.bstein.dev/bstein/metis:0.1.0-283-arm64 + - registry.bstein.dev/bstein/metis:0.1.0-291-arm64 - kind: Deployment namespace: maintenance name: oauth2-proxy-metis @@ -2013,20 +2063,6 @@ services: port: 9119 targetPort: dashboard protocol: TCP -- namespace: hermes - name: hermes-agent - type: ClusterIP - selector: - app: hermes-agent - ports: - - name: dashboard - port: 8787 - targetPort: dashboard - protocol: TCP - - name: terminal - port: 7681 - targetPort: terminal - protocol: TCP - namespace: hermes name: hermes-chat-router type: ClusterIP @@ -2061,6 +2097,56 @@ services: port: 8642 targetPort: api protocol: TCP +- namespace: hermes + name: hermes-claude-broker + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9006 + targetPort: claude-broker + protocol: TCP +- namespace: hermes + name: hermes-codex-broker + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9003 + targetPort: codex-broker + protocol: TCP +- namespace: hermes + name: hermes-gpu-handoff + type: ClusterIP + selector: + app: hermes-model-gate + ports: + - name: http + port: 11434 + targetPort: handoff + protocol: TCP +- namespace: hermes + name: hermes-image-broker + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9002 + targetPort: image-broker + protocol: TCP +- namespace: hermes + name: hermes-local-image + type: ClusterIP + selector: + app: hermes-local-image + ports: + - name: http + port: 9004 + targetPort: local-image + protocol: TCP - namespace: hermes name: hermes-model-gate type: ClusterIP @@ -2072,14 +2158,14 @@ services: targetPort: http protocol: TCP - namespace: hermes - name: hermes-ollama + name: hermes-oauth-sessions type: ClusterIP selector: - app: hermes-ollama + app: hermes-oauth-sessions ports: - - name: http - port: 11434 - targetPort: http + - name: redis + port: 6379 + targetPort: redis protocol: TCP - namespace: hermes name: hermes-stt @@ -2091,6 +2177,16 @@ services: port: 9000 targetPort: http protocol: TCP +- namespace: hermes + name: hermes-switchyard + type: ClusterIP + selector: + app: hermes-switchyard + ports: + - name: http + port: 9005 + targetPort: http + protocol: TCP - namespace: hermes name: hermes-triage type: ClusterIP @@ -2119,11 +2215,11 @@ services: name: oauth2-proxy-hermes-agent type: ClusterIP selector: - app: oauth2-proxy-hermes-agent + app: hermes-agent ports: - name: http port: 80 - targetPort: http + targetPort: auth-http protocol: TCP - namespace: hermes name: oauth2-proxy-hermes-chat @@ -2311,6 +2407,16 @@ services: port: 9401 targetPort: metrics protocol: TCP +- namespace: monitoring + name: nvidia-process-exporter-local + type: ClusterIP + selector: + app: nvidia-process-exporter + ports: + - name: metrics + port: 9401 + targetPort: metrics + protocol: TCP - namespace: monitoring name: platform-quality-gateway type: ClusterIP @@ -2588,12 +2694,23 @@ http_endpoints: namespace: hermes service: oauth2-proxy-hermes-agent port: http - workloads: + workloads: &id001 - kind: Deployment - name: oauth2-proxy-hermes-agent + name: hermes-agent via: kind: Ingress - name: hermes-sites + name: hermes-agent-dashboard + source: hermes +- host: agent.hermes.bstein.dev + path: /terminal + backend: + namespace: hermes + service: oauth2-proxy-hermes-agent + port: http + workloads: *id001 + via: + kind: Ingress + name: hermes-agent-terminal source: hermes - host: auth.bstein.dev path: / @@ -2640,7 +2757,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: &id001 + workloads: &id002 - kind: Deployment name: matrix-wellknown via: @@ -2653,7 +2770,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown-bstein-dev @@ -2716,7 +2833,7 @@ http_endpoints: namespace: cassandra service: cassandra-backend port: 80 - workloads: &id002 + workloads: &id003 - kind: Deployment name: cassandra-backend via: @@ -2729,7 +2846,7 @@ http_endpoints: namespace: cassandra service: cassandra-backend port: 80 - workloads: *id002 + workloads: *id003 via: kind: Ingress name: cassandra @@ -2740,7 +2857,7 @@ http_endpoints: namespace: cassandra service: cassandra-backend port: 80 - workloads: *id002 + workloads: *id003 via: kind: Ingress name: cassandra @@ -2842,7 +2959,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown @@ -2853,7 +2970,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown @@ -2875,7 +2992,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: &id004 + workloads: &id005 - kind: Deployment name: matrix-guest-register via: @@ -2888,7 +3005,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: &id003 + workloads: &id004 - kind: Deployment name: matrix-authentication-service via: @@ -2901,7 +3018,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -2912,7 +3029,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -2923,7 +3040,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: *id004 + workloads: *id005 via: kind: Ingress name: matrix-routing @@ -2971,7 +3088,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -2982,7 +3099,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown-matrix-live @@ -2993,7 +3110,7 @@ http_endpoints: namespace: comms service: matrix-wellknown port: 80 - workloads: *id001 + workloads: *id002 via: kind: Ingress name: matrix-wellknown-matrix-live @@ -3015,7 +3132,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: *id004 + workloads: *id005 via: kind: Ingress name: matrix-routing @@ -3026,7 +3143,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -3037,7 +3154,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -3048,7 +3165,7 @@ http_endpoints: namespace: comms service: matrix-authentication-service port: 8080 - workloads: *id003 + workloads: *id004 via: kind: Ingress name: matrix-routing @@ -3059,7 +3176,7 @@ http_endpoints: namespace: comms service: matrix-guest-register port: 8080 - workloads: *id004 + workloads: *id005 via: kind: Ingress name: matrix-routing @@ -3276,7 +3393,7 @@ http_endpoints: namespace: veles service: veles-backend port: 80 - workloads: &id005 + workloads: &id006 - kind: Deployment name: veles-backend via: @@ -3289,7 +3406,7 @@ http_endpoints: namespace: veles service: veles-backend port: 80 - workloads: *id005 + workloads: *id006 via: kind: Ingress name: veles @@ -3300,7 +3417,7 @@ http_endpoints: namespace: veles service: veles-backend port: 80 - workloads: *id005 + workloads: *id006 via: kind: Ingress name: veles diff --git a/services/comms/knowledge/catalog/runbooks.json b/services/comms/knowledge/catalog/runbooks.json index bef71b94..f0d7306f 100644 --- a/services/comms/knowledge/catalog/runbooks.json +++ b/services/comms/knowledge/catalog/runbooks.json @@ -26,7 +26,7 @@ "tags": [], "entrypoints": [], "source_paths": [], - "body": "Atlas Cluster Power Recovery (Graceful Shutdown/Startup)\n\nPurpose\n- Provide a safe operator flow for planned power events and cold-boot recovery.\n- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path.\n- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle.\n- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery.\n\nBootstrapping risk to remember\n- Flux source is Git over SSH to `scm.bstein.dev` (Gitea).\n- Gitea itself is a Flux-managed workload and depends on storage + database.\n- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images.\n- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall.\n- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.\n\nScript\n- `scripts/ops/cluster_power_recovery.sh`\n- `scripts/ops/cluster_power_console.sh`\n- Modes:\n - `prepare`\n - `shutdown`\n - `harbor-seed`\n - `startup`\n - `status`\n- Default is dry-run. Add `--execute` to actually perform actions.\n\nDry-run examples\n- Shutdown preview:\n - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`\n- Startup preview:\n - `scripts/ops/cluster_power_recovery.sh startup`\n- Harbor seed preview:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed`\n\nExecute examples\n- Prepare helper image on every node:\n - `scripts/ops/cluster_power_recovery.sh prepare --execute`\n- Seed Harbor runtime images onto `titan-05` from the control-host bundle:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`\n- Planned shutdown:\n - `scripts/ops/cluster_power_recovery.sh shutdown --execute`\n- Planned startup (canonical branch):\n - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`\n\nManual remote console examples\n- Canonical operator hosts:\n - `titan-db`\n - `tethys` (`titan-24`)\n- Both hosts now have:\n - `~/ananke-tools/cluster_power_recovery.sh`\n - `~/ananke-tools/cluster_power_console.sh`\n - `~/ananke-tools/bootstrap/recovery-config.env`\n - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt`\n - `~/ananke-tools/kubeconfig`\n - `~/ananke-cluster-power`\n - `~/bin/ananke-cluster-power`\n - `~/ananke-repo/{infrastructure,services,scripts}`\n- Both hosts also keep the Harbor bootstrap bundle at:\n - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n- Remote usage:\n - `ssh titan-db`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n - `ssh tethys`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n\nUseful options\n- `--shutdown-mode host-poweroff|cluster-only`\n- `--expected-flux-branch main`\n- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-branch main`\n- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only)\n- `--skip-local-bootstrap` (not recommended for cold-start recovery)\n- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred)\n- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node)\n- `--skip-helper-prewarm`\n- `--min-startup-battery 35`\n- `--ups-host pyrphoros@localhost`\n- `--require-ups-battery`\n- `--drain-timeout 180`\n- `--emergency-drain-timeout 45`\n- `--flux-ready-timeout 1200`\n- `--startup-checklist-timeout 900`\n- `--startup-stability-window 180`\n- `--startup-stability-timeout 900`\n- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state`\n- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n\nControlled drill checklist (recommended)\n- Operator host: use `titan-db` as canonical control host for the drill.\n- On-site coordination:\n - Have on-site operator ready before shutdown starts.\n - Confirm they will manually power cluster nodes back on after shutdown completes.\n - Confirm who will announce \"all nodes powered on\" to resume startup.\n- Preflight on `titan-db`:\n - `mkdir -p ~/ananke-logs`\n - `~/ananke-cluster-power status` and verify:\n - `ups_host=pyrphoros@localhost`\n - `ups_battery` is numeric\n - `flux_source_ready=True`\n- Warm helper image just before shutdown:\n - `~/ananke-cluster-power prepare --execute`\n- Run in a persistent shell and capture logs:\n - `tmux new -s ananke-drill`\n - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log`\n- Execute controlled shutdown with telemetry enforcement:\n - `~/ananke-cluster-power shutdown --execute --require-ups-battery`\n- After on-site power-on confirmation, execute startup:\n - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery`\n- Post-check:\n - `~/ananke-cluster-power status`\n - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures.\n\nOperational notes\n- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn.\n- Shutdown behavior is explicit:\n - `host-poweroff` schedules host poweroff after service stop.\n - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off.\n- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted.\n- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`).\n- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo.\n- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`.\n- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.\n- Harbor is reconciled after the first critical stateful services.\n- Harbor bootstrap is now designed around a control-host bundle:\n - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.\n - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.\n - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.\n- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.\n- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls.\n- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap.\n- Startup completion is strict now:\n - all non-optional Flux kustomizations must be `Ready=True`\n - external service checklist must pass (defaults include Gitea, Grafana, Harbor)\n - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`)\n - stability soak must pass with no crashloop/pull-failure churn\n- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile.\n- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster.\n- Dry-run mode no longer mutates outage recovery state.\n- `harbor-seed --execute` was validated by:\n - prewarming the helper image across all nodes\n - streaming the Harbor bootstrap bundle to `titan-05`\n - importing Harbor runtime images into host `containerd`\n - successfully running a Harbor-backed canary pod (`harbor-canary-ok`)\n- After bootstrap, Flux resources are resumed and reconciled.\n- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`." + "body": "Atlas Cluster Power Recovery (Graceful Shutdown/Startup)\n\nPurpose\n- Provide a safe operator flow for planned power events and cold-boot recovery.\n- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path.\n- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle.\n- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery.\n\nBootstrapping risk to remember\n- Flux source is Git over SSH to `scm.bstein.dev` (Gitea).\n- Gitea itself is a Flux-managed workload and depends on storage + database.\n- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images.\n- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall.\n- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.\n\nScript\n- `scripts/ops/cluster_power_recovery.sh`\n- `scripts/ops/cluster_power_console.sh`\n- Modes:\n - `prepare`\n - `shutdown`\n - `harbor-seed`\n - `startup`\n - `status`\n- Default is dry-run. Add `--execute` to actually perform actions.\n\nDry-run examples\n- Shutdown preview:\n - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`\n- Startup preview:\n - `scripts/ops/cluster_power_recovery.sh startup`\n- Harbor seed preview:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed`\n\nExecute examples\n- Prepare helper image on every node:\n - `scripts/ops/cluster_power_recovery.sh prepare --execute`\n- Seed Harbor runtime images onto `titan-05` from the control-host bundle:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`\n- Planned shutdown:\n - `scripts/ops/cluster_power_recovery.sh shutdown --execute`\n- Planned startup (canonical branch):\n - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`\n\nManual remote console examples\n- Canonical operator hosts:\n - `titan-db`\n - `tethys` (`titan-24`)\n- Both hosts now have:\n - `~/ananke-tools/cluster_power_recovery.sh`\n - `~/ananke-tools/cluster_power_console.sh`\n - `~/ananke-tools/bootstrap/recovery-config.env`\n - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt`\n - `~/ananke-tools/kubeconfig`\n - `~/ananke-cluster-power`\n - `~/bin/ananke-cluster-power`\n - `~/ananke-repo/{infrastructure,services,scripts}`\n- Both hosts also keep the Harbor bootstrap bundle at:\n - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n- Remote usage:\n - `ssh titan-db`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n - `ssh tethys`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n\nUseful options\n- `--shutdown-mode host-poweroff|cluster-only`\n- `--expected-flux-branch main`\n- `--expected-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git`\n- `--force-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git`\n- `--force-flux-branch main`\n- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only)\n- `--skip-local-bootstrap` (not recommended for cold-start recovery)\n- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred)\n- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node)\n- `--skip-helper-prewarm`\n- `--min-startup-battery 35`\n- `--ups-host pyrphoros@localhost`\n- `--require-ups-battery`\n- `--drain-timeout 180`\n- `--emergency-drain-timeout 45`\n- `--flux-ready-timeout 1200`\n- `--startup-checklist-timeout 900`\n- `--startup-stability-window 180`\n- `--startup-stability-timeout 900`\n- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state`\n- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n\nControlled drill checklist (recommended)\n- Operator host: use `titan-db` as canonical control host for the drill.\n- On-site coordination:\n - Have on-site operator ready before shutdown starts.\n - Confirm they will manually power cluster nodes back on after shutdown completes.\n - Confirm who will announce \"all nodes powered on\" to resume startup.\n- Preflight on `titan-db`:\n - `mkdir -p ~/ananke-logs`\n - `~/ananke-cluster-power status` and verify:\n - `ups_host=pyrphoros@localhost`\n - `ups_battery` is numeric\n - `flux_source_ready=True`\n- Warm helper image just before shutdown:\n - `~/ananke-cluster-power prepare --execute`\n- Run in a persistent shell and capture logs:\n - `tmux new -s ananke-drill`\n - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log`\n- Execute controlled shutdown with telemetry enforcement:\n - `~/ananke-cluster-power shutdown --execute --require-ups-battery`\n- After on-site power-on confirmation, execute startup:\n - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery`\n- Post-check:\n - `~/ananke-cluster-power status`\n - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures.\n\nOperational notes\n- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn.\n- Shutdown behavior is explicit:\n - `host-poweroff` schedules host poweroff after service stop.\n - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off.\n- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted.\n- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`).\n- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo.\n- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`.\n- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.\n- Harbor is reconciled after the first critical stateful services.\n- Harbor bootstrap is now designed around a control-host bundle:\n - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.\n - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.\n - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.\n- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.\n- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls.\n- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap.\n- Startup completion is strict now:\n - all non-optional Flux kustomizations must be `Ready=True`\n - external service checklist must pass (defaults include Gitea, Grafana, Harbor)\n - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`)\n - stability soak must pass with no crashloop/pull-failure churn\n- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile.\n- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster.\n- Dry-run mode no longer mutates outage recovery state.\n- `harbor-seed --execute` was validated by:\n - prewarming the helper image across all nodes\n - streaming the Harbor bootstrap bundle to `titan-05`\n - importing Harbor runtime images into host `containerd`\n - successfully running a Harbor-backed canary pod (`harbor-canary-ok`)\n- After bootstrap, Flux resources are resumed and reconciled.\n- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`." }, { "path": "runbooks/comms-verify.md", diff --git a/services/comms/knowledge/diagrams/atlas-http.mmd b/services/comms/knowledge/diagrams/atlas-http.mmd index 8c0eb353..11b55c8a 100644 --- a/services/comms/knowledge/diagrams/atlas-http.mmd +++ b/services/comms/knowledge/diagrams/atlas-http.mmd @@ -2,8 +2,8 @@ flowchart LR host_agent_hermes_bstein_dev["agent.hermes.bstein.dev"] svc_hermes_oauth2_proxy_hermes_agent["hermes/oauth2-proxy-hermes-agent (Service)"] host_agent_hermes_bstein_dev --> svc_hermes_oauth2_proxy_hermes_agent - wl_hermes_oauth2_proxy_hermes_agent["hermes/oauth2-proxy-hermes-agent (Deployment)"] - svc_hermes_oauth2_proxy_hermes_agent --> wl_hermes_oauth2_proxy_hermes_agent + wl_hermes_hermes_agent["hermes/hermes-agent (Deployment)"] + svc_hermes_oauth2_proxy_hermes_agent --> wl_hermes_hermes_agent host_auth_bstein_dev["auth.bstein.dev"] svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"] host_auth_bstein_dev --> svc_sso_oauth2_proxy @@ -248,7 +248,7 @@ flowchart LR end subgraph hermes[hermes] svc_hermes_oauth2_proxy_hermes_agent - wl_hermes_oauth2_proxy_hermes_agent + wl_hermes_hermes_agent svc_hermes_oauth2_proxy_hermes_chat wl_hermes_oauth2_proxy_hermes_chat svc_hermes_oauth2_proxy_hermes_triage diff --git a/services/comms/knowledge/hermes-automated-triage-24h-plan.md b/services/comms/knowledge/hermes-automated-triage-24h-plan.md index bf798a0b..7228e89e 100644 --- a/services/comms/knowledge/hermes-automated-triage-24h-plan.md +++ b/services/comms/knowledge/hermes-automated-triage-24h-plan.md @@ -1005,7 +1005,7 @@ The handoff must state plainly whether the full Hermes Agent or the Ollama fallb | Bounded source-patch proposal → branch → pull request | live (demo repo) | | Human-required metric, vmalert rule, Alertmanager | live | | Gitea issue filing for human-required incidents | live, proven on 4 real repos | -| Candidate-file selection for real service repos | live, PR opened on `bstein/ariadne` | +| Candidate-file selection for real service repos | live, PR opened on `atlas/ariadne` | | Narrowed alerting (repair failed, or unattended 6h) | live | ## What the fixture and code demos actually prove @@ -1055,7 +1055,7 @@ Code proposal: red build 22:34:59 -> pull request open 22:35:32 (33s). A one-line defect was seeded in `ariadne/utils/errors.py` (dropping the `errorMessage` key from the HTTP payload lookup) and pushed to master. Build 409 went red, and Ariadne opened -[bstein/ariadne PR #3](https://scm.bstein.dev/bstein/ariadne/pulls/3) from +[atlas/ariadne PR #3](https://scm.bstein.dev/atlas/ariadne/pulls/3) from branch `hermes-repair/409` with the correct one-line fix: ```diff diff --git a/services/comms/knowledge/runbooks/cluster-power-recovery.md b/services/comms/knowledge/runbooks/cluster-power-recovery.md index 86436d4c..ee80c101 100644 --- a/services/comms/knowledge/runbooks/cluster-power-recovery.md +++ b/services/comms/knowledge/runbooks/cluster-power-recovery.md @@ -72,8 +72,8 @@ Manual remote console examples Useful options - `--shutdown-mode host-poweroff|cluster-only` - `--expected-flux-branch main` -- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` -- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` +- `--expected-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git` +- `--force-flux-url ssh://git@scm.bstein.dev:2242/atlas/titan-iac.git` - `--force-flux-branch main` - `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only) - `--skip-local-bootstrap` (not recommended for cold-start recovery) diff --git a/services/gitea/atlas-identity-bootstrap-job.yaml b/services/gitea/atlas-identity-bootstrap-job.yaml index 3402ba03..c7e2071f 100644 --- a/services/gitea/atlas-identity-bootstrap-job.yaml +++ b/services/gitea/atlas-identity-bootstrap-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: gitea-atlas-identity-bootstrap-2 + name: gitea-atlas-identity-bootstrap-3 namespace: gitea labels: app.kubernetes.io/name: gitea @@ -54,6 +54,10 @@ spec: value: http://gitea.gitea.svc.cluster.local:3000/api/v1 - name: GITEA_PUBLIC_URL value: https://scm.bstein.dev + - name: GITEA_SOURCE_OWNER + value: bstein + - name: GITEA_MANAGED_REPOSITORIES + value: hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac - name: VAULT_ADDR value: http://vault.vault.svc.cluster.local:8200 - name: VAULT_ROLE diff --git a/services/gitea/scripts/gitea_atlas_identity_ensure.sh b/services/gitea/scripts/gitea_atlas_identity_ensure.sh index cbfe61ca..c8c3288d 100644 --- a/services/gitea/scripts/gitea_atlas_identity_ensure.sh +++ b/services/gitea/scripts/gitea_atlas_identity_ensure.sh @@ -15,6 +15,8 @@ hermes_user=hermes-automation hermes_email=hermes-automation@bstein.dev organization=atlas owner_user=bstein +source_owner="${GITEA_SOURCE_OWNER:-bstein}" +managed_repositories="${GITEA_MANAGED_REPOSITORIES:-hermes-code-demo cassandra soteria pegasus metis ananke ariadne typhon titan-iac}" contributor_team=Contributors contributor_units='["repo.actions","repo.packages","repo.code","repo.issues","repo.wiki","repo.pulls","repo.releases","repo.projects"]' @@ -157,6 +159,75 @@ vault_write() { expect_status "${status}" 200 "write Vault path ${path}" } +transfer_managed_repositories() { + transfer_tmp_dir=$(mktemp -d) + ready_file="${transfer_tmp_dir}/ready" + source_file="${transfer_tmp_dir}/source" + target_file="${transfer_tmp_dir}/target" + response_file="${transfer_tmp_dir}/response" + : >"${ready_file}" + + cleanup_transfer_tmp() { + rm -rf "${transfer_tmp_dir}" + } + trap cleanup_transfer_tmp EXIT + trap 'exit 1' HUP INT TERM + + # Validate the complete set before changing ownership so a typo or collision + # cannot leave the managed repositories split between two owners. + for repository in ${managed_repositories}; do + source_status=$(api_request GET "/repos/${source_owner}/${repository}" '' "${source_file}") + target_status=$(api_request GET "/repos/${organization}/${repository}" '' "${target_file}") + + case "${source_status}:${target_status}" in + 200:404) + printf '%s\n' "${repository}" >>"${ready_file}" + ;; + 301:200|302:200|307:200|308:200|404:200) + # Gitea may redirect the old API path after a native transfer. + ;; + 200:200) + die "repository ${repository} exists under both ${source_owner} and ${organization}" + ;; + 404:404) + die "repository ${repository} does not exist under ${source_owner} or ${organization}" + ;; + *) + die "repository ${repository} preflight returned source HTTP ${source_status}, target HTTP ${target_status}" + ;; + esac + done + + while IFS= read -r repository; do + [ -n "${repository}" ] || continue + status=$(api_request POST "/repos/${source_owner}/${repository}/transfer" "{\"new_owner\":\"${organization}\"}" "${response_file}") + case "${status}" in + 201|202) ;; + *) + die "transfer repository ${repository} returned HTTP ${status}" + ;; + esac + + attempt=0 + transferred=false + while [ "${attempt}" -lt 30 ]; do + target_status=$(api_request GET "/repos/${organization}/${repository}" '' "${response_file}") + if [ "${target_status}" = 200 ]; then + transferred=true + break + fi + attempt=$((attempt + 1)) + sleep 2 + done + if [ "${transferred}" != true ]; then + die "repository ${repository} did not appear under ${organization} after transfer" + fi + done <"${ready_file}" + + cleanup_transfer_tmp + trap - EXIT HUP INT TERM +} + wait_for_gitea ensure_user "${reconciler_user}" "${reconciler_email}" true ensure_user "${hermes_user}" "${hermes_email}" false @@ -194,8 +265,10 @@ status=$(api_request PUT "/teams/${contributors_id}/members/${hermes_user}" '' " rm -f "${response_file}" expect_status "${status}" 204 "add Hermes to Atlas contributors" +transfer_managed_repositories + vault_login vault_write gitea/atlas-reconciler "{\"data\":{\"username\":\"${reconciler_user}\",\"token\":\"${reconciler_token}\",\"base_url\":\"${public_url}\"}}" vault_write hermes/developer-gitea "{\"data\":{\"username\":\"${hermes_user}\",\"email\":\"${hermes_email}\",\"token\":\"${hermes_token}\",\"base_url\":\"${public_url}\"}}" -echo "Atlas organization and Hermes contributor identity are ready" +echo "Atlas organization, managed repositories, and Hermes contributor identity are ready" diff --git a/services/hermes/scripts/hermes_coordinator.py b/services/hermes/scripts/hermes_coordinator.py index c357c9a3..90a3bcb4 100644 --- a/services/hermes/scripts/hermes_coordinator.py +++ b/services/hermes/scripts/hermes_coordinator.py @@ -24,7 +24,7 @@ from hermes_model_routing import ( CASSANDRA_BASE_PATH = Path("/opt/data/workspace/projects/cassandra") -CASSANDRA_REMOTE = "https://scm.bstein.dev/bstein/cassandra.git" +CASSANDRA_REMOTE = "https://scm.bstein.dev/atlas/cassandra.git" def cassandra_workspace() -> Path: @@ -143,7 +143,63 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: token = env_values.get("GITEA_TOKEN", "").strip() if shutil.which("git") is None: return "git-unavailable" - if (CASSANDRA_BASE_PATH / ".git").is_dir(): + child_env = os.environ.copy() + child_env.update(env_values) + if (CASSANDRA_BASE_PATH / ".git").exists(): + try: + current_remote = subprocess.run( + [ + "git", + "-C", + str(CASSANDRA_BASE_PATH), + "remote", + "get-url", + "origin", + ], + env=child_env, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + timeout=30, + check=False, + ) + if current_remote.returncode == 0: + repair_command = [ + "git", + "-C", + str(CASSANDRA_BASE_PATH), + "remote", + "set-url", + "origin", + CASSANDRA_REMOTE, + ] + remote_needs_repair = current_remote.stdout.strip() != CASSANDRA_REMOTE + else: + repair_command = [ + "git", + "-C", + str(CASSANDRA_BASE_PATH), + "remote", + "add", + "origin", + CASSANDRA_REMOTE, + ] + remote_needs_repair = True + if remote_needs_repair: + repaired = subprocess.run( + repair_command, + env=child_env, + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=30, + check=False, + ) + if repaired.returncode != 0: + return f"remote-repair-failed-{repaired.returncode}" + except (OSError, subprocess.TimeoutExpired): + return "remote-repair-failed" if not token: return "ready; fetch skipped until Gitea token is configured" command = [ @@ -168,8 +224,6 @@ def sync_cassandra_repo(env_values: dict[str, str]) -> str: CASSANDRA_REMOTE, str(CASSANDRA_BASE_PATH), ] - child_env = os.environ.copy() - child_env.update(env_values) try: completed = subprocess.run( command, diff --git a/services/jenkins/configmap-jcasc.yaml b/services/jenkins/configmap-jcasc.yaml index 43f4cc7d..e9aa57b9 100644 --- a/services/jenkins/configmap-jcasc.yaml +++ b/services/jenkins/configmap-jcasc.yaml @@ -132,7 +132,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/ariadne.git') + url('https://scm.bstein.dev/atlas/ariadne.git') credentials('gitea-pat') } branches('*/master') @@ -161,7 +161,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/metis.git') + url('https://scm.bstein.dev/atlas/metis.git') credentials('gitea-pat') } branches('*/master') @@ -190,7 +190,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/ananke.git') + url('https://scm.bstein.dev/atlas/ananke.git') credentials('gitea-pat') } branches('*/main') @@ -274,7 +274,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/pegasus.git') + url('https://scm.bstein.dev/atlas/pegasus.git') credentials('gitea-pat') } branches('*/main') @@ -332,7 +332,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/soteria.git') + url('https://scm.bstein.dev/atlas/soteria.git') credentials('gitea-pat') } branches('*/main') @@ -361,7 +361,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/titan-iac.git') + url('https://scm.bstein.dev/atlas/titan-iac.git') credentials('gitea-pat') } branches('*/main') @@ -390,7 +390,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/titan-iac.git') + url('https://scm.bstein.dev/atlas/titan-iac.git') credentials('gitea-pat') } branches('*/main') @@ -419,7 +419,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/typhon.git') + url('https://scm.bstein.dev/atlas/typhon.git') credentials('gitea-pat') } branches('*/main') @@ -447,7 +447,7 @@ data: scm { git { remote { - url('https://scm.bstein.dev/bstein/cassandra.git') + url('https://scm.bstein.dev/atlas/cassandra.git') credentials('gitea-pat') } branches('*/main') @@ -658,7 +658,7 @@ data: source { git { id('titan-iac-quality-gate') - remote('https://scm.bstein.dev/bstein/titan-iac.git') + remote('https://scm.bstein.dev/atlas/titan-iac.git') credentialsId('gitea-pat') } } diff --git a/services/maintenance/apps/ariadne-deployment.yaml b/services/maintenance/apps/ariadne-deployment.yaml index b2cc8ca4..3956eb30 100644 --- a/services/maintenance/apps/ariadne-deployment.yaml +++ b/services/maintenance/apps/ariadne-deployment.yaml @@ -525,7 +525,7 @@ spec: - name: ARIADNE_HERMES_ISSUES_ENABLED value: "true" - name: ARIADNE_HERMES_ISSUE_REPOS - value: metis=bstein/metis,lesavka=bstein/lesavka,soteria=bstein/soteria,bstein-dev-home=bstein/bstein-dev-home,ariadne=bstein/ariadne,ananke=bstein/ananke,pegasus=bstein/pegasus,atlasbot=bstein/atlasbot + value: metis=atlas/metis,lesavka=bstein/lesavka,soteria=atlas/soteria,bstein-dev-home=bstein/bstein-dev-home,ariadne=atlas/ariadne,ananke=atlas/ananke,pegasus=atlas/pegasus,atlasbot=bstein/atlasbot - name: ARIADNE_HERMES_ISSUE_DEDUPE_SCOPE value: classification - name: ARIADNE_HERMES_ISSUE_MAX_PER_TICK @@ -598,7 +598,7 @@ spec: - name: ARIADNE_HERMES_SONAR_MAX_EFFORT_MINUTES value: "20" - name: ARIADNE_HERMES_CODE_REPOS - value: metis=bstein/metis,lesavka=bstein/lesavka,soteria=bstein/soteria,bstein-dev-home=bstein/bstein-dev-home,ariadne=bstein/ariadne,ananke=bstein/ananke,pegasus=bstein/pegasus,atlasbot=bstein/atlasbot + value: metis=atlas/metis,lesavka=bstein/lesavka,soteria=atlas/soteria,bstein-dev-home=bstein/bstein-dev-home,ariadne=atlas/ariadne,ananke=atlas/ananke,pegasus=atlas/pegasus,atlasbot=bstein/atlasbot - name: ARIADNE_HERMES_CODE_BASE_BRANCHES value: metis=master,lesavka=master,soteria=main,bstein-dev-home=master,ariadne=master,ananke=main,pegasus=main,atlasbot=main - name: ARIADNE_HERMES_CODE_PREFIXES diff --git a/testing/tests/test_hermes_chat_quality.py b/testing/tests/test_hermes_chat_quality.py index 95a6bf87..a6272a8f 100644 --- a/testing/tests/test_hermes_chat_quality.py +++ b/testing/tests/test_hermes_chat_quality.py @@ -76,7 +76,7 @@ def test_chat_config_enables_real_research_compute_and_delegation(): def test_sandbox_shares_only_the_tenant_workspace_without_credentials(): sandbox_docs = _documents(HERMES / "chat-sandbox.yaml") deployments = [doc for doc in sandbox_docs if doc["kind"] == "Deployment"] - assert len(deployments) == 4 + assert len(deployments) == 8 for ordinal, deployment in enumerate(deployments): pod_spec = deployment["spec"]["template"]["spec"] container = pod_spec["containers"][0] @@ -451,7 +451,9 @@ def test_chat_image_generation_uses_private_owner_broker(): } vault_policy = (VAULT / "scripts" / "vault_k8s_auth_configure.sh").read_text() assert ( - '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram"' + '"hermes/agent-oidc hermes/agent-tokens hermes/chat-telegram ' + 'hermes/developer-keycloak hermes/developer-gitea ' + 'hermes/developer-harbor hermes/developer-jenkins"' in vault_policy ) diff --git a/testing/tests/test_hermes_coordinator.py b/testing/tests/test_hermes_coordinator.py index e23f8365..eeef6753 100644 --- a/testing/tests/test_hermes_coordinator.py +++ b/testing/tests/test_hermes_coordinator.py @@ -9,12 +9,11 @@ import sys import tomllib from pathlib import Path +import pytest import yaml -SCRIPT = ( - Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" -) +SCRIPT = Path(__file__).parents[2] / "services/hermes/scripts/hermes_coordinator.py" sys.path.insert(0, str(SCRIPT.parent)) routing = importlib.import_module("hermes_model_routing") catalog_resolver = importlib.import_module("routing_catalog") @@ -85,15 +84,33 @@ def test_dynamic_catalog_resolves_new_models_and_preserves_last_known_good(): ) current = routing.build_routing_catalog(codex, claude) - assert catalog_resolver.resolve_model("codex", "auto", "high", current) == "gpt-5.7-sol" - assert catalog_resolver.resolve_model("codex", "terra", "medium", current) == "gpt-5.7-terra" - assert catalog_resolver.resolve_model("claude", "auto", "xhigh", current) == "claude-opus-6" - assert catalog_resolver.resolve_model("claude", "sonnet", "high", current) == "claude-sonnet-6" + assert ( + catalog_resolver.resolve_model("codex", "auto", "high", current) + == "gpt-5.7-sol" + ) + assert ( + catalog_resolver.resolve_model("codex", "terra", "medium", current) + == "gpt-5.7-terra" + ) + assert ( + catalog_resolver.resolve_model("claude", "auto", "xhigh", current) + == "claude-opus-6" + ) + assert ( + catalog_resolver.resolve_model("claude", "sonnet", "high", current) + == "claude-sonnet-6" + ) unavailable = routing.Catalog("openai-codex", [], False, True, "degraded") preserved = routing.build_routing_catalog(unavailable, unavailable, current) - assert preserved["providers"]["codex"]["resolved"] == current["providers"]["codex"]["resolved"] - assert preserved["providers"]["claude"]["tiers"] == current["providers"]["claude"]["tiers"] + assert ( + preserved["providers"]["codex"]["resolved"] + == current["providers"]["codex"]["resolved"] + ) + assert ( + preserved["providers"]["claude"]["tiers"] + == current["providers"]["claude"]["tiers"] + ) def test_live_catalog_never_routes_to_a_removed_model_tier(): @@ -312,6 +329,7 @@ def test_corrupt_cassandra_board_is_preserved_without_blocking_refresh( tmp_path: Path, monkeypatch ): """A damaged project board degrades Kanban instead of stopping all services.""" + class FakeKanbanDbCorruptError(Exception): def __init__(self): super().__init__("integrity check failed") @@ -341,6 +359,7 @@ def test_unrelated_cassandra_bootstrap_failure_remains_fatal( tmp_path: Path, monkeypatch ): """Only the known fail-closed corruption state may be degraded.""" + class FakeKanbanDbCorruptError(Exception): pass @@ -375,8 +394,63 @@ def test_cassandra_workspace_uses_valid_configured_worktree( assert coordinator.cassandra_workspace() == coordinator.CASSANDRA_BASE_PATH +@pytest.mark.parametrize( + ("current_remote", "repair_action"), + [ + ("https://scm.bstein.dev/bstein/cassandra.git", "set-url"), + (None, "add"), + (coordinator.CASSANDRA_REMOTE, None), + ], + ids=["old-origin", "missing-origin", "canonical-origin"], +) +def test_cassandra_sync_repairs_existing_worktree( + tmp_path: Path, monkeypatch, current_remote: str | None, repair_action: str | None +): + """Existing worktrees use the canonical Atlas origin before fetching.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + if command[-2:] == ["get-url", "origin"]: + return subprocess.CompletedProcess( + command, 0 if current_remote else 2, stdout=f"{current_remote or ''}\n" + ) + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + assert coordinator.sync_cassandra_repo({"GITEA_TOKEN": "configured"}) == "ready" + repair_commands = [command for command in commands if repair_action in command] + assert bool(repair_commands) is bool(repair_action) + assert commands[-1][-4:] == ["fetch", "--quiet", "--prune", "origin"] + + +def test_cassandra_sync_repairs_origin_without_token(tmp_path: Path, monkeypatch): + """Missing credentials skip only the fetch, not the local origin repair.""" + workspace = tmp_path / "cassandra" + (workspace / ".git").mkdir(parents=True) + monkeypatch.setattr(coordinator, "CASSANDRA_BASE_PATH", workspace) + monkeypatch.setattr(coordinator.shutil, "which", lambda _name: "/usr/bin/git") + commands: list[list[str]] = [] + + def run(command, **_kwargs): + commands.append(command) + stdout = "old-origin\n" if command[-2:] == ["get-url", "origin"] else None + return subprocess.CompletedProcess(command, 0, stdout=stdout) + + monkeypatch.setattr(coordinator.subprocess, "run", run) + state = coordinator.sync_cassandra_repo({}) + + assert state == "ready; fetch skipped until Gitea token is configured" + assert [command[-3] for command in commands] == ["remote", "set-url"] + + def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): """A project switch moves queued work without relocating active evidence.""" + class Task: def __init__(self, task_id: str, status: str): self.id = task_id @@ -392,7 +466,11 @@ def test_migrate_open_cassandra_tasks_preserves_running_and_done_tasks(): return None class FakeKanban: - tasks = [Task("queued", "todo"), Task("active", "running"), Task("done", "done")] + tasks = [ + Task("queued", "todo"), + Task("active", "running"), + Task("done", "done"), + ] moved: list[tuple[str, str]] = [] @staticmethod