From 26705c1f66249ac399972bf2c9a9933a8201813d Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Fri, 21 Aug 2026 09:58:42 +0000 Subject: [PATCH 1/2] chore(knowledge): regenerate stale Atlas catalogs `make knowledge` output on main no longer matched the manifests it is rendered from: Flux kustomizations added since the last regeneration (hermes-scm-broker, hermes-scm-broker-code, hermes-scm-namespace, hermes-observer-rbac, hermes-observer-bindings, vault-hermes-jenkins-token-seed) and the Atlas AI Operations dashboard panels were missing from the committed artifacts. This is a pure `make knowledge` run against unmodified main, separated into its own commit so the hostname rename that follows reviews as a hostname rename and nothing else. No hand edits. --- knowledge/catalog/atlas-summary.json | 4 +- knowledge/catalog/atlas.json | 234 +++++++++- knowledge/catalog/atlas.yaml | 163 ++++++- knowledge/catalog/metrics.json | 436 ++++++++++++++++++ .../knowledge/catalog/atlas-summary.json | 4 +- services/comms/knowledge/catalog/atlas.json | 234 +++++++++- services/comms/knowledge/catalog/atlas.yaml | 163 ++++++- services/comms/knowledge/catalog/metrics.json | 436 ++++++++++++++++++ 8 files changed, 1618 insertions(+), 56 deletions(-) diff --git a/knowledge/catalog/atlas-summary.json b/knowledge/catalog/atlas-summary.json index 9acf7417..cb8ec44b 100644 --- a/knowledge/catalog/atlas-summary.json +++ b/knowledge/catalog/atlas-summary.json @@ -2,7 +2,7 @@ "counts": { "helmrelease_host_hints": 23, "http_endpoints": 61, - "services": 91, - "workloads": 125 + "services": 98, + "workloads": 130 } } diff --git a/knowledge/catalog/atlas.json b/knowledge/catalog/atlas.json index d7247bfd..8265e842 100644 --- a/knowledge/catalog/atlas.json +++ b/knowledge/catalog/atlas.json @@ -106,6 +106,31 @@ "path": "services/hermes-chat", "targetNamespace": "hermes-chat" }, + { + "name": "hermes-observer-bindings", + "path": "services/hermes-observer-bindings", + "targetNamespace": null + }, + { + "name": "hermes-observer-rbac", + "path": "services/hermes-observer-rbac", + "targetNamespace": null + }, + { + "name": "hermes-scm-broker", + "path": "services/hermes-scm-broker", + "targetNamespace": "hermes-scm" + }, + { + "name": "hermes-scm-broker-code", + "path": "services/hermes/scm-common", + "targetNamespace": "hermes-scm" + }, + { + "name": "hermes-scm-namespace", + "path": "services/hermes-scm-namespace", + "targetNamespace": null + }, { "name": "hermes-triage-demo", "path": "services/hermes-triage-demo", @@ -251,6 +276,11 @@ "path": "infrastructure/vault-csi", "targetNamespace": "kube-system" }, + { + "name": "vault-hermes-jenkins-token-seed", + "path": "services/vault-hermes-jenkins-token-seed", + "targetNamespace": "vault" + }, { "name": "vault-injector", "path": "infrastructure/vault-injector", @@ -304,7 +334,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464" + "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-478" ] }, { @@ -320,7 +350,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464" + "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-478" ] }, { @@ -902,7 +932,7 @@ "serviceAccountName": "hermes-node-ssh-access", "nodeSelector": {}, "images": [ - "busybox:1.37" + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" ] }, { @@ -915,8 +945,8 @@ "serviceAccountName": "hermes-triage", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8", - "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37", + "registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e" ] }, { @@ -930,7 +960,7 @@ "nodeSelector": {}, "images": [ "quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561", - "registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8" + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" ] }, { @@ -943,7 +973,7 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-chat-router@sha256:a4010fdc5b6dce2696e6fec06eb38a3992817faa01ff3472122c870038b22bd2" + "registry.bstein.dev/bstein/hermes-chat-router@sha256:6744cb7b87c6050f1b97c0675cd280b8295b3b5826ba1d6d92e37aee6fd0b8c4" ] }, { @@ -1058,6 +1088,48 @@ "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-execution-mediator-0", + "labels": { + "app": "hermes-execution-mediator", + "pool-ordinal": "0" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-execution-mediator-1", + "labels": { + "app": "hermes-execution-mediator", + "pool-ordinal": "1" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-execution-mediator-2", + "labels": { + "app": "hermes-execution-mediator", + "pool-ordinal": "2" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, { "kind": "Deployment", "namespace": "hermes", @@ -1177,8 +1249,36 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8", - "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37", + "registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf" + ] + }, + { + "kind": "StatefulSet", + "namespace": "hermes", + "name": "hermes-execution-worker", + "labels": { + "app": "hermes-execution-worker", + "app.kubernetes.io/name": "hermes-execution-worker", + "app.kubernetes.io/part-of": "hermes" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes-scm", + "name": "hermes-scm-broker", + "labels": { + "app": "hermes-scm-broker" + }, + "serviceAccountName": "hermes-scm-broker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7" ] }, { @@ -1528,7 +1628,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-amd64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-amd64" ] }, { @@ -1544,7 +1644,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-arm64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-arm64" ] }, { @@ -1633,7 +1733,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/ariadne:0.1.0-454" + "registry.bstein.dev/bstein/ariadne:0.1.0-464" ] }, { @@ -1661,7 +1761,7 @@ "serviceAccountName": "metis", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/metis:0.1.0-293-arm64" + "registry.bstein.dev/bstein/metis:0.1.0-304-arm64" ] }, { @@ -3179,6 +3279,22 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-cli-lane-metrics", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "lane-metrics", + "port": 9011, + "targetPort": "lane-metrics", + "protocol": "TCP" + } + ] + }, { "namespace": "hermes", "name": "hermes-codex-broker", @@ -3195,6 +3311,82 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-execution-mediator-0", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-mediator", + "pool-ordinal": "0" + }, + "ports": [ + { + "name": "mediator", + "port": 9009, + "targetPort": "mediator", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-mediator-1", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-mediator", + "pool-ordinal": "1" + }, + "ports": [ + { + "name": "mediator", + "port": 9009, + "targetPort": "mediator", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-mediator-2", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-mediator", + "pool-ordinal": "2" + }, + "ports": [ + { + "name": "mediator", + "port": 9009, + "targetPort": "mediator", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-pool", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9007, + "targetPort": "execution-pool", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-worker", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-worker" + }, + "ports": [] + }, { "namespace": "hermes", "name": "hermes-gpu-handoff", @@ -3393,6 +3585,22 @@ } ] }, + { + "namespace": "hermes-scm", + "name": "hermes-scm-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-scm-broker" + }, + "ports": [ + { + "name": "http", + "port": 9081, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "jellyfin", "name": "jellyfin", diff --git a/knowledge/catalog/atlas.yaml b/knowledge/catalog/atlas.yaml index 8ae1b49b..544add7b 100644 --- a/knowledge/catalog/atlas.yaml +++ b/knowledge/catalog/atlas.yaml @@ -65,6 +65,21 @@ sources: - name: hermes-chat path: services/hermes-chat targetNamespace: hermes-chat +- name: hermes-observer-bindings + path: services/hermes-observer-bindings + targetNamespace: null +- name: hermes-observer-rbac + path: services/hermes-observer-rbac + targetNamespace: null +- name: hermes-scm-broker + path: services/hermes-scm-broker + targetNamespace: hermes-scm +- name: hermes-scm-broker-code + path: services/hermes/scm-common + targetNamespace: hermes-scm +- name: hermes-scm-namespace + path: services/hermes-scm-namespace + targetNamespace: null - name: hermes-triage-demo path: services/hermes-triage-demo targetNamespace: null @@ -152,6 +167,9 @@ sources: - name: vault-csi path: infrastructure/vault-csi targetNamespace: kube-system +- name: vault-hermes-jenkins-token-seed + path: services/vault-hermes-jenkins-token-seed + targetNamespace: vault - name: vault-injector path: infrastructure/vault-injector targetNamespace: vault @@ -187,7 +205,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464 + - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-478 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-frontend @@ -198,7 +216,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464 + - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-478 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-vault-sync @@ -604,7 +622,7 @@ workloads: serviceAccountName: hermes-node-ssh-access nodeSelector: {} images: - - busybox:1.37 + - python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df - kind: Deployment namespace: hermes name: hermes @@ -613,8 +631,8 @@ workloads: serviceAccountName: hermes-triage nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8 - - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 + - registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e - kind: Deployment namespace: hermes name: hermes-agent @@ -624,7 +642,7 @@ workloads: nodeSelector: {} images: - quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 - - registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8 + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 - kind: Deployment namespace: hermes name: hermes-chat-router @@ -633,7 +651,7 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-chat-router@sha256:a4010fdc5b6dce2696e6fec06eb38a3992817faa01ff3472122c870038b22bd2 + - registry.bstein.dev/bstein/hermes-chat-router@sha256:6744cb7b87c6050f1b97c0675cd280b8295b3b5826ba1d6d92e37aee6fd0b8c4 - kind: Deployment namespace: hermes name: hermes-chat-sandbox-0 @@ -714,6 +732,36 @@ workloads: nodeSelector: {} images: - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-execution-mediator-0 + labels: + app: hermes-execution-mediator + pool-ordinal: '0' + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 +- kind: Deployment + namespace: hermes + name: hermes-execution-mediator-1 + labels: + app: hermes-execution-mediator + pool-ordinal: '1' + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 +- kind: Deployment + namespace: hermes + name: hermes-execution-mediator-2 + labels: + app: hermes-execution-mediator + pool-ordinal: '2' + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 - kind: Deployment namespace: hermes name: hermes-local-image @@ -797,8 +845,28 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8 - - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 + - registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf +- kind: StatefulSet + namespace: hermes + name: hermes-execution-worker + labels: + app: hermes-execution-worker + app.kubernetes.io/name: hermes-execution-worker + app.kubernetes.io/part-of: hermes + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 +- kind: Deployment + namespace: hermes-scm + name: hermes-scm-broker + labels: + app: hermes-scm-broker + serviceAccountName: hermes-scm-broker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 - kind: Deployment namespace: jellyfin name: jellyfin @@ -1037,7 +1105,7 @@ workloads: kubernetes.io/arch: amd64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-amd64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-amd64 - kind: DaemonSet namespace: maintenance name: metis-sentinel-arm64 @@ -1048,7 +1116,7 @@ workloads: kubernetes.io/arch: arm64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-arm64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-arm64 - kind: DaemonSet namespace: maintenance name: node-image-sweeper @@ -1108,7 +1176,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/ariadne:0.1.0-454 + - registry.bstein.dev/bstein/ariadne:0.1.0-464 - kind: Deployment namespace: maintenance name: maintenance-vault-sync @@ -1127,7 +1195,7 @@ workloads: serviceAccountName: metis nodeSelector: {} images: - - registry.bstein.dev/bstein/metis:0.1.0-293-arm64 + - registry.bstein.dev/bstein/metis:0.1.0-304-arm64 - kind: Deployment namespace: maintenance name: oauth2-proxy-metis @@ -2120,6 +2188,16 @@ services: port: 9006 targetPort: claude-broker protocol: TCP +- namespace: hermes + name: hermes-cli-lane-metrics + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: lane-metrics + port: 9011 + targetPort: lane-metrics + protocol: TCP - namespace: hermes name: hermes-codex-broker type: ClusterIP @@ -2130,6 +2208,55 @@ services: port: 9003 targetPort: codex-broker protocol: TCP +- namespace: hermes + name: hermes-execution-mediator-0 + type: ClusterIP + selector: + app: hermes-execution-mediator + pool-ordinal: '0' + ports: + - name: mediator + port: 9009 + targetPort: mediator + protocol: TCP +- namespace: hermes + name: hermes-execution-mediator-1 + type: ClusterIP + selector: + app: hermes-execution-mediator + pool-ordinal: '1' + ports: + - name: mediator + port: 9009 + targetPort: mediator + protocol: TCP +- namespace: hermes + name: hermes-execution-mediator-2 + type: ClusterIP + selector: + app: hermes-execution-mediator + pool-ordinal: '2' + ports: + - name: mediator + port: 9009 + targetPort: mediator + protocol: TCP +- namespace: hermes + name: hermes-execution-pool + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9007 + targetPort: execution-pool + protocol: TCP +- namespace: hermes + name: hermes-execution-worker + type: ClusterIP + selector: + app: hermes-execution-worker + ports: [] - namespace: hermes name: hermes-gpu-handoff type: ClusterIP @@ -2254,6 +2381,16 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: hermes-scm + name: hermes-scm-broker + type: ClusterIP + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http + protocol: TCP - namespace: jellyfin name: jellyfin type: ClusterIP diff --git a/knowledge/catalog/metrics.json b/knowledge/catalog/metrics.json index f26012fb..589033f1 100644 --- a/knowledge/catalog/metrics.json +++ b/knowledge/catalog/metrics.json @@ -1,4 +1,440 @@ [ + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Weekly Remaining", + "panel_id": 1, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Spark Weekly Remaining", + "panel_id": 2, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 5h Remaining", + "panel_id": 3, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 7d Remaining", + "panel_id": 4, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Quota Collectors Healthy", + "panel_id": 5, + "panel_type": "stat", + "description": "Successful latest quota fetches. Providers are polled independently every five minutes.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(atlas_ai_quota_fetch_success) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Oldest Quota Sample", + "panel_id": 6, + "panel_type": "stat", + "description": "Age of the stalest successful provider quota snapshot.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Weekly Reset In", + "panel_id": 7, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 5h Reset In", + "panel_id": 8, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 7d Reset In", + "panel_id": 9, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Tokens (Latest Day)", + "panel_id": 10, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Tokens (7d)", + "panel_id": 11, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Switchyard Requests", + "panel_id": 12, + "panel_type": "stat", + "description": "Hosted model requests observed by Switchyard in the selected dashboard range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(switchyard_requests_total[$__range])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Model Selection Rate", + "panel_id": 13, + "panel_type": "timeseries", + "description": "AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum by (selected_model) (rate(switchyard_decisions_total[5m]))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Provider Selections (Range)", + "panel_id": 14, + "panel_type": "bargauge", + "description": "Switchyard selections grouped by provider over the selected dashboard range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~\"(route|worker)/(codex|claude|local)/.*\"}[$__range]), \"provider\", \"$2\", \"selected_model\", \"^(route|worker)/(codex|claude|local)/.*\")))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Token Throughput", + "panel_id": 15, + "panel_type": "timeseries", + "description": "Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(rate(switchyard_prompt_tokens_total[5m]))", + "sum(rate(switchyard_cached_tokens_total[5m]))", + "sum(rate(switchyard_cache_creation_tokens_total[5m]))", + "sum(rate(switchyard_reasoning_tokens_total[5m]))", + "sum(rate(switchyard_completion_tokens_total[5m]))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Model Call p95 Latency", + "panel_id": 16, + "panel_type": "timeseries", + "description": "95th percentile upstream latency for each selected model route.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Prompt Cache Share", + "panel_id": 17, + "panel_type": "stat", + "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Client Success Rate", + "panel_id": 18, + "panel_type": "stat", + "description": "Successful client-facing Switchyard responses in the selected range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Classifier Fail-Open (Range)", + "panel_id": 19, + "panel_type": "stat", + "description": "Local classifier failures that safely fell back to the conservative hosted route.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Upstream Errors (Range)", + "panel_id": 20, + "panel_type": "stat", + "description": "Hosted model attempts that returned errors in the selected range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Local Classifier Calls", + "panel_id": 21, + "panel_type": "timeseries", + "description": "Local Qwen routing-classifier activity, split by successful and failed calls.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~\"qwen.*\"}[5m]))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Routing Overhead p95", + "panel_id": 22, + "panel_type": "timeseries", + "description": "95th percentile time Switchyard spends selecting a model before the upstream call.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Traffic Lanes (Range)", + "panel_id": 23, + "panel_type": "bargauge", + "description": "Request volume split between interactive route traffic and durable worker traffic.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(sum by (lane) (label_replace(increase(switchyard_requests_total{model=~\"(route|worker)/.*\"}[$__range]), \"lane\", \"$1\", \"model\", \"^(route|worker)/.*\")))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Hermes Workload CPU (Attribution Proxy)", + "panel_id": 25, + "panel_type": "timeseries", + "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"hermes\",pod=~\"hermes-(agent|chat-tenant|switchyard|model-gate).*\",container!=\"\",image!=\"\"}[5m]))" + ] + }, { "dashboard": "Atlas GitOps", "panel_title": "Flux Source", diff --git a/services/comms/knowledge/catalog/atlas-summary.json b/services/comms/knowledge/catalog/atlas-summary.json index 9acf7417..cb8ec44b 100644 --- a/services/comms/knowledge/catalog/atlas-summary.json +++ b/services/comms/knowledge/catalog/atlas-summary.json @@ -2,7 +2,7 @@ "counts": { "helmrelease_host_hints": 23, "http_endpoints": 61, - "services": 91, - "workloads": 125 + "services": 98, + "workloads": 130 } } diff --git a/services/comms/knowledge/catalog/atlas.json b/services/comms/knowledge/catalog/atlas.json index d7247bfd..8265e842 100644 --- a/services/comms/knowledge/catalog/atlas.json +++ b/services/comms/knowledge/catalog/atlas.json @@ -106,6 +106,31 @@ "path": "services/hermes-chat", "targetNamespace": "hermes-chat" }, + { + "name": "hermes-observer-bindings", + "path": "services/hermes-observer-bindings", + "targetNamespace": null + }, + { + "name": "hermes-observer-rbac", + "path": "services/hermes-observer-rbac", + "targetNamespace": null + }, + { + "name": "hermes-scm-broker", + "path": "services/hermes-scm-broker", + "targetNamespace": "hermes-scm" + }, + { + "name": "hermes-scm-broker-code", + "path": "services/hermes/scm-common", + "targetNamespace": "hermes-scm" + }, + { + "name": "hermes-scm-namespace", + "path": "services/hermes-scm-namespace", + "targetNamespace": null + }, { "name": "hermes-triage-demo", "path": "services/hermes-triage-demo", @@ -251,6 +276,11 @@ "path": "infrastructure/vault-csi", "targetNamespace": "kube-system" }, + { + "name": "vault-hermes-jenkins-token-seed", + "path": "services/vault-hermes-jenkins-token-seed", + "targetNamespace": "vault" + }, { "name": "vault-injector", "path": "infrastructure/vault-injector", @@ -304,7 +334,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464" + "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-478" ] }, { @@ -320,7 +350,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464" + "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-478" ] }, { @@ -902,7 +932,7 @@ "serviceAccountName": "hermes-node-ssh-access", "nodeSelector": {}, "images": [ - "busybox:1.37" + "python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df" ] }, { @@ -915,8 +945,8 @@ "serviceAccountName": "hermes-triage", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8", - "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37", + "registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e" ] }, { @@ -930,7 +960,7 @@ "nodeSelector": {}, "images": [ "quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561", - "registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8" + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" ] }, { @@ -943,7 +973,7 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-chat-router@sha256:a4010fdc5b6dce2696e6fec06eb38a3992817faa01ff3472122c870038b22bd2" + "registry.bstein.dev/bstein/hermes-chat-router@sha256:6744cb7b87c6050f1b97c0675cd280b8295b3b5826ba1d6d92e37aee6fd0b8c4" ] }, { @@ -1058,6 +1088,48 @@ "registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-execution-mediator-0", + "labels": { + "app": "hermes-execution-mediator", + "pool-ordinal": "0" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-execution-mediator-1", + "labels": { + "app": "hermes-execution-mediator", + "pool-ordinal": "1" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-execution-mediator-2", + "labels": { + "app": "hermes-execution-mediator", + "pool-ordinal": "2" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, { "kind": "Deployment", "namespace": "hermes", @@ -1177,8 +1249,36 @@ "serviceAccountName": "hermes-chat", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8", - "registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1" + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37", + "registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf" + ] + }, + { + "kind": "StatefulSet", + "namespace": "hermes", + "name": "hermes-execution-worker", + "labels": { + "app": "hermes-execution-worker", + "app.kubernetes.io/name": "hermes-execution-worker", + "app.kubernetes.io/part-of": "hermes" + }, + "serviceAccountName": "hermes-execution-worker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes-scm", + "name": "hermes-scm-broker", + "labels": { + "app": "hermes-scm-broker" + }, + "serviceAccountName": "hermes-scm-broker", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/hermes-agent@sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7" ] }, { @@ -1528,7 +1628,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-amd64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-amd64" ] }, { @@ -1544,7 +1644,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-arm64" + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-arm64" ] }, { @@ -1633,7 +1733,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/ariadne:0.1.0-454" + "registry.bstein.dev/bstein/ariadne:0.1.0-464" ] }, { @@ -1661,7 +1761,7 @@ "serviceAccountName": "metis", "nodeSelector": {}, "images": [ - "registry.bstein.dev/bstein/metis:0.1.0-293-arm64" + "registry.bstein.dev/bstein/metis:0.1.0-304-arm64" ] }, { @@ -3179,6 +3279,22 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-cli-lane-metrics", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "lane-metrics", + "port": 9011, + "targetPort": "lane-metrics", + "protocol": "TCP" + } + ] + }, { "namespace": "hermes", "name": "hermes-codex-broker", @@ -3195,6 +3311,82 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes-execution-mediator-0", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-mediator", + "pool-ordinal": "0" + }, + "ports": [ + { + "name": "mediator", + "port": 9009, + "targetPort": "mediator", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-mediator-1", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-mediator", + "pool-ordinal": "1" + }, + "ports": [ + { + "name": "mediator", + "port": 9009, + "targetPort": "mediator", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-mediator-2", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-mediator", + "pool-ordinal": "2" + }, + "ports": [ + { + "name": "mediator", + "port": 9009, + "targetPort": "mediator", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-pool", + "type": "ClusterIP", + "selector": { + "app": "hermes-agent" + }, + "ports": [ + { + "name": "http", + "port": 9007, + "targetPort": "execution-pool", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-execution-worker", + "type": "ClusterIP", + "selector": { + "app": "hermes-execution-worker" + }, + "ports": [] + }, { "namespace": "hermes", "name": "hermes-gpu-handoff", @@ -3393,6 +3585,22 @@ } ] }, + { + "namespace": "hermes-scm", + "name": "hermes-scm-broker", + "type": "ClusterIP", + "selector": { + "app": "hermes-scm-broker" + }, + "ports": [ + { + "name": "http", + "port": 9081, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "jellyfin", "name": "jellyfin", diff --git a/services/comms/knowledge/catalog/atlas.yaml b/services/comms/knowledge/catalog/atlas.yaml index 8ae1b49b..544add7b 100644 --- a/services/comms/knowledge/catalog/atlas.yaml +++ b/services/comms/knowledge/catalog/atlas.yaml @@ -65,6 +65,21 @@ sources: - name: hermes-chat path: services/hermes-chat targetNamespace: hermes-chat +- name: hermes-observer-bindings + path: services/hermes-observer-bindings + targetNamespace: null +- name: hermes-observer-rbac + path: services/hermes-observer-rbac + targetNamespace: null +- name: hermes-scm-broker + path: services/hermes-scm-broker + targetNamespace: hermes-scm +- name: hermes-scm-broker-code + path: services/hermes/scm-common + targetNamespace: hermes-scm +- name: hermes-scm-namespace + path: services/hermes-scm-namespace + targetNamespace: null - name: hermes-triage-demo path: services/hermes-triage-demo targetNamespace: null @@ -152,6 +167,9 @@ sources: - name: vault-csi path: infrastructure/vault-csi targetNamespace: kube-system +- name: vault-hermes-jenkins-token-seed + path: services/vault-hermes-jenkins-token-seed + targetNamespace: vault - name: vault-injector path: infrastructure/vault-injector targetNamespace: vault @@ -187,7 +205,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-464 + - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-478 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-frontend @@ -198,7 +216,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-464 + - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-478 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-vault-sync @@ -604,7 +622,7 @@ workloads: serviceAccountName: hermes-node-ssh-access nodeSelector: {} images: - - busybox:1.37 + - python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df - kind: Deployment namespace: hermes name: hermes @@ -613,8 +631,8 @@ workloads: serviceAccountName: hermes-triage nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8 - - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 + - registry.bstein.dev/bstein/hermes-webui@sha256:ac6ba7bfd8a86227f31a9a96ebea41227ccf70391f7dcf34206f4d58e835e50e - kind: Deployment namespace: hermes name: hermes-agent @@ -624,7 +642,7 @@ workloads: nodeSelector: {} images: - quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561 - - registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8 + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 - kind: Deployment namespace: hermes name: hermes-chat-router @@ -633,7 +651,7 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-chat-router@sha256:a4010fdc5b6dce2696e6fec06eb38a3992817faa01ff3472122c870038b22bd2 + - registry.bstein.dev/bstein/hermes-chat-router@sha256:6744cb7b87c6050f1b97c0675cd280b8295b3b5826ba1d6d92e37aee6fd0b8c4 - kind: Deployment namespace: hermes name: hermes-chat-sandbox-0 @@ -714,6 +732,36 @@ workloads: nodeSelector: {} images: - registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca +- kind: Deployment + namespace: hermes + name: hermes-execution-mediator-0 + labels: + app: hermes-execution-mediator + pool-ordinal: '0' + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 +- kind: Deployment + namespace: hermes + name: hermes-execution-mediator-1 + labels: + app: hermes-execution-mediator + pool-ordinal: '1' + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 +- kind: Deployment + namespace: hermes + name: hermes-execution-mediator-2 + labels: + app: hermes-execution-mediator + pool-ordinal: '2' + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 - kind: Deployment namespace: hermes name: hermes-local-image @@ -797,8 +845,28 @@ workloads: serviceAccountName: hermes-chat nodeSelector: {} images: - - registry.bstein.dev/bstein/hermes-agent@sha256:cce1f65dc7d30fdce4b1748cc7d3434b9761ae9946a8c8be6049204c503d60a8 - - registry.bstein.dev/bstein/hermes-webui@sha256:9c2fe8341c7b650e08d10acead3151b19e2af737863268bafb39b3d9517575b1 + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 + - registry.bstein.dev/bstein/hermes-webui@sha256:c276a9e17c9057237472f39640c9ebac9d4359d9f62a34df52a759eef43167bf +- kind: StatefulSet + namespace: hermes + name: hermes-execution-worker + labels: + app: hermes-execution-worker + app.kubernetes.io/name: hermes-execution-worker + app.kubernetes.io/part-of: hermes + serviceAccountName: hermes-execution-worker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:4a385fbd04ae1e3e8ca0237db98a318606a2a03786ef467757b8da5eee4e4b37 +- kind: Deployment + namespace: hermes-scm + name: hermes-scm-broker + labels: + app: hermes-scm-broker + serviceAccountName: hermes-scm-broker + nodeSelector: {} + images: + - registry.bstein.dev/bstein/hermes-agent@sha256:37ebf720c783ae908a602916ffccf88d43d205a157957f5dc4b487867aee45e7 - kind: Deployment namespace: jellyfin name: jellyfin @@ -1037,7 +1105,7 @@ workloads: kubernetes.io/arch: amd64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-amd64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-amd64 - kind: DaemonSet namespace: maintenance name: metis-sentinel-arm64 @@ -1048,7 +1116,7 @@ workloads: kubernetes.io/arch: arm64 kubernetes.io/os: linux images: - - registry.bstein.dev/bstein/metis-sentinel:0.1.0-293-arm64 + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-304-arm64 - kind: DaemonSet namespace: maintenance name: node-image-sweeper @@ -1108,7 +1176,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/ariadne:0.1.0-454 + - registry.bstein.dev/bstein/ariadne:0.1.0-464 - kind: Deployment namespace: maintenance name: maintenance-vault-sync @@ -1127,7 +1195,7 @@ workloads: serviceAccountName: metis nodeSelector: {} images: - - registry.bstein.dev/bstein/metis:0.1.0-293-arm64 + - registry.bstein.dev/bstein/metis:0.1.0-304-arm64 - kind: Deployment namespace: maintenance name: oauth2-proxy-metis @@ -2120,6 +2188,16 @@ services: port: 9006 targetPort: claude-broker protocol: TCP +- namespace: hermes + name: hermes-cli-lane-metrics + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: lane-metrics + port: 9011 + targetPort: lane-metrics + protocol: TCP - namespace: hermes name: hermes-codex-broker type: ClusterIP @@ -2130,6 +2208,55 @@ services: port: 9003 targetPort: codex-broker protocol: TCP +- namespace: hermes + name: hermes-execution-mediator-0 + type: ClusterIP + selector: + app: hermes-execution-mediator + pool-ordinal: '0' + ports: + - name: mediator + port: 9009 + targetPort: mediator + protocol: TCP +- namespace: hermes + name: hermes-execution-mediator-1 + type: ClusterIP + selector: + app: hermes-execution-mediator + pool-ordinal: '1' + ports: + - name: mediator + port: 9009 + targetPort: mediator + protocol: TCP +- namespace: hermes + name: hermes-execution-mediator-2 + type: ClusterIP + selector: + app: hermes-execution-mediator + pool-ordinal: '2' + ports: + - name: mediator + port: 9009 + targetPort: mediator + protocol: TCP +- namespace: hermes + name: hermes-execution-pool + type: ClusterIP + selector: + app: hermes-agent + ports: + - name: http + port: 9007 + targetPort: execution-pool + protocol: TCP +- namespace: hermes + name: hermes-execution-worker + type: ClusterIP + selector: + app: hermes-execution-worker + ports: [] - namespace: hermes name: hermes-gpu-handoff type: ClusterIP @@ -2254,6 +2381,16 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: hermes-scm + name: hermes-scm-broker + type: ClusterIP + selector: + app: hermes-scm-broker + ports: + - name: http + port: 9081 + targetPort: http + protocol: TCP - namespace: jellyfin name: jellyfin type: ClusterIP diff --git a/services/comms/knowledge/catalog/metrics.json b/services/comms/knowledge/catalog/metrics.json index f26012fb..589033f1 100644 --- a/services/comms/knowledge/catalog/metrics.json +++ b/services/comms/knowledge/catalog/metrics.json @@ -1,4 +1,440 @@ [ + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Weekly Remaining", + "panel_id": 1, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Spark Weekly Remaining", + "panel_id": 2, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"openai\",limit=\"gpt-5-3-codex-spark\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 5h Remaining", + "panel_id": 3, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 7d Remaining", + "panel_id": 4, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Quota Collectors Healthy", + "panel_id": 5, + "panel_type": "stat", + "description": "Successful latest quota fetches. Providers are polled independently every five minutes.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(atlas_ai_quota_fetch_success) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Oldest Quota Sample", + "panel_id": 6, + "panel_type": "stat", + "description": "Age of the stalest successful provider quota snapshot.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((time() - atlas_ai_quota_last_success_timestamp_seconds) and (atlas_ai_quota_last_success_timestamp_seconds > 0)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Weekly Reset In", + "panel_id": 7, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"openai\",limit=\"codex\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 5h Reset In", + "panel_id": 8, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Claude 7d Reset In", + "panel_id": 9, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Tokens (Latest Day)", + "panel_id": 10, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_account_tokens{provider=\"openai\",period=\"latest_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Codex Tokens (7d)", + "panel_id": 11, + "panel_type": "stat", + "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(atlas_ai_account_tokens{provider=\"openai\",period=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"openai\"} == 1)) or on() vector(-1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Switchyard Requests", + "panel_id": 12, + "panel_type": "stat", + "description": "Hosted model requests observed by Switchyard in the selected dashboard range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(switchyard_requests_total[$__range])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Model Selection Rate", + "panel_id": 13, + "panel_type": "timeseries", + "description": "AUTO and fixed-route decisions by selected provider, model family, and reasoning effort.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum by (selected_model) (rate(switchyard_decisions_total[5m]))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Provider Selections (Range)", + "panel_id": 14, + "panel_type": "bargauge", + "description": "Switchyard selections grouped by provider over the selected dashboard range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(sum by (provider) (label_replace(increase(switchyard_decisions_total{selected_model=~\"(route|worker)/(codex|claude|local)/.*\"}[$__range]), \"provider\", \"$2\", \"selected_model\", \"^(route|worker)/(codex|claude|local)/.*\")))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Token Throughput", + "panel_id": 15, + "panel_type": "timeseries", + "description": "Prompt, cache, reasoning, and completion token rates reported by hosted Switchyard calls.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(rate(switchyard_prompt_tokens_total[5m]))", + "sum(rate(switchyard_cached_tokens_total[5m]))", + "sum(rate(switchyard_cache_creation_tokens_total[5m]))", + "sum(rate(switchyard_reasoning_tokens_total[5m]))", + "sum(rate(switchyard_completion_tokens_total[5m]))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Model Call p95 Latency", + "panel_id": 16, + "panel_type": "timeseries", + "description": "95th percentile upstream latency for each selected model route.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "histogram_quantile(0.95, sum by (le, model) (rate(switchyard_model_call_latency_ms_bucket[5m])))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Prompt Cache Share", + "panel_id": 17, + "panel_type": "stat", + "description": "Cached tokens as a share of prompt plus cached tokens; higher generally means less repeated provider work.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(rate(switchyard_cached_tokens_total[5m])) / clamp_min(sum(rate(switchyard_prompt_tokens_total[5m])) + sum(rate(switchyard_cached_tokens_total[5m])), 1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Client Success Rate", + "panel_id": 18, + "panel_type": "stat", + "description": "Successful client-facing Switchyard responses in the selected range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(increase(switchyard_client_responses_total{outcome=\"success\"}[$__range])) / clamp_min(sum(increase(switchyard_client_responses_total[$__range])), 1)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Classifier Fail-Open (Range)", + "panel_id": 19, + "panel_type": "stat", + "description": "Local classifier failures that safely fell back to the conservative hosted route.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(switchyard_classifier_fail_open_total[$__range])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Upstream Errors (Range)", + "panel_id": 20, + "panel_type": "stat", + "description": "Hosted model attempts that returned errors in the selected range.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(switchyard_errors_total[$__range])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Local Classifier Calls", + "panel_id": 21, + "panel_type": "timeseries", + "description": "Local Qwen routing-classifier activity, split by successful and failed calls.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum by (outcome) (rate(switchyard_llm_calls_total{selected_model=~\"qwen.*\"}[5m]))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Routing Overhead p95", + "panel_id": 22, + "panel_type": "timeseries", + "description": "95th percentile time Switchyard spends selecting a model before the upstream call.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "histogram_quantile(0.95, sum by (le, algorithm) (rate(switchyard_routing_overhead_ms_bucket[5m])))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Traffic Lanes (Range)", + "panel_id": 23, + "panel_type": "bargauge", + "description": "Request volume split between interactive route traffic and durable worker traffic.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(sum by (lane) (label_replace(increase(switchyard_requests_total{model=~\"(route|worker)/.*\"}[$__range]), \"lane\", \"$1\", \"model\", \"^(route|worker)/.*\")))" + ] + }, + { + "dashboard": "Atlas AI Operations", + "panel_title": "Hermes Workload CPU (Attribution Proxy)", + "panel_id": 25, + "panel_type": "timeseries", + "description": "Compute use by Hermes pod/container. Switchyard currently exposes model and worker-vs-route attribution, but not tenant-slot token labels; CPU is clearly marked as a proxy rather than token usage.", + "tags": [ + "atlas", + "ai", + "hermes", + "switchyard" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum by (pod, container) (rate(container_cpu_usage_seconds_total{namespace=\"hermes\",pod=~\"hermes-(agent|chat-tenant|switchyard|model-gate).*\",container!=\"\",image!=\"\"}[5m]))" + ] + }, { "dashboard": "Atlas GitOps", "panel_title": "Flux Source", -- 2.47.2 From 92ec0c0297d0dcf555202eba3f7917108fd2d674 Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Fri, 21 Aug 2026 10:00:15 +0000 Subject: [PATCH 2/2] refactor(hermes): rename the owner agent host to worker.bstein.dev Introduce worker.bstein.dev as the canonical hostname for the owner-only Hermes coordinator, previously agent.hermes.bstein.dev. The old host is deliberately kept alive. CoreDNS, both agent Ingresses and the hermes-sites certificate now serve BOTH names, so merging this cannot take away the endpoint the operator uses to reach the coordinator. Retiring agent.hermes.bstein.dev is a separate, separately scheduled change. No redirect middleware is added. What switches to the new host: - HERMES_DASHBOARD_PUBLIC_URL and the oauth2-proxy --redirect-url - the Keycloak hermes-agent-proxy rootUrl - operator docs, skills, the ZAP baseline target and the triage monitor default What stays dual-homed until retirement: - CoreDNS hosts entry, both agent Ingress rules, certificate SANs - API_SERVER_CORS_ORIGINS (now a comma-separated pair) - the Keycloak redirect URIs, web origins and post-logout origins, so a rollback only needs the oauth2-proxy --redirect-url reverted and does not require re-running the ensure job ensure_proxy_client now takes optional trailing legacy URLs; called with a single URL it emits the identical payload as before, so the chat and triage clients are unaffected. The ensure Job is bumped to -11 so Flux re-runs it; login on the new host fails until that Job completes. Because the session and CSRF cookies use the __Host- prefix they are bound to one origin, so a fresh login must start on worker.bstein.dev and existing sessions do not carry over -- re-login is required after rollout. Knowledge catalogs and diagrams regenerated with `make knowledge`. --- infrastructure/core/coredns-custom.yaml | 1 + knowledge/catalog/atlas-summary.json | 2 +- knowledge/catalog/atlas.json | 40 +++++++++++++++++++ knowledge/catalog/atlas.yaml | 22 ++++++++++ knowledge/catalog/runbooks.json | 2 +- knowledge/diagrams/atlas-http.mmd | 2 + knowledge/runbooks/hermes-triage-demo.md | 2 +- scripts/ops/hermes_triage_monitor.py | 2 +- .../knowledge/catalog/atlas-summary.json | 2 +- services/comms/knowledge/catalog/atlas.json | 40 +++++++++++++++++++ services/comms/knowledge/catalog/atlas.yaml | 22 ++++++++++ .../comms/knowledge/catalog/runbooks.json | 2 +- .../comms/knowledge/diagrams/atlas-http.mmd | 2 + .../knowledge/runbooks/hermes-triage-demo.md | 2 +- services/hermes/NOTES.md | 4 +- services/hermes/agent-certificate.yaml | 3 ++ services/hermes/agent-configmap.yaml | 6 +-- services/hermes/agent-deployment.yaml | 11 +++-- services/hermes/agent-ingress.yaml | 31 +++++++++++++- services/hermes/configmap.yaml | 2 +- .../references/architecture.md | 2 +- .../hermes-access-oidc-client-job.yaml | 2 +- .../scripts/hermes_access_oidc_ensure.sh | 24 +++++++---- services/quality/zap-baseline-configmap.yaml | 2 +- .../test_hermes_agent_runtime_patches.py | 37 ++++++++++++++++- .../tests/test_hermes_cli_lanes_toolchain.py | 4 +- 26 files changed, 239 insertions(+), 32 deletions(-) diff --git a/infrastructure/core/coredns-custom.yaml b/infrastructure/core/coredns-custom.yaml index 0f39506f..b54d6fb4 100644 --- a/infrastructure/core/coredns-custom.yaml +++ b/infrastructure/core/coredns-custom.yaml @@ -47,6 +47,7 @@ data: 192.168.22.9 tasks.bstein.dev 192.168.22.9 triage.bstein.dev 192.168.22.9 vault.bstein.dev + 192.168.22.9 worker.bstein.dev fallthrough } } diff --git a/knowledge/catalog/atlas-summary.json b/knowledge/catalog/atlas-summary.json index cb8ec44b..05ba796d 100644 --- a/knowledge/catalog/atlas-summary.json +++ b/knowledge/catalog/atlas-summary.json @@ -1,7 +1,7 @@ { "counts": { "helmrelease_host_hints": 23, - "http_endpoints": 61, + "http_endpoints": 63, "services": 98, "workloads": 130 } diff --git a/knowledge/catalog/atlas.json b/knowledge/catalog/atlas.json index 8265e842..aaeb8ca5 100644 --- a/knowledge/catalog/atlas.json +++ b/knowledge/catalog/atlas.json @@ -5510,6 +5510,46 @@ "name": "wolf", "source": "game-stream" } + }, + { + "host": "worker.bstein.dev", + "path": "/", + "backend": { + "namespace": "hermes", + "service": "oauth2-proxy-hermes-agent", + "port": "http", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes-agent" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "hermes-agent-dashboard", + "source": "hermes" + } + }, + { + "host": "worker.bstein.dev", + "path": "/terminal", + "backend": { + "namespace": "hermes", + "service": "oauth2-proxy-hermes-agent", + "port": "http", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes-agent" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "hermes-agent-terminal", + "source": "hermes" + } } ], "helmrelease_host_hints": { diff --git a/knowledge/catalog/atlas.yaml b/knowledge/catalog/atlas.yaml index 544add7b..496ba92c 100644 --- a/knowledge/catalog/atlas.yaml +++ b/knowledge/catalog/atlas.yaml @@ -3585,6 +3585,28 @@ http_endpoints: kind: Ingress name: wolf source: game-stream +- host: worker.bstein.dev + path: / + backend: + namespace: hermes + service: oauth2-proxy-hermes-agent + port: http + workloads: *id001 + via: + kind: Ingress + name: hermes-agent-dashboard + source: hermes +- host: worker.bstein.dev + path: /terminal + backend: + namespace: hermes + service: oauth2-proxy-hermes-agent + port: http + workloads: *id001 + via: + kind: Ingress + name: hermes-agent-terminal + source: hermes helmrelease_host_hints: cert-manager:cert-manager/cert-manager: - atlas.bstein.dev diff --git a/knowledge/catalog/runbooks.json b/knowledge/catalog/runbooks.json index f72f166b..c9f8e588 100644 --- a/knowledge/catalog/runbooks.json +++ b/knowledge/catalog/runbooks.json @@ -50,7 +50,7 @@ "tags": [], "entrypoints": [], "source_paths": [], - "body": "# Hermes Automated Triage Demo \u2014 Runbook\n\nHow to arm, run, watch, and roll back the automated failure-to-repair demo.\nArchitecture background: `knowledge/hermes-automated-triage-24h-plan.md`.\n\n## What the loop does\n\n```text\nYou arm the failure (SEED_FAILURE=true)\n -> Jenkins job hermes-triage-demo fails (fixture reads \"unhealthy\")\n -> test-runner pod also writes the incident JSON to stdout\n (Fluent Bit ships it to OpenSearch kube-*)\n -> Ariadne polls the job every minute, opens incident /\n -> Ariadne bundles Jenkins evidence + bounded OpenSearch excerpts\n -> Ariadne calls the Hermes Agent API (/v1/runs) with\n $triage-titan-test-failures\n -> Hermes returns schema-valid diagnosis + requested_action\n -> Ariadne authorizes (twelve gates) and, if remediation is enabled,\n patches the fixture ConfigMap back to \"healthy\" in process\n -> Ariadne triggers ONE rebuild with SEED_FAILURE=false\n -> rebuild passes -> incident resolved\nAnything else -> issue in the service repo + human_required metric\n```\n\nThe model behind Hermes is `anthropic/claude-opus-5` as of 2026-08-06, with\n`openai-codex/gpt-5.6-terra` as first fallback and a local `gpt-oss:20b`\nbehind that, so an expired Anthropic credential degrades rather than stops.\n\nTwo behaviours worth knowing before you demo, because both look like nothing\nhappening:\n\n- **A build that never finishes** is escalated once it passes\n `ARIADNE_HERMES_HUNG_BUILD_MINUTES` (default 45). No model is consulted -\n the console is still being written - so the issue says only that the build\n overran and is holding an agent slot.\n- **Alerting no longer fires on every escalation.** The issue in the service\n repository is the durable artifact. Email now means either a repair ran and\n failed, or an escalation has sat untouched for six hours.\n\n## Arming the demo\n\nJenkins UI: `https://ci.bstein.dev/job/hermes-triage-demo/` \u2192 *Build with\nParameters* \u2192 check `SEED_FAILURE` \u2192 Build.\n\nCLI (any Jenkins user API token):\n\n```bash\ncurl -u : -X POST \\\n \"https://ci.bstein.dev/job/hermes-triage-demo/buildWithParameters?SEED_FAILURE=true\"\n```\n\nThat is the only manual step. Everything after the red build is automatic.\n\nRun `scripts/ops/hermes_triage_demo.sh preflight` first. The check that most\noften decides whether a rehearsal holds its timings is the agent pool: the\nKubernetes cloud caps concurrent agent pods at `containerCapStr: \"5\"`, and when\nreal CI has taken all five the demo build sits in the queue reporting *\"All\nnodes of label ... are offline\"* with no other symptom. Observed 2026-08-06: a\nrun armed at 02:13:42 did not start seeding until 02:19:37, close to six\nminutes of dead air, purely because five other builds held the pool. Wait for a\nfree slot before starting, or quiesce CI.\n\n## Expected timings (measured live 2026-08-06, Ariadne 0.1.0-402)\n\nTime the demo from the moment the build goes **red**, not from arming. Arming\nonly queues a Jenkins build, and that leg is at the mercy of the agent pool.\n\n- Red \u2192 fixture patched back to healthy: **25s** (\u226460 s detection tick, ~16 s\n Hermes diagnosis, then a single in-process Kubernetes API call).\n- Red \u2192 rebuild triggered: **38s**.\n- Red \u2192 rebuild green and incident resolved: **1m04s**.\n\nThe whole automated leg is just over a minute, and there is no silent phase\nlonger than the detection tick. Budgeted Hermes timeout is 420 s; observed\ndiagnosis runs are 15\u201321 s.\n\nArming \u2192 red was **6m21s** on this run, but 5m35s of that was queue wait\nbehind a saturated agent pool. On an idle pool expect roughly 1m45s.\n\nThe earlier 2026-08-05 figures (1m15s red\u2192repaired, 4m00s total) were measured\nwhen the repair spawned its own Kubernetes Job. Converting the repair to an\nin-process call removed a whole pod launch from the critical path.\n\n## Watching it live\n\n- Jenkins: `https://ci.bstein.dev/job/hermes-triage-demo/` (red build N,\n then green build N+1 with `SEED_FAILURE=false`).\n- Ariadne incident state:\n `GET http://ariadne.maintenance/api/internal/audit/events` (in-cluster) or\n `/api/admin/audit/events` (Keycloak JWT) \u2014 event types\n `hermes_autotriage_incident`, `hermes_autotriage_diagnosis`,\n `hermes_autotriage_action`. Status flow:\n `detected \u2192 diagnosed \u2192 repairing \u2192 awaiting_rebuild \u2192 resolved`.\n- Metrics (VictoriaMetrics / Grafana Explore):\n `ariadne_hermes_triage_incident{jenkins_job=\"hermes-triage-demo\"}`,\n `ariadne_hermes_triage_action_total`,\n `ariadne_hermes_triage_duration_seconds`.\n- Repair evidence: `kubectl -n hermes-triage-demo get jobs` shows\n `hermes-demo-test-` (failed) and `hermes-demo-repair-` (succeeded);\n both TTL-clean after 1 h.\n- Hermes side: the run appears in the dashboard at\n `https://agent.hermes.bstein.dev` (session/run history).\n- Escalation path: alert `HermesTriageHumanRequired` in vmalert\n (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`)\n fires to Alertmanager for any `human_required` incident. Gauges are\n republished every tick from stored incident state (restart-safe), and\n the alert self-clears once a newer build of the same job is green.\n Note: Alertmanager's default receiver is currently null \u2014 the alert is\n visible in vmalert/Alertmanager/Grafana but pushes no notification.\n\n## Demonstrating safe escalation (second path)\n\nAny failure that does not match the demo-fixture signature \u2014 or any\ninvalid/low-confidence/unknown-action Hermes response \u2014 ends as\n`status=\"human_required\"` with **no mutation**. The simplest live demo:\ntemporarily set `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` (see below)\nand arm the failure; Ariadne diagnoses fully but executes nothing, and the\nalert fires instead.\n\n## Kill switch and rollback\n\n- Instant behavioral off-switch (Flux-managed, in\n `services/maintenance/apps/ariadne-deployment.yaml`):\n `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` \u2192 diagnose-only.\n `ARIADNE_HERMES_AUTOTRIAGE_ENABLED=false` \u2192 fully off.\n- The automatic loop can only ever: create Jobs named\n `hermes-demo-repair-*` in namespace `hermes-triage-demo`, and trigger\n rebuilds of allowlisted jobs (`ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST`,\n currently `hermes-triage-demo` only). One action + one rebuild per\n incident, ever (storage-backed idempotency).\n- Full teardown: remove `hermes-triage-demo` from\n `clusters/atlas/flux-system/applications/kustomization.yaml` (Flux prunes\n the namespace) and delete the `pipelineJob('hermes-triage-demo')` block\n from `services/jenkins/configmap-jcasc.yaml`.\n\n## Credentials\n\n- Ariadne \u2192 Hermes: `Authorization: Bearer` key from\n `kv/atlas/hermes/triage-api`. Vault Agent injects it into both workloads at\n runtime; it is not stored in either persistent Hermes state or a manually\n maintained Kubernetes Secret. Rotate the Vault value and roll both tracked\n deployments through Flux.\n- Ariadne \u2192 Jenkins: existing `JENKINS_API_USER/TOKEN` from Vault\n (`atlas/maintenance/ariadne-db`)." + "body": "# Hermes Automated Triage Demo \u2014 Runbook\n\nHow to arm, run, watch, and roll back the automated failure-to-repair demo.\nArchitecture background: `knowledge/hermes-automated-triage-24h-plan.md`.\n\n## What the loop does\n\n```text\nYou arm the failure (SEED_FAILURE=true)\n -> Jenkins job hermes-triage-demo fails (fixture reads \"unhealthy\")\n -> test-runner pod also writes the incident JSON to stdout\n (Fluent Bit ships it to OpenSearch kube-*)\n -> Ariadne polls the job every minute, opens incident /\n -> Ariadne bundles Jenkins evidence + bounded OpenSearch excerpts\n -> Ariadne calls the Hermes Agent API (/v1/runs) with\n $triage-titan-test-failures\n -> Hermes returns schema-valid diagnosis + requested_action\n -> Ariadne authorizes (twelve gates) and, if remediation is enabled,\n patches the fixture ConfigMap back to \"healthy\" in process\n -> Ariadne triggers ONE rebuild with SEED_FAILURE=false\n -> rebuild passes -> incident resolved\nAnything else -> issue in the service repo + human_required metric\n```\n\nThe model behind Hermes is `anthropic/claude-opus-5` as of 2026-08-06, with\n`openai-codex/gpt-5.6-terra` as first fallback and a local `gpt-oss:20b`\nbehind that, so an expired Anthropic credential degrades rather than stops.\n\nTwo behaviours worth knowing before you demo, because both look like nothing\nhappening:\n\n- **A build that never finishes** is escalated once it passes\n `ARIADNE_HERMES_HUNG_BUILD_MINUTES` (default 45). No model is consulted -\n the console is still being written - so the issue says only that the build\n overran and is holding an agent slot.\n- **Alerting no longer fires on every escalation.** The issue in the service\n repository is the durable artifact. Email now means either a repair ran and\n failed, or an escalation has sat untouched for six hours.\n\n## Arming the demo\n\nJenkins UI: `https://ci.bstein.dev/job/hermes-triage-demo/` \u2192 *Build with\nParameters* \u2192 check `SEED_FAILURE` \u2192 Build.\n\nCLI (any Jenkins user API token):\n\n```bash\ncurl -u : -X POST \\\n \"https://ci.bstein.dev/job/hermes-triage-demo/buildWithParameters?SEED_FAILURE=true\"\n```\n\nThat is the only manual step. Everything after the red build is automatic.\n\nRun `scripts/ops/hermes_triage_demo.sh preflight` first. The check that most\noften decides whether a rehearsal holds its timings is the agent pool: the\nKubernetes cloud caps concurrent agent pods at `containerCapStr: \"5\"`, and when\nreal CI has taken all five the demo build sits in the queue reporting *\"All\nnodes of label ... are offline\"* with no other symptom. Observed 2026-08-06: a\nrun armed at 02:13:42 did not start seeding until 02:19:37, close to six\nminutes of dead air, purely because five other builds held the pool. Wait for a\nfree slot before starting, or quiesce CI.\n\n## Expected timings (measured live 2026-08-06, Ariadne 0.1.0-402)\n\nTime the demo from the moment the build goes **red**, not from arming. Arming\nonly queues a Jenkins build, and that leg is at the mercy of the agent pool.\n\n- Red \u2192 fixture patched back to healthy: **25s** (\u226460 s detection tick, ~16 s\n Hermes diagnosis, then a single in-process Kubernetes API call).\n- Red \u2192 rebuild triggered: **38s**.\n- Red \u2192 rebuild green and incident resolved: **1m04s**.\n\nThe whole automated leg is just over a minute, and there is no silent phase\nlonger than the detection tick. Budgeted Hermes timeout is 420 s; observed\ndiagnosis runs are 15\u201321 s.\n\nArming \u2192 red was **6m21s** on this run, but 5m35s of that was queue wait\nbehind a saturated agent pool. On an idle pool expect roughly 1m45s.\n\nThe earlier 2026-08-05 figures (1m15s red\u2192repaired, 4m00s total) were measured\nwhen the repair spawned its own Kubernetes Job. Converting the repair to an\nin-process call removed a whole pod launch from the critical path.\n\n## Watching it live\n\n- Jenkins: `https://ci.bstein.dev/job/hermes-triage-demo/` (red build N,\n then green build N+1 with `SEED_FAILURE=false`).\n- Ariadne incident state:\n `GET http://ariadne.maintenance/api/internal/audit/events` (in-cluster) or\n `/api/admin/audit/events` (Keycloak JWT) \u2014 event types\n `hermes_autotriage_incident`, `hermes_autotriage_diagnosis`,\n `hermes_autotriage_action`. Status flow:\n `detected \u2192 diagnosed \u2192 repairing \u2192 awaiting_rebuild \u2192 resolved`.\n- Metrics (VictoriaMetrics / Grafana Explore):\n `ariadne_hermes_triage_incident{jenkins_job=\"hermes-triage-demo\"}`,\n `ariadne_hermes_triage_action_total`,\n `ariadne_hermes_triage_duration_seconds`.\n- Repair evidence: `kubectl -n hermes-triage-demo get jobs` shows\n `hermes-demo-test-` (failed) and `hermes-demo-repair-` (succeeded);\n both TTL-clean after 1 h.\n- Hermes side: the run appears in the dashboard at\n `https://worker.bstein.dev` (session/run history).\n- Escalation path: alert `HermesTriageHumanRequired` in vmalert\n (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`)\n fires to Alertmanager for any `human_required` incident. Gauges are\n republished every tick from stored incident state (restart-safe), and\n the alert self-clears once a newer build of the same job is green.\n Note: Alertmanager's default receiver is currently null \u2014 the alert is\n visible in vmalert/Alertmanager/Grafana but pushes no notification.\n\n## Demonstrating safe escalation (second path)\n\nAny failure that does not match the demo-fixture signature \u2014 or any\ninvalid/low-confidence/unknown-action Hermes response \u2014 ends as\n`status=\"human_required\"` with **no mutation**. The simplest live demo:\ntemporarily set `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` (see below)\nand arm the failure; Ariadne diagnoses fully but executes nothing, and the\nalert fires instead.\n\n## Kill switch and rollback\n\n- Instant behavioral off-switch (Flux-managed, in\n `services/maintenance/apps/ariadne-deployment.yaml`):\n `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` \u2192 diagnose-only.\n `ARIADNE_HERMES_AUTOTRIAGE_ENABLED=false` \u2192 fully off.\n- The automatic loop can only ever: create Jobs named\n `hermes-demo-repair-*` in namespace `hermes-triage-demo`, and trigger\n rebuilds of allowlisted jobs (`ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST`,\n currently `hermes-triage-demo` only). One action + one rebuild per\n incident, ever (storage-backed idempotency).\n- Full teardown: remove `hermes-triage-demo` from\n `clusters/atlas/flux-system/applications/kustomization.yaml` (Flux prunes\n the namespace) and delete the `pipelineJob('hermes-triage-demo')` block\n from `services/jenkins/configmap-jcasc.yaml`.\n\n## Credentials\n\n- Ariadne \u2192 Hermes: `Authorization: Bearer` key from\n `kv/atlas/hermes/triage-api`. Vault Agent injects it into both workloads at\n runtime; it is not stored in either persistent Hermes state or a manually\n maintained Kubernetes Secret. Rotate the Vault value and roll both tracked\n deployments through Flux.\n- Ariadne \u2192 Jenkins: existing `JENKINS_API_USER/TOKEN` from Vault\n (`atlas/maintenance/ariadne-db`)." }, { "path": "runbooks/kb-authoring.md", diff --git a/knowledge/diagrams/atlas-http.mmd b/knowledge/diagrams/atlas-http.mmd index 5c0c9be7..28e35b5e 100644 --- a/knowledge/diagrams/atlas-http.mmd +++ b/knowledge/diagrams/atlas-http.mmd @@ -194,6 +194,8 @@ flowchart LR host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"] svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf + host_worker_bstein_dev["worker.bstein.dev"] + host_worker_bstein_dev --> svc_hermes_oauth2_proxy_hermes_agent subgraph bstein_dev_home[bstein-dev-home] svc_bstein_dev_home_bstein_dev_home_frontend diff --git a/knowledge/runbooks/hermes-triage-demo.md b/knowledge/runbooks/hermes-triage-demo.md index 1a5a64e6..c73d92e9 100644 --- a/knowledge/runbooks/hermes-triage-demo.md +++ b/knowledge/runbooks/hermes-triage-demo.md @@ -99,7 +99,7 @@ in-process call removed a whole pod launch from the critical path. `hermes-demo-test-` (failed) and `hermes-demo-repair-` (succeeded); both TTL-clean after 1 h. - Hermes side: the run appears in the dashboard at - `https://agent.hermes.bstein.dev` (session/run history). + `https://worker.bstein.dev` (session/run history). - Escalation path: alert `HermesTriageHumanRequired` in vmalert (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`) fires to Alertmanager for any `human_required` incident. Gauges are diff --git a/scripts/ops/hermes_triage_monitor.py b/scripts/ops/hermes_triage_monitor.py index 24ea9714..5ceb44be 100755 --- a/scripts/ops/hermes_triage_monitor.py +++ b/scripts/ops/hermes_triage_monitor.py @@ -40,7 +40,7 @@ JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev") GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev") CODE_REPO_OWNER = os.environ.get("CODE_REPO_OWNER", "atlas") GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev") -HERMES_UI = os.environ.get("HERMES_URL", "https://agent.hermes.bstein.dev") +HERMES_UI = os.environ.get("HERMES_URL", "https://worker.bstein.dev") POLL_SECONDS = 6 # `--filter ` restricts the monitor to incidents whose id contains that diff --git a/services/comms/knowledge/catalog/atlas-summary.json b/services/comms/knowledge/catalog/atlas-summary.json index cb8ec44b..05ba796d 100644 --- a/services/comms/knowledge/catalog/atlas-summary.json +++ b/services/comms/knowledge/catalog/atlas-summary.json @@ -1,7 +1,7 @@ { "counts": { "helmrelease_host_hints": 23, - "http_endpoints": 61, + "http_endpoints": 63, "services": 98, "workloads": 130 } diff --git a/services/comms/knowledge/catalog/atlas.json b/services/comms/knowledge/catalog/atlas.json index 8265e842..aaeb8ca5 100644 --- a/services/comms/knowledge/catalog/atlas.json +++ b/services/comms/knowledge/catalog/atlas.json @@ -5510,6 +5510,46 @@ "name": "wolf", "source": "game-stream" } + }, + { + "host": "worker.bstein.dev", + "path": "/", + "backend": { + "namespace": "hermes", + "service": "oauth2-proxy-hermes-agent", + "port": "http", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes-agent" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "hermes-agent-dashboard", + "source": "hermes" + } + }, + { + "host": "worker.bstein.dev", + "path": "/terminal", + "backend": { + "namespace": "hermes", + "service": "oauth2-proxy-hermes-agent", + "port": "http", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes-agent" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "hermes-agent-terminal", + "source": "hermes" + } } ], "helmrelease_host_hints": { diff --git a/services/comms/knowledge/catalog/atlas.yaml b/services/comms/knowledge/catalog/atlas.yaml index 544add7b..496ba92c 100644 --- a/services/comms/knowledge/catalog/atlas.yaml +++ b/services/comms/knowledge/catalog/atlas.yaml @@ -3585,6 +3585,28 @@ http_endpoints: kind: Ingress name: wolf source: game-stream +- host: worker.bstein.dev + path: / + backend: + namespace: hermes + service: oauth2-proxy-hermes-agent + port: http + workloads: *id001 + via: + kind: Ingress + name: hermes-agent-dashboard + source: hermes +- host: worker.bstein.dev + path: /terminal + backend: + namespace: hermes + service: oauth2-proxy-hermes-agent + port: http + workloads: *id001 + via: + kind: Ingress + name: hermes-agent-terminal + source: hermes helmrelease_host_hints: cert-manager:cert-manager/cert-manager: - atlas.bstein.dev diff --git a/services/comms/knowledge/catalog/runbooks.json b/services/comms/knowledge/catalog/runbooks.json index f72f166b..c9f8e588 100644 --- a/services/comms/knowledge/catalog/runbooks.json +++ b/services/comms/knowledge/catalog/runbooks.json @@ -50,7 +50,7 @@ "tags": [], "entrypoints": [], "source_paths": [], - "body": "# Hermes Automated Triage Demo \u2014 Runbook\n\nHow to arm, run, watch, and roll back the automated failure-to-repair demo.\nArchitecture background: `knowledge/hermes-automated-triage-24h-plan.md`.\n\n## What the loop does\n\n```text\nYou arm the failure (SEED_FAILURE=true)\n -> Jenkins job hermes-triage-demo fails (fixture reads \"unhealthy\")\n -> test-runner pod also writes the incident JSON to stdout\n (Fluent Bit ships it to OpenSearch kube-*)\n -> Ariadne polls the job every minute, opens incident /\n -> Ariadne bundles Jenkins evidence + bounded OpenSearch excerpts\n -> Ariadne calls the Hermes Agent API (/v1/runs) with\n $triage-titan-test-failures\n -> Hermes returns schema-valid diagnosis + requested_action\n -> Ariadne authorizes (twelve gates) and, if remediation is enabled,\n patches the fixture ConfigMap back to \"healthy\" in process\n -> Ariadne triggers ONE rebuild with SEED_FAILURE=false\n -> rebuild passes -> incident resolved\nAnything else -> issue in the service repo + human_required metric\n```\n\nThe model behind Hermes is `anthropic/claude-opus-5` as of 2026-08-06, with\n`openai-codex/gpt-5.6-terra` as first fallback and a local `gpt-oss:20b`\nbehind that, so an expired Anthropic credential degrades rather than stops.\n\nTwo behaviours worth knowing before you demo, because both look like nothing\nhappening:\n\n- **A build that never finishes** is escalated once it passes\n `ARIADNE_HERMES_HUNG_BUILD_MINUTES` (default 45). No model is consulted -\n the console is still being written - so the issue says only that the build\n overran and is holding an agent slot.\n- **Alerting no longer fires on every escalation.** The issue in the service\n repository is the durable artifact. Email now means either a repair ran and\n failed, or an escalation has sat untouched for six hours.\n\n## Arming the demo\n\nJenkins UI: `https://ci.bstein.dev/job/hermes-triage-demo/` \u2192 *Build with\nParameters* \u2192 check `SEED_FAILURE` \u2192 Build.\n\nCLI (any Jenkins user API token):\n\n```bash\ncurl -u : -X POST \\\n \"https://ci.bstein.dev/job/hermes-triage-demo/buildWithParameters?SEED_FAILURE=true\"\n```\n\nThat is the only manual step. Everything after the red build is automatic.\n\nRun `scripts/ops/hermes_triage_demo.sh preflight` first. The check that most\noften decides whether a rehearsal holds its timings is the agent pool: the\nKubernetes cloud caps concurrent agent pods at `containerCapStr: \"5\"`, and when\nreal CI has taken all five the demo build sits in the queue reporting *\"All\nnodes of label ... are offline\"* with no other symptom. Observed 2026-08-06: a\nrun armed at 02:13:42 did not start seeding until 02:19:37, close to six\nminutes of dead air, purely because five other builds held the pool. Wait for a\nfree slot before starting, or quiesce CI.\n\n## Expected timings (measured live 2026-08-06, Ariadne 0.1.0-402)\n\nTime the demo from the moment the build goes **red**, not from arming. Arming\nonly queues a Jenkins build, and that leg is at the mercy of the agent pool.\n\n- Red \u2192 fixture patched back to healthy: **25s** (\u226460 s detection tick, ~16 s\n Hermes diagnosis, then a single in-process Kubernetes API call).\n- Red \u2192 rebuild triggered: **38s**.\n- Red \u2192 rebuild green and incident resolved: **1m04s**.\n\nThe whole automated leg is just over a minute, and there is no silent phase\nlonger than the detection tick. Budgeted Hermes timeout is 420 s; observed\ndiagnosis runs are 15\u201321 s.\n\nArming \u2192 red was **6m21s** on this run, but 5m35s of that was queue wait\nbehind a saturated agent pool. On an idle pool expect roughly 1m45s.\n\nThe earlier 2026-08-05 figures (1m15s red\u2192repaired, 4m00s total) were measured\nwhen the repair spawned its own Kubernetes Job. Converting the repair to an\nin-process call removed a whole pod launch from the critical path.\n\n## Watching it live\n\n- Jenkins: `https://ci.bstein.dev/job/hermes-triage-demo/` (red build N,\n then green build N+1 with `SEED_FAILURE=false`).\n- Ariadne incident state:\n `GET http://ariadne.maintenance/api/internal/audit/events` (in-cluster) or\n `/api/admin/audit/events` (Keycloak JWT) \u2014 event types\n `hermes_autotriage_incident`, `hermes_autotriage_diagnosis`,\n `hermes_autotriage_action`. Status flow:\n `detected \u2192 diagnosed \u2192 repairing \u2192 awaiting_rebuild \u2192 resolved`.\n- Metrics (VictoriaMetrics / Grafana Explore):\n `ariadne_hermes_triage_incident{jenkins_job=\"hermes-triage-demo\"}`,\n `ariadne_hermes_triage_action_total`,\n `ariadne_hermes_triage_duration_seconds`.\n- Repair evidence: `kubectl -n hermes-triage-demo get jobs` shows\n `hermes-demo-test-` (failed) and `hermes-demo-repair-` (succeeded);\n both TTL-clean after 1 h.\n- Hermes side: the run appears in the dashboard at\n `https://agent.hermes.bstein.dev` (session/run history).\n- Escalation path: alert `HermesTriageHumanRequired` in vmalert\n (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`)\n fires to Alertmanager for any `human_required` incident. Gauges are\n republished every tick from stored incident state (restart-safe), and\n the alert self-clears once a newer build of the same job is green.\n Note: Alertmanager's default receiver is currently null \u2014 the alert is\n visible in vmalert/Alertmanager/Grafana but pushes no notification.\n\n## Demonstrating safe escalation (second path)\n\nAny failure that does not match the demo-fixture signature \u2014 or any\ninvalid/low-confidence/unknown-action Hermes response \u2014 ends as\n`status=\"human_required\"` with **no mutation**. The simplest live demo:\ntemporarily set `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` (see below)\nand arm the failure; Ariadne diagnoses fully but executes nothing, and the\nalert fires instead.\n\n## Kill switch and rollback\n\n- Instant behavioral off-switch (Flux-managed, in\n `services/maintenance/apps/ariadne-deployment.yaml`):\n `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` \u2192 diagnose-only.\n `ARIADNE_HERMES_AUTOTRIAGE_ENABLED=false` \u2192 fully off.\n- The automatic loop can only ever: create Jobs named\n `hermes-demo-repair-*` in namespace `hermes-triage-demo`, and trigger\n rebuilds of allowlisted jobs (`ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST`,\n currently `hermes-triage-demo` only). One action + one rebuild per\n incident, ever (storage-backed idempotency).\n- Full teardown: remove `hermes-triage-demo` from\n `clusters/atlas/flux-system/applications/kustomization.yaml` (Flux prunes\n the namespace) and delete the `pipelineJob('hermes-triage-demo')` block\n from `services/jenkins/configmap-jcasc.yaml`.\n\n## Credentials\n\n- Ariadne \u2192 Hermes: `Authorization: Bearer` key from\n `kv/atlas/hermes/triage-api`. Vault Agent injects it into both workloads at\n runtime; it is not stored in either persistent Hermes state or a manually\n maintained Kubernetes Secret. Rotate the Vault value and roll both tracked\n deployments through Flux.\n- Ariadne \u2192 Jenkins: existing `JENKINS_API_USER/TOKEN` from Vault\n (`atlas/maintenance/ariadne-db`)." + "body": "# Hermes Automated Triage Demo \u2014 Runbook\n\nHow to arm, run, watch, and roll back the automated failure-to-repair demo.\nArchitecture background: `knowledge/hermes-automated-triage-24h-plan.md`.\n\n## What the loop does\n\n```text\nYou arm the failure (SEED_FAILURE=true)\n -> Jenkins job hermes-triage-demo fails (fixture reads \"unhealthy\")\n -> test-runner pod also writes the incident JSON to stdout\n (Fluent Bit ships it to OpenSearch kube-*)\n -> Ariadne polls the job every minute, opens incident /\n -> Ariadne bundles Jenkins evidence + bounded OpenSearch excerpts\n -> Ariadne calls the Hermes Agent API (/v1/runs) with\n $triage-titan-test-failures\n -> Hermes returns schema-valid diagnosis + requested_action\n -> Ariadne authorizes (twelve gates) and, if remediation is enabled,\n patches the fixture ConfigMap back to \"healthy\" in process\n -> Ariadne triggers ONE rebuild with SEED_FAILURE=false\n -> rebuild passes -> incident resolved\nAnything else -> issue in the service repo + human_required metric\n```\n\nThe model behind Hermes is `anthropic/claude-opus-5` as of 2026-08-06, with\n`openai-codex/gpt-5.6-terra` as first fallback and a local `gpt-oss:20b`\nbehind that, so an expired Anthropic credential degrades rather than stops.\n\nTwo behaviours worth knowing before you demo, because both look like nothing\nhappening:\n\n- **A build that never finishes** is escalated once it passes\n `ARIADNE_HERMES_HUNG_BUILD_MINUTES` (default 45). No model is consulted -\n the console is still being written - so the issue says only that the build\n overran and is holding an agent slot.\n- **Alerting no longer fires on every escalation.** The issue in the service\n repository is the durable artifact. Email now means either a repair ran and\n failed, or an escalation has sat untouched for six hours.\n\n## Arming the demo\n\nJenkins UI: `https://ci.bstein.dev/job/hermes-triage-demo/` \u2192 *Build with\nParameters* \u2192 check `SEED_FAILURE` \u2192 Build.\n\nCLI (any Jenkins user API token):\n\n```bash\ncurl -u : -X POST \\\n \"https://ci.bstein.dev/job/hermes-triage-demo/buildWithParameters?SEED_FAILURE=true\"\n```\n\nThat is the only manual step. Everything after the red build is automatic.\n\nRun `scripts/ops/hermes_triage_demo.sh preflight` first. The check that most\noften decides whether a rehearsal holds its timings is the agent pool: the\nKubernetes cloud caps concurrent agent pods at `containerCapStr: \"5\"`, and when\nreal CI has taken all five the demo build sits in the queue reporting *\"All\nnodes of label ... are offline\"* with no other symptom. Observed 2026-08-06: a\nrun armed at 02:13:42 did not start seeding until 02:19:37, close to six\nminutes of dead air, purely because five other builds held the pool. Wait for a\nfree slot before starting, or quiesce CI.\n\n## Expected timings (measured live 2026-08-06, Ariadne 0.1.0-402)\n\nTime the demo from the moment the build goes **red**, not from arming. Arming\nonly queues a Jenkins build, and that leg is at the mercy of the agent pool.\n\n- Red \u2192 fixture patched back to healthy: **25s** (\u226460 s detection tick, ~16 s\n Hermes diagnosis, then a single in-process Kubernetes API call).\n- Red \u2192 rebuild triggered: **38s**.\n- Red \u2192 rebuild green and incident resolved: **1m04s**.\n\nThe whole automated leg is just over a minute, and there is no silent phase\nlonger than the detection tick. Budgeted Hermes timeout is 420 s; observed\ndiagnosis runs are 15\u201321 s.\n\nArming \u2192 red was **6m21s** on this run, but 5m35s of that was queue wait\nbehind a saturated agent pool. On an idle pool expect roughly 1m45s.\n\nThe earlier 2026-08-05 figures (1m15s red\u2192repaired, 4m00s total) were measured\nwhen the repair spawned its own Kubernetes Job. Converting the repair to an\nin-process call removed a whole pod launch from the critical path.\n\n## Watching it live\n\n- Jenkins: `https://ci.bstein.dev/job/hermes-triage-demo/` (red build N,\n then green build N+1 with `SEED_FAILURE=false`).\n- Ariadne incident state:\n `GET http://ariadne.maintenance/api/internal/audit/events` (in-cluster) or\n `/api/admin/audit/events` (Keycloak JWT) \u2014 event types\n `hermes_autotriage_incident`, `hermes_autotriage_diagnosis`,\n `hermes_autotriage_action`. Status flow:\n `detected \u2192 diagnosed \u2192 repairing \u2192 awaiting_rebuild \u2192 resolved`.\n- Metrics (VictoriaMetrics / Grafana Explore):\n `ariadne_hermes_triage_incident{jenkins_job=\"hermes-triage-demo\"}`,\n `ariadne_hermes_triage_action_total`,\n `ariadne_hermes_triage_duration_seconds`.\n- Repair evidence: `kubectl -n hermes-triage-demo get jobs` shows\n `hermes-demo-test-` (failed) and `hermes-demo-repair-` (succeeded);\n both TTL-clean after 1 h.\n- Hermes side: the run appears in the dashboard at\n `https://worker.bstein.dev` (session/run history).\n- Escalation path: alert `HermesTriageHumanRequired` in vmalert\n (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`)\n fires to Alertmanager for any `human_required` incident. Gauges are\n republished every tick from stored incident state (restart-safe), and\n the alert self-clears once a newer build of the same job is green.\n Note: Alertmanager's default receiver is currently null \u2014 the alert is\n visible in vmalert/Alertmanager/Grafana but pushes no notification.\n\n## Demonstrating safe escalation (second path)\n\nAny failure that does not match the demo-fixture signature \u2014 or any\ninvalid/low-confidence/unknown-action Hermes response \u2014 ends as\n`status=\"human_required\"` with **no mutation**. The simplest live demo:\ntemporarily set `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` (see below)\nand arm the failure; Ariadne diagnoses fully but executes nothing, and the\nalert fires instead.\n\n## Kill switch and rollback\n\n- Instant behavioral off-switch (Flux-managed, in\n `services/maintenance/apps/ariadne-deployment.yaml`):\n `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` \u2192 diagnose-only.\n `ARIADNE_HERMES_AUTOTRIAGE_ENABLED=false` \u2192 fully off.\n- The automatic loop can only ever: create Jobs named\n `hermes-demo-repair-*` in namespace `hermes-triage-demo`, and trigger\n rebuilds of allowlisted jobs (`ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST`,\n currently `hermes-triage-demo` only). One action + one rebuild per\n incident, ever (storage-backed idempotency).\n- Full teardown: remove `hermes-triage-demo` from\n `clusters/atlas/flux-system/applications/kustomization.yaml` (Flux prunes\n the namespace) and delete the `pipelineJob('hermes-triage-demo')` block\n from `services/jenkins/configmap-jcasc.yaml`.\n\n## Credentials\n\n- Ariadne \u2192 Hermes: `Authorization: Bearer` key from\n `kv/atlas/hermes/triage-api`. Vault Agent injects it into both workloads at\n runtime; it is not stored in either persistent Hermes state or a manually\n maintained Kubernetes Secret. Rotate the Vault value and roll both tracked\n deployments through Flux.\n- Ariadne \u2192 Jenkins: existing `JENKINS_API_USER/TOKEN` from Vault\n (`atlas/maintenance/ariadne-db`)." }, { "path": "runbooks/kb-authoring.md", diff --git a/services/comms/knowledge/diagrams/atlas-http.mmd b/services/comms/knowledge/diagrams/atlas-http.mmd index 5c0c9be7..28e35b5e 100644 --- a/services/comms/knowledge/diagrams/atlas-http.mmd +++ b/services/comms/knowledge/diagrams/atlas-http.mmd @@ -194,6 +194,8 @@ flowchart LR host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"] svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf + host_worker_bstein_dev["worker.bstein.dev"] + host_worker_bstein_dev --> svc_hermes_oauth2_proxy_hermes_agent subgraph bstein_dev_home[bstein-dev-home] svc_bstein_dev_home_bstein_dev_home_frontend diff --git a/services/comms/knowledge/runbooks/hermes-triage-demo.md b/services/comms/knowledge/runbooks/hermes-triage-demo.md index 1a5a64e6..c73d92e9 100644 --- a/services/comms/knowledge/runbooks/hermes-triage-demo.md +++ b/services/comms/knowledge/runbooks/hermes-triage-demo.md @@ -99,7 +99,7 @@ in-process call removed a whole pod launch from the critical path. `hermes-demo-test-` (failed) and `hermes-demo-repair-` (succeeded); both TTL-clean after 1 h. - Hermes side: the run appears in the dashboard at - `https://agent.hermes.bstein.dev` (session/run history). + `https://worker.bstein.dev` (session/run history). - Escalation path: alert `HermesTriageHumanRequired` in vmalert (`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`) fires to Alertmanager for any `human_required` incident. Gauges are diff --git a/services/hermes/NOTES.md b/services/hermes/NOTES.md index ef10bb65..0cbb31ad 100644 --- a/services/hermes/NOTES.md +++ b/services/hermes/NOTES.md @@ -5,7 +5,7 @@ This is the mental model and demonstration script for the operator instance at consumer instance at `chat.bstein.dev` is intentionally separate and is not the place to perform infrastructure triage. -`agent.hermes.bstein.dev` is the owner-only engineering control plane. Its root +`worker.bstein.dev` is the owner-only engineering control plane. Its root serves the stock Hermes dashboard, including embedded chat/TUI and the native sessions, files, models, logs, Kanban, skills, plugins, MCP, profiles, and configuration surfaces. `/terminal/` remains the raw full-screen TUI. The @@ -143,7 +143,7 @@ off. The final TypeScript build, Python compile, and source assertions remain mandatory, so an omitted or drifted replay fails before any image can pass release verification. -From agent.hermes, trigger that one fixed job with: +From worker.bstein.dev, trigger that one fixed job with: ```sh jenkins_image_build_trigger.py '' diff --git a/services/hermes/agent-certificate.yaml b/services/hermes/agent-certificate.yaml index 0733bc83..b444ca63 100644 --- a/services/hermes/agent-certificate.yaml +++ b/services/hermes/agent-certificate.yaml @@ -10,6 +10,9 @@ spec: kind: ClusterIssuer name: letsencrypt dnsNames: + # worker.bstein.dev is the canonical owner-agent host. agent.hermes.bstein.dev + # stays on the certificate until the old host is retired in a separate change. - agent.hermes.bstein.dev - chat.bstein.dev - triage.bstein.dev + - worker.bstein.dev diff --git a/services/hermes/agent-configmap.yaml b/services/hermes/agent-configmap.yaml index bcec67ac..a7bee49d 100644 --- a/services/hermes/agent-configmap.yaml +++ b/services/hermes/agent-configmap.yaml @@ -142,7 +142,7 @@ data: - "*git clean -f*" dashboard: - public_url: https://agent.hermes.bstein.dev + public_url: https://worker.bstein.dev display: compact: true @@ -174,7 +174,7 @@ data: backup_keep: 5 non_interactive_local_changes: stash SOUL.md: | - You are Brad's private Hermes coordinator at agent.hermes.bstein.dev. Turn + You are Brad's private Hermes coordinator at worker.bstein.dev. Turn objectives into organized, reviewable delivery without making Brad manage model names, terminals, or provider capacity. @@ -408,7 +408,7 @@ data: START-HERE.md: | # Agent Hermes - The authenticated root of agent.hermes.bstein.dev opens Hermes' stock + The authenticated root of worker.bstein.dev opens Hermes' stock dashboard with embedded chat/TUI, sessions, files, models, logs, Kanban, skills, plugins, MCP, profiles, and configuration. `/terminal/` opens the raw full-screen Hermes TUI. Give Hermes diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index ddc47895..76484921 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -624,14 +624,15 @@ spec: - {name: HERMES_DASHBOARD_HOST, value: 127.0.0.1} - {name: HERMES_DASHBOARD_PORT, value: "9119"} - {name: HERMES_DASHBOARD_FILES_ROOT, value: /opt/data/workspace} - - {name: HERMES_DASHBOARD_PUBLIC_URL, value: https://agent.hermes.bstein.dev} + - {name: HERMES_DASHBOARD_PUBLIC_URL, value: https://worker.bstein.dev} - {name: HERMES_TUI_AGENT_INIT_TIMEOUT_S, value: "180"} - {name: HERMES_STREAM_STALE_TIMEOUT, value: "600"} - {name: HERMES_API_CALL_STALE_TIMEOUT, value: "600"} - {name: API_SERVER_ENABLED, value: "true"} - {name: API_SERVER_HOST, value: 0.0.0.0} - {name: API_SERVER_PORT, value: "8642"} - - {name: API_SERVER_CORS_ORIGINS, value: https://agent.hermes.bstein.dev} + # Comma-separated; the legacy origin stays until agent.hermes.bstein.dev is retired. + - {name: API_SERVER_CORS_ORIGINS, value: "https://worker.bstein.dev,https://agent.hermes.bstein.dev"} - {name: HERMES_MEDIA_DELIVERY_STRICT, value: "1"} - {name: HERMES_MEDIA_ALLOW_DIRS, value: /opt/data/workspace} - {name: HERMES_AUTO_ROUTER_PROFILE, value: agent} @@ -694,7 +695,11 @@ spec: args: - --provider=oidc - --config=/vault/secrets/oidc-config - - --redirect-url=https://agent.hermes.bstein.dev/oauth2/callback + # Single canonical callback. The __Host- cookie prefix below binds both + # the session and the CSRF cookie to one host, so a fresh login must be + # started on worker.bstein.dev; agent.hermes.bstein.dev keeps serving + # already-authenticated sessions until it is retired separately. + - --redirect-url=https://worker.bstein.dev/oauth2/callback - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas - --user-id-claim=sub - --oidc-groups-claim=groups diff --git a/services/hermes/agent-ingress.yaml b/services/hermes/agent-ingress.yaml index 8d636d20..53bd4008 100644 --- a/services/hermes/agent-ingress.yaml +++ b/services/hermes/agent-ingress.yaml @@ -6,8 +6,10 @@ metadata: namespace: hermes spec: redirectRegex: - regex: ^https://agent\.hermes\.bstein\.dev/terminal$ - replacement: https://agent.hermes.bstein.dev/terminal/ + # Host-preserving so the trailing-slash fixup works on both the canonical + # worker.bstein.dev and the legacy agent.hermes.bstein.dev host. + regex: ^https://(worker\.bstein\.dev|agent\.hermes\.bstein\.dev)/terminal$ + replacement: https://${1}/terminal/ permanent: false --- apiVersion: traefik.io/v1alpha1 @@ -33,9 +35,21 @@ spec: ingressClassName: traefik tls: - hosts: + - worker.bstein.dev - agent.hermes.bstein.dev secretName: hermes-sites-tls rules: + - host: worker.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy-hermes-agent + port: + name: http + # Legacy host, served unchanged until it is retired in a separate change. - host: agent.hermes.bstein.dev http: paths: @@ -60,9 +74,21 @@ spec: ingressClassName: traefik tls: - hosts: + - worker.bstein.dev - agent.hermes.bstein.dev secretName: hermes-sites-tls rules: + - host: worker.bstein.dev + http: + paths: + - path: /terminal + pathType: Prefix + backend: + service: + name: oauth2-proxy-hermes-agent + port: + name: http + # Legacy host, served unchanged until it is retired in a separate change. - host: agent.hermes.bstein.dev http: paths: @@ -90,6 +116,7 @@ spec: - agent.hermes.bstein.dev - chat.bstein.dev - triage.bstein.dev + - worker.bstein.dev secretName: hermes-sites-tls rules: - host: chat.bstein.dev diff --git a/services/hermes/configmap.yaml b/services/hermes/configmap.yaml index 722ce75c..6fcaba70 100644 --- a/services/hermes/configmap.yaml +++ b/services/hermes/configmap.yaml @@ -113,7 +113,7 @@ data: This is the dedicated triage appliance at triage.bstein.dev. Keep automated Ariadne intake and testing conversations here. Project delivery - and coding orchestration belong to agent.hermes.bstein.dev; general user + and coding orchestration belong to worker.bstein.dev; general user chat belongs to chat.bstein.dev. Start in AUTO routing with a careful, intelligence-biased posture. Every diff --git a/services/hermes/skills/master-hermes-on-atlas/references/architecture.md b/services/hermes/skills/master-hermes-on-atlas/references/architecture.md index 17636d75..21ed350c 100644 --- a/services/hermes/skills/master-hermes-on-atlas/references/architecture.md +++ b/services/hermes/skills/master-hermes-on-atlas/references/architecture.md @@ -8,7 +8,7 @@ asserting health, placement, ownership, or current model availability. | Surface | Purpose | Identity boundary | State and permissions | | --- | --- | --- | --- | | `triage.bstein.dev` | Brad's automated testing triage | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace, its own PVC and service account; read-only cluster triage plus approved internal evidence endpoints | -| `agent.hermes.bstein.dev` | Brad's project coordinator | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace and separate PVC; native Hermes delegates bounded work while Herdr preserves real Codex and Claude Code CLI sessions when needed | +| `worker.bstein.dev` | Brad's project coordinator | Keycloak plus an outer oauth2-proxy exact-email allow-list for `brad@bstein.dev` | `hermes` namespace and separate PVC; native Hermes delegates bounded work while Herdr preserves real Codex and Claude Code CLI sessions when needed | | `chat.bstein.dev` | Private consumer chat and research through Hermes WebUI or a linked Telegram DM | Keycloak login plus one-time Telegram account link | One Hermes process and PVC per assigned Keycloak subject; no Kubernetes RBAC, terminal, or private-service access | The instances do not share conversation state, credentials, profiles, skills diff --git a/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml b/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml index 7eae3a1e..b2020e8e 100644 --- a/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml +++ b/services/keycloak/bootstrap-jobs/hermes-access-oidc-client-job.yaml @@ -3,7 +3,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: hermes-access-oidc-client-ensure-10 + name: hermes-access-oidc-client-ensure-11 namespace: sso spec: backoffLimit: 3 diff --git a/services/keycloak/scripts/hermes_access_oidc_ensure.sh b/services/keycloak/scripts/hermes_access_oidc_ensure.sh index 7fcc4569..9f009be7 100755 --- a/services/keycloak/scripts/hermes_access_oidc_ensure.sh +++ b/services/keycloak/scripts/hermes_access_oidc_ensure.sh @@ -80,15 +80,25 @@ if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then exit 1 fi +# Usage: ensure_proxy_client [legacy_url...] +# Any legacy URLs are registered alongside the canonical one so that a host +# rename can be rolled back by reverting the proxy's --redirect-url alone, +# without re-running this job. rootUrl always tracks the canonical host. ensure_proxy_client() { client_id="$1" public_url="$2" vault_path="$3" - payload="$(jq -nc \ + shift 3 + origin_lines="${public_url}" + for legacy_url in "$@"; do + origin_lines="${origin_lines} +${legacy_url}" + done + payload="$(printf '%s\n' "${origin_lines}" | jq -Rsc \ --arg client_id "${client_id}" \ - --arg redirect_uri "${public_url}/oauth2/callback" \ --arg web_origin "${public_url}" \ - '{ + '(split("\n") | map(select(length > 0))) as $origins + | { clientId:$client_id, name:$client_id, enabled:true, @@ -98,13 +108,13 @@ ensure_proxy_client() { implicitFlowEnabled:false, directAccessGrantsEnabled:false, serviceAccountsEnabled:false, - redirectUris:[$redirect_uri], - webOrigins:[$web_origin], + redirectUris:($origins | map(. + "/oauth2/callback")), + webOrigins:$origins, rootUrl:$web_origin, baseUrl:"/", attributes:{ "pkce.code.challenge.method":"S256", - "post.logout.redirect.uris":$web_origin, + "post.logout.redirect.uris":($origins | join("##")), "access.token.lifespan":"1200" } }')" @@ -328,7 +338,7 @@ ensure_telegram_config() { ensure_hermes_owner ensure_proxy_client "hermes-chat-proxy" "https://chat.bstein.dev" "hermes/chat-oidc" -ensure_proxy_client "hermes-agent-proxy" "https://agent.hermes.bstein.dev" "hermes/agent-oidc" +ensure_proxy_client "hermes-agent-proxy" "https://worker.bstein.dev" "hermes/agent-oidc" "https://agent.hermes.bstein.dev" ensure_proxy_client "hermes-triage-proxy" "https://triage.bstein.dev" "hermes/triage-oidc" ensure_service_account_client "hermes-automation" "hermes/developer-keycloak" ensure_telegram_config diff --git a/services/quality/zap-baseline-configmap.yaml b/services/quality/zap-baseline-configmap.yaml index c659edbb..0a01c16d 100644 --- a/services/quality/zap-baseline-configmap.yaml +++ b/services/quality/zap-baseline-configmap.yaml @@ -26,7 +26,7 @@ data: https://budget.bstein.dev https://money.bstein.dev https://health.bstein.dev - https://agent.hermes.bstein.dev + https://worker.bstein.dev https://chat.bstein.dev https://triage.bstein.dev https://cassandra.bstein.dev diff --git a/testing/tests/test_hermes_agent_runtime_patches.py b/testing/tests/test_hermes_agent_runtime_patches.py index b28ce798..a7ad1bad 100644 --- a/testing/tests/test_hermes_agent_runtime_patches.py +++ b/testing/tests/test_hermes_agent_runtime_patches.py @@ -63,7 +63,7 @@ def test_owner_agent_installs_the_pinned_operator_toolchain(): assert 'chmod 0644 "${profile_file}"' in init_command -def test_owner_agent_uses_only_the_canonical_hostname(): +def test_owner_agent_uses_the_canonical_hostname(): paths = [ HERMES / "agent-configmap.yaml", HERMES / "agent-deployment.yaml", @@ -73,7 +73,40 @@ def test_owner_agent_uses_only_the_canonical_hostname(): for path in paths: content = path.read_text() assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content + assert "worker.bstein.dev" in content + + +def test_legacy_owner_host_stays_served_until_it_is_retired_separately(): + """The rename must not cut off the host the operator reaches Hermes on. + + ``agent.hermes.bstein.dev`` keeps resolving, routing and validating TLS + until it is retired in its own change, so rolling the rename back never + needs more than reverting the oauth2-proxy callback. + """ + legacy = "agent.hermes.bstein.dev" + canonical = "worker.bstein.dev" + + coredns = ( + Path(__file__).parents[2] / "infrastructure/core/coredns-custom.yaml" + ).read_text() + assert f"192.168.22.9 {legacy}" in coredns + assert f"192.168.22.9 {canonical}" in coredns + + certificate = yaml.safe_load((HERMES / "agent-certificate.yaml").read_text()) + assert {legacy, canonical} <= set(certificate["spec"]["dnsNames"]) + + ingresses = [ + doc + for doc in yaml.safe_load_all((HERMES / "agent-ingress.yaml").read_text()) + if doc + and doc.get("kind") == "Ingress" + and doc["metadata"]["name"].startswith("hermes-agent-") + ] + assert len(ingresses) == 2 + for ingress in ingresses: + assert {legacy, canonical} <= {rule["host"] for rule in ingress["spec"]["rules"]} + for tls in ingress["spec"]["tls"]: + assert {legacy, canonical} <= set(tls["hosts"]) def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): diff --git a/testing/tests/test_hermes_cli_lanes_toolchain.py b/testing/tests/test_hermes_cli_lanes_toolchain.py index 2ed232cb..d7741a24 100644 --- a/testing/tests/test_hermes_cli_lanes_toolchain.py +++ b/testing/tests/test_hermes_cli_lanes_toolchain.py @@ -122,7 +122,7 @@ def test_owner_agent_installs_the_pinned_operator_toolchain(): assert 'chmod 0644 "${profile_file}"' in init_command -def test_owner_agent_uses_only_the_canonical_hostname(): +def test_owner_agent_uses_the_canonical_hostname(): paths = [ HERMES / "agent-configmap.yaml", HERMES / "agent-deployment.yaml", @@ -132,7 +132,7 @@ def test_owner_agent_uses_only_the_canonical_hostname(): for path in paths: content = path.read_text() assert "agent.bstein.dev" not in content - assert "agent.hermes.bstein.dev" in content + assert "worker.bstein.dev" in content def test_agent_reconnect_retains_complete_history_and_long_tool_budget(): -- 2.47.2