diff --git a/scripts/render/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py index b9be8a99..e0986bdc 100644 --- a/scripts/render/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -5181,10 +5181,24 @@ def build_ai_dashboard(): ], } unavailable_mapping = [ - {"type": "value", "options": {"-1": {"text": "unavailable", "color": "gray"}}} + { + "type": "value", + "options": { + "-2": {"text": "not exposed", "color": "blue"}, + "-1": {"text": "unavailable", "color": "gray"}, + }, + } ] - def quota_stat(panel_id, title, expr, grid, *, unit="percent"): + def quota_stat( + panel_id, + title, + expr, + grid, + *, + unit="percent", + authenticated_expr=None, + ): thresholds = remaining_thresholds if unit != "percent": thresholds = { @@ -5194,6 +5208,8 @@ def build_ai_dashboard(): {"color": "blue", "value": 0}, ], } + if authenticated_expr: + expr = f"({expr}) or on() (({authenticated_expr}) * 0 - 2)" panel = stat_panel( panel_id, title, @@ -5203,13 +5219,16 @@ def build_ai_dashboard(): decimals=1, thresholds=thresholds, instant=True, - description="Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.", + description="Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy.", ) panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping return panel openai_ok = 'atlas_ai_quota_fetch_success{provider="openai"} == 1' anthropic_ok = 'atlas_ai_quota_fetch_success{provider="anthropic"} == 1' + anthropic_authenticated = ( + 'atlas_ai_provider_authenticated{provider="anthropic"} == 1' + ) quota = "atlas_ai_quota_remaining_percent" reset = "atlas_ai_quota_reset_timestamp_seconds" panels = [ @@ -5230,17 +5249,19 @@ def build_ai_dashboard(): "Claude 5h Remaining", f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_ok})', {"h": 4, "w": 4, "x": 8, "y": 0}, + authenticated_expr=anthropic_authenticated, ), quota_stat( 4, "Claude 7d Remaining", f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_ok})', {"h": 4, "w": 4, "x": 12, "y": 0}, + authenticated_expr=anthropic_authenticated, ), stat_panel( 5, - "Quota Collectors Healthy", - "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "Provider Access Healthy", + "sum(atlas_ai_provider_authenticated) or on() vector(0)", {"h": 4, "w": 4, "x": 16, "y": 0}, instant=True, thresholds={ @@ -5252,7 +5273,7 @@ def build_ai_dashboard(): ], }, value_suffix=" / 2", - description="Successful latest quota fetches. Providers are polled independently every five minutes.", + description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.", ), stat_panel( 6, @@ -5285,6 +5306,7 @@ def build_ai_dashboard(): f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})', {"h": 4, "w": 4, "x": 4, "y": 4}, unit="s", + authenticated_expr=anthropic_authenticated, ), quota_stat( 9, @@ -5292,6 +5314,7 @@ def build_ai_dashboard(): f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})', {"h": 4, "w": 4, "x": 8, "y": 4}, unit="s", + authenticated_expr=anthropic_authenticated, ), quota_stat( 10, diff --git a/services/hermes/agent-deployment.yaml b/services/hermes/agent-deployment.yaml index 4282211d..7ab8cf46 100644 --- a/services/hermes/agent-deployment.yaml +++ b/services/hermes/agent-deployment.yaml @@ -25,12 +25,13 @@ spec: ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available - ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3" + ai.bstein.dev/config-rev: "20260822-claude-oauth-self-heal-v1" prometheus.io/scrape: "true" prometheus.io/path: /metrics prometheus.io/port: "9010" vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/role: hermes-agent + vault.hashicorp.com/agent-inject-containers: stage-runtime-access vault.hashicorp.com/agent-inject-secret-agent-api-key: kv/data/atlas/hermes/agent-tokens vault.hashicorp.com/agent-inject-template-agent-api-key: | {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} @@ -41,6 +42,11 @@ spec: {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} {{ .Data.data.claude_credentials_json }} {{- end }} + vault.hashicorp.com/agent-inject-secret-claude-oauth-token: kv/data/atlas/hermes/agent-tokens + vault.hashicorp.com/agent-inject-template-claude-oauth-token: | + {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} + {{ .Data.data.claude_oauth_token }} + {{- end }} vault.hashicorp.com/agent-inject-secret-codex-auth: kv/data/atlas/hermes/agent-tokens vault.hashicorp.com/agent-inject-template-codex-auth: | {{- with secret "kv/data/atlas/hermes/agent-tokens" -}} @@ -255,8 +261,10 @@ spec: seccompProfile: type: RuntimeDefault volumeMounts: + - {name: claude-oauth-access, mountPath: /claude-oauth-access} - {name: home, mountPath: /opt/data} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} + - {name: oauth2-config, mountPath: /oauth2-config} - {name: runtime-access, mountPath: /runtime-access} resources: requests: {cpu: 25m, memory: 32Mi} @@ -521,6 +529,7 @@ spec: seccompProfile: type: RuntimeDefault volumeMounts: + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} @@ -563,6 +572,7 @@ spec: seccompProfile: type: RuntimeDefault volumeMounts: + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} @@ -615,6 +625,7 @@ spec: - {name: HOME, value: /opt/data/home} - {name: CODEX_HOME, value: /runtime-access/codex} - {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude} + - {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude} - {name: KUBECONFIG, value: /opt/data/home/.kube/config} - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin} - {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright} @@ -641,6 +652,7 @@ spec: - {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json} - {name: HERMES_IMAGE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key} volumeMounts: + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} @@ -694,7 +706,7 @@ spec: imagePullPolicy: IfNotPresent args: - --provider=oidc - - --config=/vault/secrets/oidc-config + - --config=/oauth2-config/oidc-config # Single canonical callback. The __Host- cookie prefix below binds both the # session and the CSRF cookie to one host, so a fresh login must start on # worker.bstein.dev; agent.hermes.bstein.dev keeps serving already @@ -755,6 +767,7 @@ spec: limits: {cpu: 250m, memory: 256Mi} volumeMounts: - {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true} + - {name: oauth2-config, mountPath: /oauth2-config, readOnly: true} - {name: oauth-templates, mountPath: /etc/oauth2-proxy/templates, readOnly: true} - {name: oauth-tmp, mountPath: /tmp} - name: terminal @@ -800,6 +813,7 @@ spec: - {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json} - {name: CODEX_HOME, value: /runtime-access/codex} - {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude} + - {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude} - {name: KUBECONFIG, value: /opt/data/home/.kube/config} - {name: PYTHONPATH, value: /opt/hermes} - {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin} @@ -814,6 +828,7 @@ spec: volumeMounts: - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: codex-runtime-patch, mountPath: /opt/hermes/hermes_cli/runtime_provider.py, subPath: runtime_provider.py} @@ -877,6 +892,7 @@ spec: - {name: HOME, value: /opt/data/home} - {name: CODEX_HOME, value: /runtime-access/codex} - {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude} + - {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude} - {name: KUBECONFIG, value: /opt/data/home/.kube/config} - {name: PYTHONPATH, value: /opt/hermes} - {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"} @@ -898,6 +914,7 @@ spec: volumeMounts: - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true} @@ -940,6 +957,7 @@ spec: volumeMounts: - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: routing-catalog, mountPath: /routing-catalog} @@ -998,8 +1016,10 @@ spec: type: RuntimeDefault volumeMounts: - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - - {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude, readOnly: true} - - {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex, readOnly: true} + # Reconcile provider rotations to Vault and restore runtime files if a + # client leaves an incomplete credential behind after refresh. + - {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude} + - {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex} resources: requests: {cpu: 10m, memory: 32Mi} limits: {cpu: 100m, memory: 128Mi} @@ -1016,6 +1036,7 @@ spec: - {name: CODEX_HOME, value: /runtime-access/codex} - {name: ATLAS_AI_CODEX_BIN, value: /opt/data/tools/bin/codex} - {name: ATLAS_AI_CLAUDE_CREDENTIALS, value: /runtime-access/claude/.credentials.json} + - {name: ATLAS_AI_PROVIDER_HEALTH_ROOT, value: /provider-health} - {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"} - {name: ATLAS_AI_USAGE_PORT, value: "9010"} readinessProbe: @@ -1038,6 +1059,7 @@ spec: type: RuntimeDefault volumeMounts: - {name: home, mountPath: /opt/data/tools, subPath: tools, readOnly: true} + - {name: home, mountPath: /provider-health, subPath: provider-health, readOnly: true} - {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude} - {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} @@ -1186,6 +1208,7 @@ spec: volumeMounts: - {name: home, mountPath: /opt/data} - {name: runtime-access, mountPath: /runtime-access} + - {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true} - {name: coordinator, mountPath: /opt/coordinator, readOnly: true} - {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py} - {name: tmp, mountPath: /tmp} @@ -1201,6 +1224,10 @@ spec: emptyDir: medium: Memory sizeLimit: 128Mi + - name: claude-oauth-access + emptyDir: + medium: Memory + sizeLimit: 1Mi - name: routing-catalog persistentVolumeClaim: claimName: hermes-routing-catalog @@ -1257,6 +1284,10 @@ spec: - name: oauth-tmp emptyDir: sizeLimit: 64Mi + - name: oauth2-config + emptyDir: + medium: Memory + sizeLimit: 1Mi - name: allowlist configMap: name: hermes-owner-allowlist diff --git a/services/hermes/scripts/ai_usage_exporter.py b/services/hermes/scripts/ai_usage_exporter.py index b0db51fb..d1729087 100644 --- a/services/hermes/scripts/ai_usage_exporter.py +++ b/services/hermes/scripts/ai_usage_exporter.py @@ -23,6 +23,9 @@ CLAUDE_CREDENTIALS = Path( "/runtime-access/claude/.credentials.json", ) ) +PROVIDER_HEALTH_ROOT = Path( + os.environ.get("ATLAS_AI_PROVIDER_HEALTH_ROOT", "/provider-health") +) CLAUDE_USAGE_URL = os.environ.get( "ATLAS_AI_CLAUDE_USAGE_URL", "https://api.anthropic.com/api/oauth/usage", @@ -37,6 +40,7 @@ METRIC_HELP = { "atlas_ai_account_tokens": "First-party account token usage for a fixed period.", "atlas_ai_account_usage_summary": "First-party account usage summary values.", "atlas_ai_extra_usage_enabled": "Whether metered extra usage is enabled for the account.", + "atlas_ai_provider_authenticated": "Whether the first-party provider access boundary is authenticated.", "atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.", "atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.", "atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.", @@ -64,6 +68,7 @@ class ProviderState: """Latest safe samples and fetch health for one provider.""" samples: list[Sample] = field(default_factory=list) + authenticated: bool = False fetch_success: bool = False last_attempt: float = 0 last_success: float = 0 @@ -254,6 +259,18 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]: query_codex = codex_query.query_codex +def _provider_authenticated(provider: str) -> bool: + """Read one fresh non-secret broker authentication snapshot.""" + path = PROVIDER_HEALTH_ROOT / ("codex.json" if provider == "openai" else "claude.json") + try: + if time.time() - path.stat().st_mtime > 10 * 60: + return False + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, TypeError, ValueError, json.JSONDecodeError): + return False + return isinstance(document, dict) and document.get("authenticated") is True + + def query_claude() -> dict[str, Any]: """Read Claude account quota with the runtime OAuth token held only in memory.""" document = json.loads(CLAUDE_CREDENTIALS.read_text(encoding="utf-8")) @@ -311,6 +328,11 @@ class Collector: started = time.time() monotonic_started = time.monotonic() try: + if provider not in self._providers: + raise ValueError("unknown provider") + authenticated = _provider_authenticated(provider) + with self._lock: + self._providers[provider].authenticated = authenticated if provider == "openai": samples = parse_codex_payloads(*query_codex()) elif provider == "anthropic": @@ -343,12 +365,19 @@ class Collector: provider: ProviderState(**vars(state)) for provider, state in self._providers.items() } + for provider, state in states.items(): + state.authenticated = _provider_authenticated(provider) samples: list[Sample] = [] for provider, state in states.items(): labels = {"provider": provider} samples.extend(state.samples) samples.extend( ( + Sample( + "atlas_ai_provider_authenticated", + labels, + float(state.authenticated), + ), Sample( "atlas_ai_quota_fetch_success", labels, diff --git a/services/hermes/scripts/claude b/services/hermes/scripts/claude index 33548d79..4803cd14 100755 --- a/services/hermes/scripts/claude +++ b/services/hermes/scripts/claude @@ -1,9 +1,17 @@ #!/bin/sh # Run Claude Code unattended inside the externally isolated Hermes agent pod. set -eu +umask 077 + +config_dir="${CLAUDE_CONFIG_DIR:-/runtime-access/claude}" +oauth_token_file="${CLAUDE_CODE_OAUTH_TOKEN_FILE:-/claude-oauth-access/token}" +if [ -s "$oauth_token_file" ]; then + CLAUDE_CODE_OAUTH_TOKEN="$(tr -d '\r\n' < "$oauth_token_file")" + export CLAUDE_CODE_OAUTH_TOKEN +fi exec /opt/data/tools/bin/claude \ --dangerously-skip-permissions \ --autocompact auto \ - --settings "${CLAUDE_CONFIG_DIR:-/runtime-access/claude}/settings.json" \ + --settings "$config_dir/settings.json" \ "$@" diff --git a/services/hermes/scripts/cli_lane_config.py b/services/hermes/scripts/cli_lane_config.py index 895aa5cc..4ff598fb 100644 --- a/services/hermes/scripts/cli_lane_config.py +++ b/services/hermes/scripts/cli_lane_config.py @@ -23,7 +23,7 @@ SWITCHYARD_URL = os.environ.get( ) STATE_ROOT = DATA_ROOT / "cli-lanes" CODEX_BIN = DATA_ROOT / "tools/bin/codex" -CLAUDE_BIN = DATA_ROOT / "tools/bin/claude" +CLAUDE_BIN = Path(os.environ.get("HERMES_CLAUDE_BIN", DATA_ROOT / "tools/bin/claude")) CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json" RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json" EFFORTS = ("low", "medium", "high", "xhigh") diff --git a/services/hermes/scripts/stage_runtime_access.py b/services/hermes/scripts/stage_runtime_access.py index fa953a70..627e88db 100644 --- a/services/hermes/scripts/stage_runtime_access.py +++ b/services/hermes/scripts/stage_runtime_access.py @@ -15,6 +15,8 @@ from pathlib import Path VAULT_ROOT = Path("/vault/secrets") RUNTIME_ROOT = Path("/runtime-access") +CLAUDE_OAUTH_ACCESS_ROOT = Path("/claude-oauth-access") +OAUTH2_CONFIG_ROOT = Path("/oauth2-config") PERSISTENT_HOME = Path("/opt/data/home") WORKER_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace")) PROVIDER_ACCESS_ROOT = Path( @@ -23,9 +25,13 @@ PROVIDER_ACCESS_ROOT = Path( POOL_ACCESS_ROOT = Path(os.environ.get("HERMES_POOL_ACCESS_ROOT", "/pool-access")) OWNER_UID = 10000 OWNER_GID = 10000 +OAUTH2_PROXY_UID = 65532 +OAUTH2_PROXY_GID = 65532 -def _owned_directory(path: Path) -> None: +def _owned_directory( + path: Path, *, owner_uid: int = OWNER_UID, owner_gid: int = OWNER_GID +) -> None: """Create one private directory owned by the unprivileged Hermes user.""" if path.is_symlink(): raise RuntimeError(f"private directory {path.name} must not be a symlink") @@ -33,7 +39,7 @@ def _owned_directory(path: Path) -> None: if path.is_symlink() or not path.is_dir(): raise RuntimeError(f"private directory {path.name} is invalid") path.chmod(0o700) - os.chown(path, OWNER_UID, OWNER_GID) + os.chown(path, owner_uid, owner_gid) def _read_secret(source_name: str) -> str: @@ -52,7 +58,13 @@ def _read_secret(source_name: str) -> str: return value -def _write_secret(destination: Path, value: str) -> None: +def _write_secret( + destination: Path, + value: str, + *, + owner_uid: int = OWNER_UID, + owner_gid: int = OWNER_GID, +) -> None: temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) descriptor = os.open(temporary, flags, 0o600) @@ -62,7 +74,7 @@ def _write_secret(destination: Path, value: str) -> None: remaining = remaining[os.write(descriptor, remaining) :] os.fsync(descriptor) os.fchmod(descriptor, 0o600) - os.fchown(descriptor, OWNER_UID, OWNER_GID) + os.fchown(descriptor, owner_uid, owner_gid) finally: os.close(descriptor) try: @@ -71,10 +83,21 @@ def _write_secret(destination: Path, value: str) -> None: temporary.unlink(missing_ok=True) -def _copy_secret(source_name: str, destination: Path) -> str: +def _copy_secret( + source_name: str, + destination: Path, + *, + owner_uid: int = OWNER_UID, + owner_gid: int = OWNER_GID, +) -> str: """Copy a non-empty Vault projection without logging its value.""" value = _read_secret(source_name) - _write_secret(destination, value) + _write_secret( + destination, + value, + owner_uid=owner_uid, + owner_gid=owner_gid, + ) return value @@ -180,6 +203,23 @@ def _stage_node_ssh_config() -> None: def stage_agent() -> None: """Stage the owner agent's complete runtime access set.""" + _owned_directory(PERSISTENT_HOME.parent / "provider-health") + _owned_directory(CLAUDE_OAUTH_ACCESS_ROOT) + _copy_secret( + "claude-oauth-token", + CLAUDE_OAUTH_ACCESS_ROOT / "token", + ) + _owned_directory( + OAUTH2_CONFIG_ROOT, + owner_uid=OAUTH2_PROXY_UID, + owner_gid=OAUTH2_PROXY_GID, + ) + _copy_secret( + "oidc-config", + OAUTH2_CONFIG_ROOT / "oidc-config", + owner_uid=OAUTH2_PROXY_UID, + owner_gid=OAUTH2_PROXY_GID, + ) for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"): _owned_directory(path) for name in ( diff --git a/services/hermes/scripts/sync_runtime_credentials.py b/services/hermes/scripts/sync_runtime_credentials.py index f8323f55..645e9428 100644 --- a/services/hermes/scripts/sync_runtime_credentials.py +++ b/services/hermes/scripts/sync_runtime_credentials.py @@ -1,11 +1,12 @@ #!/usr/bin/env python3 -"""Persist refreshed coding-client OAuth documents back into Vault.""" +"""Reconcile coding-client OAuth documents between runtime storage and Vault.""" from __future__ import annotations import json import os import time +import uuid from pathlib import Path from typing import Any from urllib.error import HTTPError, URLError @@ -87,8 +88,45 @@ def _credential_document(path: Path, required: tuple[str, ...]) -> str | None: return value -def sync_once(token: str) -> list[str]: - """CAS-update only credential fields whose runtime documents changed.""" +def _credential_value(value: object, required: tuple[str, ...]) -> str | None: + """Return a complete Vault credential document or reject invalid content.""" + if not isinstance(value, str) or not value.strip(): + return None + try: + document = json.loads(value) + except json.JSONDecodeError: + return None + current = document + for key in required: + if not isinstance(current, dict) or key not in current: + return None + current = current[key] + if not isinstance(current, str) or not current: + return None + return value.strip() + + +def _write_credential(path: Path, value: str) -> None: + """Atomically restore one private runtime credential from Vault.""" + temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(temporary, flags, 0o600) + try: + remaining = memoryview((value + "\n").encode("utf-8")) + while remaining: + remaining = remaining[os.write(descriptor, remaining) :] + os.fsync(descriptor) + os.fchmod(descriptor, 0o600) + finally: + os.close(descriptor) + try: + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def sync_once(token: str) -> tuple[list[str], list[str]]: + """Persist valid rotations and restore incomplete runtime credentials.""" response = _request("GET", SECRET_ENDPOINT, token=token) envelope = response.get("data") if not isinstance(envelope, dict): @@ -101,20 +139,26 @@ def sync_once(token: str) -> list[str]: if not isinstance(version, int) or version < 1: raise RuntimeError("Vault KV response has no version") updated = dict(data) - changed: list[str] = [] + persisted: list[str] = [] + restored: list[str] = [] for field, (path, required) in CREDENTIALS.items(): - value = _credential_document(path, required) - if value is not None and value != data.get(field): - updated[field] = value - changed.append(field) - if changed: + runtime_value = _credential_document(path, required) + vault_value = _credential_value(data.get(field), required) + if runtime_value is not None: + if runtime_value != vault_value: + updated[field] = runtime_value + persisted.append(field) + elif vault_value is not None: + _write_credential(path, vault_value) + restored.append(field) + if persisted: _request( "POST", SECRET_ENDPOINT, {"options": {"cas": version}, "data": updated}, token=token, ) - return changed + return persisted, restored def main() -> int: @@ -123,10 +167,17 @@ def main() -> int: while True: try: token = _login() - changed = sync_once(token) - if changed: + persisted, restored = sync_once(token) + if persisted: print( - "Persisted refreshed runtime credentials: " + ", ".join(changed), + "Persisted refreshed runtime credentials: " + + ", ".join(persisted), + flush=True, + ) + if restored: + print( + "Restored incomplete runtime credentials from Vault: " + + ", ".join(restored), flush=True, ) except (OSError, RuntimeError, HTTPError, URLError, ValueError) as error: diff --git a/services/monitoring/dashboards/atlas-ai.json b/services/monitoring/dashboards/atlas-ai.json index 60b6dc9c..bc9e1d12 100644 --- a/services/monitoring/dashboards/atlas-ai.json +++ b/services/monitoring/dashboards/atlas-ai.json @@ -34,6 +34,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -87,7 +91,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 2, @@ -119,6 +123,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -172,7 +180,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 3, @@ -190,7 +198,7 @@ }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -204,6 +212,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -257,7 +269,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 4, @@ -275,7 +287,7 @@ }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -289,6 +301,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -342,12 +358,12 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 5, "type": "stat", - "title": "Quota Collectors Healthy", + "title": "Provider Access Healthy", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -360,7 +376,7 @@ }, "targets": [ { - "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)", "refId": "A", "instant": true } @@ -409,7 +425,7 @@ }, "textMode": "value" }, - "description": "Successful latest quota fetches. Providers are polled independently every five minutes." + "description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." }, { "id": 6, @@ -511,6 +527,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -552,7 +572,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 8, @@ -570,7 +590,7 @@ }, "targets": [ { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -584,6 +604,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -625,7 +649,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 9, @@ -643,7 +667,7 @@ }, "targets": [ { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -657,6 +681,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -698,7 +726,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 10, @@ -730,6 +758,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -771,7 +803,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 11, @@ -803,6 +835,10 @@ { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -844,7 +880,7 @@ }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 12, diff --git a/services/monitoring/grafana-dashboard-ai.yaml b/services/monitoring/grafana-dashboard-ai.yaml index 5024ebea..f078bcf0 100644 --- a/services/monitoring/grafana-dashboard-ai.yaml +++ b/services/monitoring/grafana-dashboard-ai.yaml @@ -43,6 +43,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -96,7 +100,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 2, @@ -128,6 +132,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -181,7 +189,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 3, @@ -199,7 +207,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -213,6 +221,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -266,7 +278,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 4, @@ -284,7 +296,7 @@ data: }, "targets": [ { - "expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -298,6 +310,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -351,12 +367,12 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 5, "type": "stat", - "title": "Quota Collectors Healthy", + "title": "Provider Access Healthy", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -369,7 +385,7 @@ data: }, "targets": [ { - "expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)", + "expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)", "refId": "A", "instant": true } @@ -418,7 +434,7 @@ data: }, "textMode": "value" }, - "description": "Successful latest quota fetches. Providers are polled independently every five minutes." + "description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access." }, { "id": 6, @@ -520,6 +536,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -561,7 +581,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 8, @@ -579,7 +599,7 @@ data: }, "targets": [ { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -593,6 +613,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -634,7 +658,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 9, @@ -652,7 +676,7 @@ data: }, "targets": [ { - "expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)", + "expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)", "refId": "A", "instant": true } @@ -666,6 +690,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -707,7 +735,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 10, @@ -739,6 +767,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -780,7 +812,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 11, @@ -812,6 +844,10 @@ data: { "type": "value", "options": { + "-2": { + "text": "not exposed", + "color": "dark-blue" + }, "-1": { "text": "unavailable", "color": "gray" @@ -853,7 +889,7 @@ data: }, "textMode": "value" }, - "description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value." + "description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy." }, { "id": 12, diff --git a/testing/tests/test_atlas_ai_dashboard.py b/testing/tests/test_atlas_ai_dashboard.py index ed9dbeae..5d9e205e 100644 --- a/testing/tests/test_atlas_ai_dashboard.py +++ b/testing/tests/test_atlas_ai_dashboard.py @@ -37,11 +37,13 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics(): assert "Codex Weekly Remaining" in panels assert "Claude 5h Remaining" in panels assert "Claude 7d Remaining" in panels + assert "Provider Access Healthy" in panels assert "Provider Selections (Range)" in panels assert "Local Classifier Calls" in panels assert "Hermes Workload CPU (Attribution Proxy)" in panels assert "atlas_ai_quota_remaining_percent" in expressions assert "atlas_ai_quota_reset_timestamp_seconds" in expressions + assert "atlas_ai_provider_authenticated" in expressions assert "switchyard_decisions_total" in expressions assert "switchyard_cached_tokens_total" in expressions assert "switchyard_model_call_latency_ms_bucket" in expressions diff --git a/testing/tests/test_hermes_ai_usage_exporter_coverage.py b/testing/tests/test_hermes_ai_usage_exporter_coverage.py index affaf336..d9bf1336 100644 --- a/testing/tests/test_hermes_ai_usage_exporter_coverage.py +++ b/testing/tests/test_hermes_ai_usage_exporter_coverage.py @@ -133,6 +133,17 @@ def test_provider_queries_validate_credentials_and_response_shape( assert mod.query_codex is mod.codex_query.query_codex +def test_provider_authentication_uses_only_fresh_non_secret_health(tmp_path, monkeypatch): + mod = load_module() + monkeypatch.setattr(mod, "PROVIDER_HEALTH_ROOT", tmp_path) + health = tmp_path / "claude.json" + health.write_text('{"authenticated":true}\n', encoding="utf-8") + + assert mod._provider_authenticated("anthropic") + monkeypatch.setattr(mod.time, "time", lambda: health.stat().st_mtime + 601) + assert not mod._provider_authenticated("anthropic") + + def test_collector_refreshes_both_providers_and_contains_unknown_provider( monkeypatch, capsys ): diff --git a/testing/tests/test_hermes_runtime_access.py b/testing/tests/test_hermes_runtime_access.py index 68644623..8ec89cc4 100644 --- a/testing/tests/test_hermes_runtime_access.py +++ b/testing/tests/test_hermes_runtime_access.py @@ -87,6 +87,8 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp vault = tmp_path / "vault" runtime = tmp_path / "runtime" home = tmp_path / "home" + claude_oauth_access = tmp_path / "claude-oauth-access" + oauth2_config = tmp_path / "oauth2-config" vault.mkdir() (home / ".claude").mkdir(parents=True) (home / ".claude" / "backups").mkdir() @@ -98,6 +100,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp ) values = { "agent-api-key": "agent-key", + "claude-oauth-token": "long-lived-claude-token", "execution-pool-key": "e" * 64, "chat-relay-key": "relay-key", # gitea-token/username are present in Vault but the agent staging must @@ -109,6 +112,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp "node-ssh-private-key": "private-key", "node-ssh-config": "host-config", "node-ssh-known-hosts": "known-hosts", + "oidc-config": "client_id = \"hermes-agent\"", "claude-credentials": json.dumps( {"claudeAiOauth": {"refreshToken": "claude-refresh"}} ), @@ -119,14 +123,21 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp monkeypatch.setattr(stage, "VAULT_ROOT", vault) monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime) monkeypatch.setattr(stage, "PERSISTENT_HOME", home) + monkeypatch.setattr(stage, "CLAUDE_OAUTH_ACCESS_ROOT", claude_oauth_access) + monkeypatch.setattr(stage, "OAUTH2_CONFIG_ROOT", oauth2_config) monkeypatch.setattr(stage.os, "chown", lambda *_args: None) monkeypatch.setattr(stage.os, "fchown", lambda *_args: None) stage.stage_agent() assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600 + assert (claude_oauth_access / "token").read_text(encoding="utf-8").strip() == ( + "long-lived-claude-token" + ) + assert (claude_oauth_access / "token").stat().st_mode & 0o777 == 0o600 assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600 assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600 + assert (oauth2_config / "oidc-config").stat().st_mode & 0o777 == 0o600 assert (runtime / "claude/settings.json").is_symlink() assert (runtime / "codex/skills").is_symlink() assert not (runtime / "claude/backups").exists() @@ -289,14 +300,78 @@ def test_runtime_refresh_sync_uses_cas_and_preserves_other_fields( monkeypatch.setattr(sync, "_request", request) - assert sync.sync_once("vault-token") == [ - "claude_credentials_json", - "codex_auth_json", - ] + assert sync.sync_once("vault-token") == ( + ["claude_credentials_json", "codex_auth_json"], + [], + ) assert writes[0][1]["options"] == {"cas": 7} assert writes[0][1]["data"]["agent_api_key"] == "preserve-me" +def test_runtime_refresh_sync_restores_incomplete_credentials_from_vault( + tmp_path: Path, monkeypatch +): + sync = _load("sync_runtime_credentials") + claude = tmp_path / "claude.json" + claude.write_text( + json.dumps( + {"claudeAiOauth": {"accessToken": "", "refreshToken": ""}} + ), + encoding="utf-8", + ) + vault_value = json.dumps( + { + "claudeAiOauth": { + "accessToken": "vault-access", + "refreshToken": "vault-refresh", + } + } + ) + monkeypatch.setattr( + sync, + "CREDENTIALS", + { + "claude_credentials_json": ( + claude, + ("claudeAiOauth", "refreshToken"), + ) + }, + ) + + def request(method, _path, payload=None, *, token=""): + assert method == "GET" + assert payload is None + assert token == "vault-token" + return { + "data": { + "data": {"claude_credentials_json": vault_value}, + "metadata": {"version": 8}, + } + } + + monkeypatch.setattr(sync, "_request", request) + + assert sync.sync_once("vault-token") == ([], ["claude_credentials_json"]) + restored = json.loads(claude.read_text(encoding="utf-8")) + assert restored["claudeAiOauth"]["refreshToken"] == "vault-refresh" + assert claude.stat().st_mode & 0o077 == 0 + + +def test_claude_clients_use_the_vault_staged_long_lived_token(): + wrapper = (SCRIPTS / "claude").read_text(encoding="utf-8") + deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text()) + runner = next( + item + for item in deployment["spec"]["template"]["spec"]["containers"] + if item["name"] == "cli-lane-runner" + ) + environment = {item["name"]: item["value"] for item in runner["env"]} + + assert 'CLAUDE_CODE_OAUTH_TOKEN_FILE:-/claude-oauth-access/token' in wrapper + assert 'export CLAUDE_CODE_OAUTH_TOKEN' in wrapper + assert environment["HERMES_CLAUDE_BIN"] == "/opt/coordinator/claude" + + def test_subprocess_patches_strip_and_redact_runtime_credentials(tmp_path: Path): boundary = _load("patch_subprocess_secret_boundary") process = _load("patch_process_output_redaction") @@ -346,6 +421,38 @@ def test_manifests_never_seed_access_material_into_persistent_env(): assert f"upsert_env {key}" not in command annotations = agent["spec"]["template"]["metadata"]["annotations"] + assert annotations["vault.hashicorp.com/agent-inject-containers"] == ( + "stage-runtime-access" + ) + pod = agent["spec"]["template"]["spec"] + token_containers = { + item["name"] + for item in pod["containers"] + if any( + mount["name"] == "claude-oauth-access" + for mount in item.get("volumeMounts", []) + ) + } + assert token_containers == { + "hermes", + "terminal", + "cli-lane-runner", + "model-steward", + "claude-broker", + } + token_init_containers = { + item["name"] + for item in pod["initContainers"] + if any( + mount["name"] == "claude-oauth-access" + for mount in item.get("volumeMounts", []) + ) + } + assert token_init_containers == { + "stage-runtime-access", + "bootstrap-coordinator", + "configure-agent-clients", + } agent_runtime = next( item for item in agent["spec"]["template"]["spec"]["volumes"] @@ -359,6 +466,9 @@ def test_manifests_never_seed_access_material_into_persistent_env(): assert annotations["vault.hashicorp.com/agent-inject-secret-claude-credentials"] == ( "kv/data/atlas/hermes/agent-tokens" ) + assert annotations["vault.hashicorp.com/agent-inject-secret-claude-oauth-token"] == ( + "kv/data/atlas/hermes/agent-tokens" + ) assert annotations["vault.hashicorp.com/agent-inject-secret-codex-auth"] == ( "kv/data/atlas/hermes/agent-tokens" )