hermes: persist Claude subscription access
This commit is contained in:
jenkins
parent
9bf41dc0de
commit
bd50a7e4ad
@ -5181,10 +5181,24 @@ def build_ai_dashboard():
|
||||
],
|
||||
}
|
||||
unavailable_mapping = [
|
||||
{"type": "value", "options": {"-1": {"text": "unavailable", "color": "gray"}}}
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {"text": "not exposed", "color": "blue"},
|
||||
"-1": {"text": "unavailable", "color": "gray"},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
def quota_stat(panel_id, title, expr, grid, *, unit="percent"):
|
||||
def quota_stat(
|
||||
panel_id,
|
||||
title,
|
||||
expr,
|
||||
grid,
|
||||
*,
|
||||
unit="percent",
|
||||
authenticated_expr=None,
|
||||
):
|
||||
thresholds = remaining_thresholds
|
||||
if unit != "percent":
|
||||
thresholds = {
|
||||
@ -5194,6 +5208,8 @@ def build_ai_dashboard():
|
||||
{"color": "blue", "value": 0},
|
||||
],
|
||||
}
|
||||
if authenticated_expr:
|
||||
expr = f"({expr}) or on() (({authenticated_expr}) * 0 - 2)"
|
||||
panel = stat_panel(
|
||||
panel_id,
|
||||
title,
|
||||
@ -5203,13 +5219,16 @@ def build_ai_dashboard():
|
||||
decimals=1,
|
||||
thresholds=thresholds,
|
||||
instant=True,
|
||||
description="Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value.",
|
||||
description="Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy.",
|
||||
)
|
||||
panel["fieldConfig"]["defaults"]["mappings"] = unavailable_mapping
|
||||
return panel
|
||||
|
||||
openai_ok = 'atlas_ai_quota_fetch_success{provider="openai"} == 1'
|
||||
anthropic_ok = 'atlas_ai_quota_fetch_success{provider="anthropic"} == 1'
|
||||
anthropic_authenticated = (
|
||||
'atlas_ai_provider_authenticated{provider="anthropic"} == 1'
|
||||
)
|
||||
quota = "atlas_ai_quota_remaining_percent"
|
||||
reset = "atlas_ai_quota_reset_timestamp_seconds"
|
||||
panels = [
|
||||
@ -5230,17 +5249,19 @@ def build_ai_dashboard():
|
||||
"Claude 5h Remaining",
|
||||
f'{quota}{{provider="anthropic",window="five_hour"}} and on(provider) ({anthropic_ok})',
|
||||
{"h": 4, "w": 4, "x": 8, "y": 0},
|
||||
authenticated_expr=anthropic_authenticated,
|
||||
),
|
||||
quota_stat(
|
||||
4,
|
||||
"Claude 7d Remaining",
|
||||
f'{quota}{{provider="anthropic",window="seven_day"}} and on(provider) ({anthropic_ok})',
|
||||
{"h": 4, "w": 4, "x": 12, "y": 0},
|
||||
authenticated_expr=anthropic_authenticated,
|
||||
),
|
||||
stat_panel(
|
||||
5,
|
||||
"Quota Collectors Healthy",
|
||||
"sum(atlas_ai_quota_fetch_success) or on() vector(0)",
|
||||
"Provider Access Healthy",
|
||||
"sum(atlas_ai_provider_authenticated) or on() vector(0)",
|
||||
{"h": 4, "w": 4, "x": 16, "y": 0},
|
||||
instant=True,
|
||||
thresholds={
|
||||
@ -5252,7 +5273,7 @@ def build_ai_dashboard():
|
||||
],
|
||||
},
|
||||
value_suffix=" / 2",
|
||||
description="Successful latest quota fetches. Providers are polled independently every five minutes.",
|
||||
description="Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access.",
|
||||
),
|
||||
stat_panel(
|
||||
6,
|
||||
@ -5285,6 +5306,7 @@ def build_ai_dashboard():
|
||||
f'clamp_min({reset}{{provider="anthropic",window="five_hour"}} - time(), 0) and on(provider) ({anthropic_ok})',
|
||||
{"h": 4, "w": 4, "x": 4, "y": 4},
|
||||
unit="s",
|
||||
authenticated_expr=anthropic_authenticated,
|
||||
),
|
||||
quota_stat(
|
||||
9,
|
||||
@ -5292,6 +5314,7 @@ def build_ai_dashboard():
|
||||
f'clamp_min({reset}{{provider="anthropic",window="seven_day"}} - time(), 0) and on(provider) ({anthropic_ok})',
|
||||
{"h": 4, "w": 4, "x": 8, "y": 4},
|
||||
unit="s",
|
||||
authenticated_expr=anthropic_authenticated,
|
||||
),
|
||||
quota_stat(
|
||||
10,
|
||||
|
||||
@ -25,12 +25,13 @@ spec:
|
||||
ai.bstein.dev/execution: Hermes Kanban with durable direct Codex and Claude Code CLI workers
|
||||
ai.bstein.dev/model-policy: Jetson-assisted AUTO routing, low through xhigh, cross-provider fallback
|
||||
ai.bstein.dev/placement: rpi5 preferred; Jetson deferred until state storage is available
|
||||
ai.bstein.dev/config-rev: "20260818-ai-usage-exporter-split-v3"
|
||||
ai.bstein.dev/config-rev: "20260822-claude-oauth-self-heal-v1"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/path: /metrics
|
||||
prometheus.io/port: "9010"
|
||||
vault.hashicorp.com/agent-inject: "true"
|
||||
vault.hashicorp.com/role: hermes-agent
|
||||
vault.hashicorp.com/agent-inject-containers: stage-runtime-access
|
||||
vault.hashicorp.com/agent-inject-secret-agent-api-key: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-template-agent-api-key: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
@ -41,6 +42,11 @@ spec:
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.claude_credentials_json }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-claude-oauth-token: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-template-claude-oauth-token: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
{{ .Data.data.claude_oauth_token }}
|
||||
{{- end }}
|
||||
vault.hashicorp.com/agent-inject-secret-codex-auth: kv/data/atlas/hermes/agent-tokens
|
||||
vault.hashicorp.com/agent-inject-template-codex-auth: |
|
||||
{{- with secret "kv/data/atlas/hermes/agent-tokens" -}}
|
||||
@ -255,8 +261,10 @@ spec:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access}
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: oauth2-config, mountPath: /oauth2-config}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
resources:
|
||||
requests: {cpu: 25m, memory: 32Mi}
|
||||
@ -521,6 +529,7 @@ spec:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
@ -563,6 +572,7 @@ spec:
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
@ -615,6 +625,7 @@ spec:
|
||||
- {name: HOME, value: /opt/data/home}
|
||||
- {name: CODEX_HOME, value: /runtime-access/codex}
|
||||
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
|
||||
- {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude}
|
||||
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
||||
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/opt/hermes/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}
|
||||
- {name: PLAYWRIGHT_BROWSERS_PATH, value: /opt/hermes/.playwright}
|
||||
@ -641,6 +652,7 @@ spec:
|
||||
- {name: HERMES_CLAUDE_HEALTH_PATH, value: /opt/data/provider-health/claude.json}
|
||||
- {name: HERMES_IMAGE_BROKER_KEY_FILE, value: /runtime-access/chat-relay-key}
|
||||
volumeMounts:
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
@ -694,7 +706,7 @@ spec:
|
||||
imagePullPolicy: IfNotPresent
|
||||
args:
|
||||
- --provider=oidc
|
||||
- --config=/vault/secrets/oidc-config
|
||||
- --config=/oauth2-config/oidc-config
|
||||
# Single canonical callback. The __Host- cookie prefix below binds both the
|
||||
# session and the CSRF cookie to one host, so a fresh login must start on
|
||||
# worker.bstein.dev; agent.hermes.bstein.dev keeps serving already
|
||||
@ -755,6 +767,7 @@ spec:
|
||||
limits: {cpu: 250m, memory: 256Mi}
|
||||
volumeMounts:
|
||||
- {name: allowlist, mountPath: /etc/oauth2-proxy, readOnly: true}
|
||||
- {name: oauth2-config, mountPath: /oauth2-config, readOnly: true}
|
||||
- {name: oauth-templates, mountPath: /etc/oauth2-proxy/templates, readOnly: true}
|
||||
- {name: oauth-tmp, mountPath: /tmp}
|
||||
- name: terminal
|
||||
@ -800,6 +813,7 @@ spec:
|
||||
- {name: HERMES_AUTH_FILE, value: /runtime-access/hermes-auth.json}
|
||||
- {name: CODEX_HOME, value: /runtime-access/codex}
|
||||
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
|
||||
- {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude}
|
||||
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||
@ -814,6 +828,7 @@ spec:
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: codex-runtime-patch, mountPath: /opt/hermes/hermes_cli/runtime_provider.py, subPath: runtime_provider.py}
|
||||
@ -877,6 +892,7 @@ spec:
|
||||
- {name: HOME, value: /opt/data/home}
|
||||
- {name: CODEX_HOME, value: /runtime-access/codex}
|
||||
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
|
||||
- {name: HERMES_CLAUDE_BIN, value: /opt/coordinator/claude}
|
||||
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
||||
- {name: PYTHONPATH, value: /opt/hermes}
|
||||
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
|
||||
@ -898,6 +914,7 @@ spec:
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
|
||||
@ -940,6 +957,7 @@ spec:
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: routing-catalog, mountPath: /routing-catalog}
|
||||
@ -998,8 +1016,10 @@ spec:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude, readOnly: true}
|
||||
- {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex, readOnly: true}
|
||||
# Reconcile provider rotations to Vault and restore runtime files if a
|
||||
# client leaves an incomplete credential behind after refresh.
|
||||
- {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude}
|
||||
- {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex}
|
||||
resources:
|
||||
requests: {cpu: 10m, memory: 32Mi}
|
||||
limits: {cpu: 100m, memory: 128Mi}
|
||||
@ -1016,6 +1036,7 @@ spec:
|
||||
- {name: CODEX_HOME, value: /runtime-access/codex}
|
||||
- {name: ATLAS_AI_CODEX_BIN, value: /opt/data/tools/bin/codex}
|
||||
- {name: ATLAS_AI_CLAUDE_CREDENTIALS, value: /runtime-access/claude/.credentials.json}
|
||||
- {name: ATLAS_AI_PROVIDER_HEALTH_ROOT, value: /provider-health}
|
||||
- {name: ATLAS_AI_USAGE_INTERVAL_SECONDS, value: "300"}
|
||||
- {name: ATLAS_AI_USAGE_PORT, value: "9010"}
|
||||
readinessProbe:
|
||||
@ -1038,6 +1059,7 @@ spec:
|
||||
type: RuntimeDefault
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data/tools, subPath: tools, readOnly: true}
|
||||
- {name: home, mountPath: /provider-health, subPath: provider-health, readOnly: true}
|
||||
- {name: runtime-access, mountPath: /runtime-access/claude, subPath: claude}
|
||||
- {name: runtime-access, mountPath: /runtime-access/codex, subPath: codex}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
@ -1186,6 +1208,7 @@ spec:
|
||||
volumeMounts:
|
||||
- {name: home, mountPath: /opt/data}
|
||||
- {name: runtime-access, mountPath: /runtime-access}
|
||||
- {name: claude-oauth-access, mountPath: /claude-oauth-access, readOnly: true}
|
||||
- {name: coordinator, mountPath: /opt/coordinator, readOnly: true}
|
||||
- {name: auth-patch, mountPath: /opt/hermes/hermes_cli/auth.py, subPath: auth.py}
|
||||
- {name: tmp, mountPath: /tmp}
|
||||
@ -1201,6 +1224,10 @@ spec:
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 128Mi
|
||||
- name: claude-oauth-access
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 1Mi
|
||||
- name: routing-catalog
|
||||
persistentVolumeClaim:
|
||||
claimName: hermes-routing-catalog
|
||||
@ -1257,6 +1284,10 @@ spec:
|
||||
- name: oauth-tmp
|
||||
emptyDir:
|
||||
sizeLimit: 64Mi
|
||||
- name: oauth2-config
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
sizeLimit: 1Mi
|
||||
- name: allowlist
|
||||
configMap:
|
||||
name: hermes-owner-allowlist
|
||||
|
||||
@ -23,6 +23,9 @@ CLAUDE_CREDENTIALS = Path(
|
||||
"/runtime-access/claude/.credentials.json",
|
||||
)
|
||||
)
|
||||
PROVIDER_HEALTH_ROOT = Path(
|
||||
os.environ.get("ATLAS_AI_PROVIDER_HEALTH_ROOT", "/provider-health")
|
||||
)
|
||||
CLAUDE_USAGE_URL = os.environ.get(
|
||||
"ATLAS_AI_CLAUDE_USAGE_URL",
|
||||
"https://api.anthropic.com/api/oauth/usage",
|
||||
@ -37,6 +40,7 @@ METRIC_HELP = {
|
||||
"atlas_ai_account_tokens": "First-party account token usage for a fixed period.",
|
||||
"atlas_ai_account_usage_summary": "First-party account usage summary values.",
|
||||
"atlas_ai_extra_usage_enabled": "Whether metered extra usage is enabled for the account.",
|
||||
"atlas_ai_provider_authenticated": "Whether the first-party provider access boundary is authenticated.",
|
||||
"atlas_ai_quota_fetch_duration_seconds": "Duration of the latest provider quota fetch.",
|
||||
"atlas_ai_quota_fetch_success": "Whether the latest provider quota fetch succeeded.",
|
||||
"atlas_ai_quota_last_attempt_timestamp_seconds": "Unix timestamp of the latest quota fetch attempt.",
|
||||
@ -64,6 +68,7 @@ class ProviderState:
|
||||
"""Latest safe samples and fetch health for one provider."""
|
||||
|
||||
samples: list[Sample] = field(default_factory=list)
|
||||
authenticated: bool = False
|
||||
fetch_success: bool = False
|
||||
last_attempt: float = 0
|
||||
last_success: float = 0
|
||||
@ -254,6 +259,18 @@ def parse_claude_payload(payload: dict[str, Any]) -> list[Sample]:
|
||||
query_codex = codex_query.query_codex
|
||||
|
||||
|
||||
def _provider_authenticated(provider: str) -> bool:
|
||||
"""Read one fresh non-secret broker authentication snapshot."""
|
||||
path = PROVIDER_HEALTH_ROOT / ("codex.json" if provider == "openai" else "claude.json")
|
||||
try:
|
||||
if time.time() - path.stat().st_mtime > 10 * 60:
|
||||
return False
|
||||
document = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, TypeError, ValueError, json.JSONDecodeError):
|
||||
return False
|
||||
return isinstance(document, dict) and document.get("authenticated") is True
|
||||
|
||||
|
||||
def query_claude() -> dict[str, Any]:
|
||||
"""Read Claude account quota with the runtime OAuth token held only in memory."""
|
||||
document = json.loads(CLAUDE_CREDENTIALS.read_text(encoding="utf-8"))
|
||||
@ -311,6 +328,11 @@ class Collector:
|
||||
started = time.time()
|
||||
monotonic_started = time.monotonic()
|
||||
try:
|
||||
if provider not in self._providers:
|
||||
raise ValueError("unknown provider")
|
||||
authenticated = _provider_authenticated(provider)
|
||||
with self._lock:
|
||||
self._providers[provider].authenticated = authenticated
|
||||
if provider == "openai":
|
||||
samples = parse_codex_payloads(*query_codex())
|
||||
elif provider == "anthropic":
|
||||
@ -343,12 +365,19 @@ class Collector:
|
||||
provider: ProviderState(**vars(state))
|
||||
for provider, state in self._providers.items()
|
||||
}
|
||||
for provider, state in states.items():
|
||||
state.authenticated = _provider_authenticated(provider)
|
||||
samples: list[Sample] = []
|
||||
for provider, state in states.items():
|
||||
labels = {"provider": provider}
|
||||
samples.extend(state.samples)
|
||||
samples.extend(
|
||||
(
|
||||
Sample(
|
||||
"atlas_ai_provider_authenticated",
|
||||
labels,
|
||||
float(state.authenticated),
|
||||
),
|
||||
Sample(
|
||||
"atlas_ai_quota_fetch_success",
|
||||
labels,
|
||||
|
||||
@ -1,9 +1,17 @@
|
||||
#!/bin/sh
|
||||
# Run Claude Code unattended inside the externally isolated Hermes agent pod.
|
||||
set -eu
|
||||
umask 077
|
||||
|
||||
config_dir="${CLAUDE_CONFIG_DIR:-/runtime-access/claude}"
|
||||
oauth_token_file="${CLAUDE_CODE_OAUTH_TOKEN_FILE:-/claude-oauth-access/token}"
|
||||
if [ -s "$oauth_token_file" ]; then
|
||||
CLAUDE_CODE_OAUTH_TOKEN="$(tr -d '\r\n' < "$oauth_token_file")"
|
||||
export CLAUDE_CODE_OAUTH_TOKEN
|
||||
fi
|
||||
|
||||
exec /opt/data/tools/bin/claude \
|
||||
--dangerously-skip-permissions \
|
||||
--autocompact auto \
|
||||
--settings "${CLAUDE_CONFIG_DIR:-/runtime-access/claude}/settings.json" \
|
||||
--settings "$config_dir/settings.json" \
|
||||
"$@"
|
||||
|
||||
@ -23,7 +23,7 @@ SWITCHYARD_URL = os.environ.get(
|
||||
)
|
||||
STATE_ROOT = DATA_ROOT / "cli-lanes"
|
||||
CODEX_BIN = DATA_ROOT / "tools/bin/codex"
|
||||
CLAUDE_BIN = DATA_ROOT / "tools/bin/claude"
|
||||
CLAUDE_BIN = Path(os.environ.get("HERMES_CLAUDE_BIN", DATA_ROOT / "tools/bin/claude"))
|
||||
CLAUDE_SETTINGS = DATA_ROOT / "home/.claude/settings.json"
|
||||
RESULT_SCHEMA_PATH = STATE_ROOT / "worker-result.schema.json"
|
||||
EFFORTS = ("low", "medium", "high", "xhigh")
|
||||
|
||||
@ -15,6 +15,8 @@ from pathlib import Path
|
||||
|
||||
VAULT_ROOT = Path("/vault/secrets")
|
||||
RUNTIME_ROOT = Path("/runtime-access")
|
||||
CLAUDE_OAUTH_ACCESS_ROOT = Path("/claude-oauth-access")
|
||||
OAUTH2_CONFIG_ROOT = Path("/oauth2-config")
|
||||
PERSISTENT_HOME = Path("/opt/data/home")
|
||||
WORKER_ROOT = Path(os.environ.get("HERMES_WORKER_ROOT", "/workspace"))
|
||||
PROVIDER_ACCESS_ROOT = Path(
|
||||
@ -23,9 +25,13 @@ PROVIDER_ACCESS_ROOT = Path(
|
||||
POOL_ACCESS_ROOT = Path(os.environ.get("HERMES_POOL_ACCESS_ROOT", "/pool-access"))
|
||||
OWNER_UID = 10000
|
||||
OWNER_GID = 10000
|
||||
OAUTH2_PROXY_UID = 65532
|
||||
OAUTH2_PROXY_GID = 65532
|
||||
|
||||
|
||||
def _owned_directory(path: Path) -> None:
|
||||
def _owned_directory(
|
||||
path: Path, *, owner_uid: int = OWNER_UID, owner_gid: int = OWNER_GID
|
||||
) -> None:
|
||||
"""Create one private directory owned by the unprivileged Hermes user."""
|
||||
if path.is_symlink():
|
||||
raise RuntimeError(f"private directory {path.name} must not be a symlink")
|
||||
@ -33,7 +39,7 @@ def _owned_directory(path: Path) -> None:
|
||||
if path.is_symlink() or not path.is_dir():
|
||||
raise RuntimeError(f"private directory {path.name} is invalid")
|
||||
path.chmod(0o700)
|
||||
os.chown(path, OWNER_UID, OWNER_GID)
|
||||
os.chown(path, owner_uid, owner_gid)
|
||||
|
||||
|
||||
def _read_secret(source_name: str) -> str:
|
||||
@ -52,7 +58,13 @@ def _read_secret(source_name: str) -> str:
|
||||
return value
|
||||
|
||||
|
||||
def _write_secret(destination: Path, value: str) -> None:
|
||||
def _write_secret(
|
||||
destination: Path,
|
||||
value: str,
|
||||
*,
|
||||
owner_uid: int = OWNER_UID,
|
||||
owner_gid: int = OWNER_GID,
|
||||
) -> None:
|
||||
temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.tmp")
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(temporary, flags, 0o600)
|
||||
@ -62,7 +74,7 @@ def _write_secret(destination: Path, value: str) -> None:
|
||||
remaining = remaining[os.write(descriptor, remaining) :]
|
||||
os.fsync(descriptor)
|
||||
os.fchmod(descriptor, 0o600)
|
||||
os.fchown(descriptor, OWNER_UID, OWNER_GID)
|
||||
os.fchown(descriptor, owner_uid, owner_gid)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
@ -71,10 +83,21 @@ def _write_secret(destination: Path, value: str) -> None:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _copy_secret(source_name: str, destination: Path) -> str:
|
||||
def _copy_secret(
|
||||
source_name: str,
|
||||
destination: Path,
|
||||
*,
|
||||
owner_uid: int = OWNER_UID,
|
||||
owner_gid: int = OWNER_GID,
|
||||
) -> str:
|
||||
"""Copy a non-empty Vault projection without logging its value."""
|
||||
value = _read_secret(source_name)
|
||||
_write_secret(destination, value)
|
||||
_write_secret(
|
||||
destination,
|
||||
value,
|
||||
owner_uid=owner_uid,
|
||||
owner_gid=owner_gid,
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
@ -180,6 +203,23 @@ def _stage_node_ssh_config() -> None:
|
||||
|
||||
def stage_agent() -> None:
|
||||
"""Stage the owner agent's complete runtime access set."""
|
||||
_owned_directory(PERSISTENT_HOME.parent / "provider-health")
|
||||
_owned_directory(CLAUDE_OAUTH_ACCESS_ROOT)
|
||||
_copy_secret(
|
||||
"claude-oauth-token",
|
||||
CLAUDE_OAUTH_ACCESS_ROOT / "token",
|
||||
)
|
||||
_owned_directory(
|
||||
OAUTH2_CONFIG_ROOT,
|
||||
owner_uid=OAUTH2_PROXY_UID,
|
||||
owner_gid=OAUTH2_PROXY_GID,
|
||||
)
|
||||
_copy_secret(
|
||||
"oidc-config",
|
||||
OAUTH2_CONFIG_ROOT / "oidc-config",
|
||||
owner_uid=OAUTH2_PROXY_UID,
|
||||
owner_gid=OAUTH2_PROXY_GID,
|
||||
)
|
||||
for path in (RUNTIME_ROOT, RUNTIME_ROOT / "claude", RUNTIME_ROOT / "codex"):
|
||||
_owned_directory(path)
|
||||
for name in (
|
||||
|
||||
@ -1,11 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Persist refreshed coding-client OAuth documents back into Vault."""
|
||||
"""Reconcile coding-client OAuth documents between runtime storage and Vault."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
@ -87,8 +88,45 @@ def _credential_document(path: Path, required: tuple[str, ...]) -> str | None:
|
||||
return value
|
||||
|
||||
|
||||
def sync_once(token: str) -> list[str]:
|
||||
"""CAS-update only credential fields whose runtime documents changed."""
|
||||
def _credential_value(value: object, required: tuple[str, ...]) -> str | None:
|
||||
"""Return a complete Vault credential document or reject invalid content."""
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
return None
|
||||
try:
|
||||
document = json.loads(value)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
current = document
|
||||
for key in required:
|
||||
if not isinstance(current, dict) or key not in current:
|
||||
return None
|
||||
current = current[key]
|
||||
if not isinstance(current, str) or not current:
|
||||
return None
|
||||
return value.strip()
|
||||
|
||||
|
||||
def _write_credential(path: Path, value: str) -> None:
|
||||
"""Atomically restore one private runtime credential from Vault."""
|
||||
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
||||
descriptor = os.open(temporary, flags, 0o600)
|
||||
try:
|
||||
remaining = memoryview((value + "\n").encode("utf-8"))
|
||||
while remaining:
|
||||
remaining = remaining[os.write(descriptor, remaining) :]
|
||||
os.fsync(descriptor)
|
||||
os.fchmod(descriptor, 0o600)
|
||||
finally:
|
||||
os.close(descriptor)
|
||||
try:
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def sync_once(token: str) -> tuple[list[str], list[str]]:
|
||||
"""Persist valid rotations and restore incomplete runtime credentials."""
|
||||
response = _request("GET", SECRET_ENDPOINT, token=token)
|
||||
envelope = response.get("data")
|
||||
if not isinstance(envelope, dict):
|
||||
@ -101,20 +139,26 @@ def sync_once(token: str) -> list[str]:
|
||||
if not isinstance(version, int) or version < 1:
|
||||
raise RuntimeError("Vault KV response has no version")
|
||||
updated = dict(data)
|
||||
changed: list[str] = []
|
||||
persisted: list[str] = []
|
||||
restored: list[str] = []
|
||||
for field, (path, required) in CREDENTIALS.items():
|
||||
value = _credential_document(path, required)
|
||||
if value is not None and value != data.get(field):
|
||||
updated[field] = value
|
||||
changed.append(field)
|
||||
if changed:
|
||||
runtime_value = _credential_document(path, required)
|
||||
vault_value = _credential_value(data.get(field), required)
|
||||
if runtime_value is not None:
|
||||
if runtime_value != vault_value:
|
||||
updated[field] = runtime_value
|
||||
persisted.append(field)
|
||||
elif vault_value is not None:
|
||||
_write_credential(path, vault_value)
|
||||
restored.append(field)
|
||||
if persisted:
|
||||
_request(
|
||||
"POST",
|
||||
SECRET_ENDPOINT,
|
||||
{"options": {"cas": version}, "data": updated},
|
||||
token=token,
|
||||
)
|
||||
return changed
|
||||
return persisted, restored
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@ -123,10 +167,17 @@ def main() -> int:
|
||||
while True:
|
||||
try:
|
||||
token = _login()
|
||||
changed = sync_once(token)
|
||||
if changed:
|
||||
persisted, restored = sync_once(token)
|
||||
if persisted:
|
||||
print(
|
||||
"Persisted refreshed runtime credentials: " + ", ".join(changed),
|
||||
"Persisted refreshed runtime credentials: "
|
||||
+ ", ".join(persisted),
|
||||
flush=True,
|
||||
)
|
||||
if restored:
|
||||
print(
|
||||
"Restored incomplete runtime credentials from Vault: "
|
||||
+ ", ".join(restored),
|
||||
flush=True,
|
||||
)
|
||||
except (OSError, RuntimeError, HTTPError, URLError, ValueError) as error:
|
||||
|
||||
@ -34,6 +34,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -87,7 +91,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
@ -119,6 +123,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -172,7 +180,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
@ -190,7 +198,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -204,6 +212,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -257,7 +269,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
@ -275,7 +287,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -289,6 +301,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -342,12 +358,12 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Quota Collectors Healthy",
|
||||
"title": "Provider Access Healthy",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "atlas-vm"
|
||||
@ -360,7 +376,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)",
|
||||
"expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -409,7 +425,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Successful latest quota fetches. Providers are polled independently every five minutes."
|
||||
"description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access."
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
@ -511,6 +527,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -552,7 +572,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
@ -570,7 +590,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -584,6 +604,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -625,7 +649,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
@ -643,7 +667,7 @@
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -657,6 +681,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -698,7 +726,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
@ -730,6 +758,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -771,7 +803,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
@ -803,6 +835,10 @@
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -844,7 +880,7 @@
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
|
||||
@ -43,6 +43,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -96,7 +100,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
@ -128,6 +132,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -181,7 +189,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
@ -199,7 +207,7 @@ data:
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"five_hour\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -213,6 +221,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -266,7 +278,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
@ -284,7 +296,7 @@ data:
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((atlas_ai_quota_remaining_percent{provider=\"anthropic\",window=\"seven_day\"} and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -298,6 +310,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -351,12 +367,12 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "stat",
|
||||
"title": "Quota Collectors Healthy",
|
||||
"title": "Provider Access Healthy",
|
||||
"datasource": {
|
||||
"type": "prometheus",
|
||||
"uid": "atlas-vm"
|
||||
@ -369,7 +385,7 @@ data:
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "sum(atlas_ai_quota_fetch_success) or on() vector(0)",
|
||||
"expr": "sum(atlas_ai_provider_authenticated) or on() vector(0)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -418,7 +434,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Successful latest quota fetches. Providers are polled independently every five minutes."
|
||||
"description": "Authenticated first-party provider boundaries. Quota visibility is tracked separately from model access."
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
@ -520,6 +536,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -561,7 +581,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
@ -579,7 +599,7 @@ data:
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"five_hour\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -593,6 +613,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -634,7 +658,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
@ -652,7 +676,7 @@ data:
|
||||
},
|
||||
"targets": [
|
||||
{
|
||||
"expr": "(clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() vector(-1)",
|
||||
"expr": "((clamp_min(atlas_ai_quota_reset_timestamp_seconds{provider=\"anthropic\",window=\"seven_day\"} - time(), 0) and on(provider) (atlas_ai_quota_fetch_success{provider=\"anthropic\"} == 1)) or on() ((atlas_ai_provider_authenticated{provider=\"anthropic\"} == 1) * 0 - 2)) or on() vector(-1)",
|
||||
"refId": "A",
|
||||
"instant": true
|
||||
}
|
||||
@ -666,6 +690,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -707,7 +735,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
@ -739,6 +767,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -780,7 +812,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
@ -812,6 +844,10 @@ data:
|
||||
{
|
||||
"type": "value",
|
||||
"options": {
|
||||
"-2": {
|
||||
"text": "not exposed",
|
||||
"color": "dark-blue"
|
||||
},
|
||||
"-1": {
|
||||
"text": "unavailable",
|
||||
"color": "gray"
|
||||
@ -853,7 +889,7 @@ data:
|
||||
},
|
||||
"textMode": "value"
|
||||
},
|
||||
"description": "Live first-party CLI account telemetry. Unavailable means the provider did not return a fresh structured value."
|
||||
"description": "Live first-party CLI account telemetry. Not exposed means access is healthy but that credential scope cannot read quota; unavailable means provider access is unhealthy."
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
|
||||
@ -37,11 +37,13 @@ def test_ai_dashboard_is_internal_and_uses_real_quota_and_switchyard_metrics():
|
||||
assert "Codex Weekly Remaining" in panels
|
||||
assert "Claude 5h Remaining" in panels
|
||||
assert "Claude 7d Remaining" in panels
|
||||
assert "Provider Access Healthy" in panels
|
||||
assert "Provider Selections (Range)" in panels
|
||||
assert "Local Classifier Calls" in panels
|
||||
assert "Hermes Workload CPU (Attribution Proxy)" in panels
|
||||
assert "atlas_ai_quota_remaining_percent" in expressions
|
||||
assert "atlas_ai_quota_reset_timestamp_seconds" in expressions
|
||||
assert "atlas_ai_provider_authenticated" in expressions
|
||||
assert "switchyard_decisions_total" in expressions
|
||||
assert "switchyard_cached_tokens_total" in expressions
|
||||
assert "switchyard_model_call_latency_ms_bucket" in expressions
|
||||
|
||||
@ -133,6 +133,17 @@ def test_provider_queries_validate_credentials_and_response_shape(
|
||||
assert mod.query_codex is mod.codex_query.query_codex
|
||||
|
||||
|
||||
def test_provider_authentication_uses_only_fresh_non_secret_health(tmp_path, monkeypatch):
|
||||
mod = load_module()
|
||||
monkeypatch.setattr(mod, "PROVIDER_HEALTH_ROOT", tmp_path)
|
||||
health = tmp_path / "claude.json"
|
||||
health.write_text('{"authenticated":true}\n', encoding="utf-8")
|
||||
|
||||
assert mod._provider_authenticated("anthropic")
|
||||
monkeypatch.setattr(mod.time, "time", lambda: health.stat().st_mtime + 601)
|
||||
assert not mod._provider_authenticated("anthropic")
|
||||
|
||||
|
||||
def test_collector_refreshes_both_providers_and_contains_unknown_provider(
|
||||
monkeypatch, capsys
|
||||
):
|
||||
|
||||
@ -87,6 +87,8 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
vault = tmp_path / "vault"
|
||||
runtime = tmp_path / "runtime"
|
||||
home = tmp_path / "home"
|
||||
claude_oauth_access = tmp_path / "claude-oauth-access"
|
||||
oauth2_config = tmp_path / "oauth2-config"
|
||||
vault.mkdir()
|
||||
(home / ".claude").mkdir(parents=True)
|
||||
(home / ".claude" / "backups").mkdir()
|
||||
@ -98,6 +100,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
)
|
||||
values = {
|
||||
"agent-api-key": "agent-key",
|
||||
"claude-oauth-token": "long-lived-claude-token",
|
||||
"execution-pool-key": "e" * 64,
|
||||
"chat-relay-key": "relay-key",
|
||||
# gitea-token/username are present in Vault but the agent staging must
|
||||
@ -109,6 +112,7 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
"node-ssh-private-key": "private-key",
|
||||
"node-ssh-config": "host-config",
|
||||
"node-ssh-known-hosts": "known-hosts",
|
||||
"oidc-config": "client_id = \"hermes-agent\"",
|
||||
"claude-credentials": json.dumps(
|
||||
{"claudeAiOauth": {"refreshToken": "claude-refresh"}}
|
||||
),
|
||||
@ -119,14 +123,21 @@ def test_agent_runtime_stage_keeps_credentials_in_memory(tmp_path: Path, monkeyp
|
||||
monkeypatch.setattr(stage, "VAULT_ROOT", vault)
|
||||
monkeypatch.setattr(stage, "RUNTIME_ROOT", runtime)
|
||||
monkeypatch.setattr(stage, "PERSISTENT_HOME", home)
|
||||
monkeypatch.setattr(stage, "CLAUDE_OAUTH_ACCESS_ROOT", claude_oauth_access)
|
||||
monkeypatch.setattr(stage, "OAUTH2_CONFIG_ROOT", oauth2_config)
|
||||
monkeypatch.setattr(stage.os, "chown", lambda *_args: None)
|
||||
monkeypatch.setattr(stage.os, "fchown", lambda *_args: None)
|
||||
|
||||
stage.stage_agent()
|
||||
|
||||
assert (runtime / "claude/.credentials.json").stat().st_mode & 0o777 == 0o600
|
||||
assert (claude_oauth_access / "token").read_text(encoding="utf-8").strip() == (
|
||||
"long-lived-claude-token"
|
||||
)
|
||||
assert (claude_oauth_access / "token").stat().st_mode & 0o777 == 0o600
|
||||
assert (runtime / "codex/auth.json").stat().st_mode & 0o777 == 0o600
|
||||
assert (runtime / "jenkins-image-build-token").stat().st_mode & 0o777 == 0o600
|
||||
assert (oauth2_config / "oidc-config").stat().st_mode & 0o777 == 0o600
|
||||
assert (runtime / "claude/settings.json").is_symlink()
|
||||
assert (runtime / "codex/skills").is_symlink()
|
||||
assert not (runtime / "claude/backups").exists()
|
||||
@ -289,14 +300,78 @@ def test_runtime_refresh_sync_uses_cas_and_preserves_other_fields(
|
||||
|
||||
monkeypatch.setattr(sync, "_request", request)
|
||||
|
||||
assert sync.sync_once("vault-token") == [
|
||||
"claude_credentials_json",
|
||||
"codex_auth_json",
|
||||
]
|
||||
assert sync.sync_once("vault-token") == (
|
||||
["claude_credentials_json", "codex_auth_json"],
|
||||
[],
|
||||
)
|
||||
assert writes[0][1]["options"] == {"cas": 7}
|
||||
assert writes[0][1]["data"]["agent_api_key"] == "preserve-me"
|
||||
|
||||
|
||||
def test_runtime_refresh_sync_restores_incomplete_credentials_from_vault(
|
||||
tmp_path: Path, monkeypatch
|
||||
):
|
||||
sync = _load("sync_runtime_credentials")
|
||||
claude = tmp_path / "claude.json"
|
||||
claude.write_text(
|
||||
json.dumps(
|
||||
{"claudeAiOauth": {"accessToken": "", "refreshToken": ""}}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
vault_value = json.dumps(
|
||||
{
|
||||
"claudeAiOauth": {
|
||||
"accessToken": "vault-access",
|
||||
"refreshToken": "vault-refresh",
|
||||
}
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
sync,
|
||||
"CREDENTIALS",
|
||||
{
|
||||
"claude_credentials_json": (
|
||||
claude,
|
||||
("claudeAiOauth", "refreshToken"),
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
def request(method, _path, payload=None, *, token=""):
|
||||
assert method == "GET"
|
||||
assert payload is None
|
||||
assert token == "vault-token"
|
||||
return {
|
||||
"data": {
|
||||
"data": {"claude_credentials_json": vault_value},
|
||||
"metadata": {"version": 8},
|
||||
}
|
||||
}
|
||||
|
||||
monkeypatch.setattr(sync, "_request", request)
|
||||
|
||||
assert sync.sync_once("vault-token") == ([], ["claude_credentials_json"])
|
||||
restored = json.loads(claude.read_text(encoding="utf-8"))
|
||||
assert restored["claudeAiOauth"]["refreshToken"] == "vault-refresh"
|
||||
assert claude.stat().st_mode & 0o077 == 0
|
||||
|
||||
|
||||
def test_claude_clients_use_the_vault_staged_long_lived_token():
|
||||
wrapper = (SCRIPTS / "claude").read_text(encoding="utf-8")
|
||||
deployment = yaml.safe_load((HERMES / "agent-deployment.yaml").read_text())
|
||||
runner = next(
|
||||
item
|
||||
for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||
if item["name"] == "cli-lane-runner"
|
||||
)
|
||||
environment = {item["name"]: item["value"] for item in runner["env"]}
|
||||
|
||||
assert 'CLAUDE_CODE_OAUTH_TOKEN_FILE:-/claude-oauth-access/token' in wrapper
|
||||
assert 'export CLAUDE_CODE_OAUTH_TOKEN' in wrapper
|
||||
assert environment["HERMES_CLAUDE_BIN"] == "/opt/coordinator/claude"
|
||||
|
||||
|
||||
def test_subprocess_patches_strip_and_redact_runtime_credentials(tmp_path: Path):
|
||||
boundary = _load("patch_subprocess_secret_boundary")
|
||||
process = _load("patch_process_output_redaction")
|
||||
@ -346,6 +421,38 @@ def test_manifests_never_seed_access_material_into_persistent_env():
|
||||
assert f"upsert_env {key}" not in command
|
||||
|
||||
annotations = agent["spec"]["template"]["metadata"]["annotations"]
|
||||
assert annotations["vault.hashicorp.com/agent-inject-containers"] == (
|
||||
"stage-runtime-access"
|
||||
)
|
||||
pod = agent["spec"]["template"]["spec"]
|
||||
token_containers = {
|
||||
item["name"]
|
||||
for item in pod["containers"]
|
||||
if any(
|
||||
mount["name"] == "claude-oauth-access"
|
||||
for mount in item.get("volumeMounts", [])
|
||||
)
|
||||
}
|
||||
assert token_containers == {
|
||||
"hermes",
|
||||
"terminal",
|
||||
"cli-lane-runner",
|
||||
"model-steward",
|
||||
"claude-broker",
|
||||
}
|
||||
token_init_containers = {
|
||||
item["name"]
|
||||
for item in pod["initContainers"]
|
||||
if any(
|
||||
mount["name"] == "claude-oauth-access"
|
||||
for mount in item.get("volumeMounts", [])
|
||||
)
|
||||
}
|
||||
assert token_init_containers == {
|
||||
"stage-runtime-access",
|
||||
"bootstrap-coordinator",
|
||||
"configure-agent-clients",
|
||||
}
|
||||
agent_runtime = next(
|
||||
item
|
||||
for item in agent["spec"]["template"]["spec"]["volumes"]
|
||||
@ -359,6 +466,9 @@ def test_manifests_never_seed_access_material_into_persistent_env():
|
||||
assert annotations["vault.hashicorp.com/agent-inject-secret-claude-credentials"] == (
|
||||
"kv/data/atlas/hermes/agent-tokens"
|
||||
)
|
||||
assert annotations["vault.hashicorp.com/agent-inject-secret-claude-oauth-token"] == (
|
||||
"kv/data/atlas/hermes/agent-tokens"
|
||||
)
|
||||
assert annotations["vault.hashicorp.com/agent-inject-secret-codex-auth"] == (
|
||||
"kv/data/atlas/hermes/agent-tokens"
|
||||
)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user