diff --git a/scripts/ops/hermes_suite_probe.py b/scripts/ops/hermes_suite_probe.py index e96551c2..c678e6f7 100644 --- a/scripts/ops/hermes_suite_probe.py +++ b/scripts/ops/hermes_suite_probe.py @@ -99,7 +99,8 @@ def run(size): if "result" in result: coverage = Counter(c["alias"] for c in request["cases"]) == Counter( a for g in result["result"]["groups"] for a in g["members"]) - return {"size": size, "http_status": code, "request_bytes": len(json.dumps(request).encode()), + return {"size": size, "http_status": code, + "request_bytes": len(json.dumps(request, separators=(",", ":")).encode()), "client_wall_seconds": round(time.monotonic() - started, 3), "exact_alias_coverage": coverage, "job": result} diff --git a/services/hermes/scripts/suite_api.py b/services/hermes/scripts/suite_api.py index 92acee69..c46c6892 100644 --- a/services/hermes/scripts/suite_api.py +++ b/services/hermes/scripts/suite_api.py @@ -40,18 +40,27 @@ def credential(header, directory="/vault/secrets"): candidate = header[7:] if header.startswith("Bearer ") else "" if not candidate or len(candidate) > 256: raise Problem("authentication", 401) - for name, providers in (("token", []), ("synthetic-token", ["claude", "codex"])): + for name, providers in (("token", []), ("synthetic-token", ["claude", "codex"]), + ("operational-token", ["claude"])): expected = Path(directory, name).read_text().strip() if expected and hmac.compare_digest(candidate, expected): return name, providers raise Problem("authentication", 401) -def authorize(raw, providers): - """Validate policy and keep external approval limited to exact synthetic suites.""" +def generalized_claude_approved(): + """Read the explicit operator permission for this configured Claude account.""" + return os.environ.get("PLANNING_GENERALIZED_CLAUDE_APPROVED") == "true" + + +def authorize(raw, providers, *, operational=False): + """Intersect client permissions, request policy, and approved data scope.""" request = validate_request(raw, providers) if request["routing"]["allow_external"] and not allowed_synthetic(request): - raise Problem("external_data_not_approved", 403) + if not operational or not generalized_claude_approved(): + raise Problem("external_data_not_approved", 403) + if request["routing"]["allowed_external_providers"] != ["claude"]: + raise Problem("provider_forbidden", 403) return request @@ -110,7 +119,11 @@ class Handler(BaseHTTPRequestHandler): return self.send(200, {"status": "ready", "configuration_revision": REVISION}) if method == "GET" and self.path == "/v1/capabilities": return self.send(200, {"configuration_revision": REVISION, "models": MODELS, - "allowed_external_providers": providers, "external_scope": "exact_synthetic_fixtures", + "allowed_external_providers": providers, + "external_scope": "generalized_claude_and_exact_synthetic_fixtures" + if generalized_claude_approved() and owner == "operational-token" else "exact_synthetic_fixtures", + "generalized_external_providers": ["claude"] + if generalized_claude_approved() and owner == "operational-token" else [], "strategy": ["whole_suite"], "max_request_bytes": MAX_BODY, "max_result_bytes": MAX_RESULT, "max_cases": MAX_CASES, "max_seconds": TIMEOUT, "concurrency": 1, "queue": False, @@ -120,7 +133,7 @@ class Handler(BaseHTTPRequestHandler): if method == "GET" and match: return self.send(200, fixture(int(match[1]))[0]) if method == "POST" and self.path in {"/v1/preflight", "/v1/jobs"}: - request = authorize(self.body(), providers) + request = authorize(self.body(), providers, operational=owner == "operational-token") selected = preflight(request) if self.path == "/v1/preflight": return self.send(200, {"status": "eligible", "routing": request["routing"], diff --git a/services/hermes/scripts/suite_backends.py b/services/hermes/scripts/suite_backends.py index 66c51bb4..19da59b7 100644 --- a/services/hermes/scripts/suite_backends.py +++ b/services/hermes/scripts/suite_backends.py @@ -68,9 +68,13 @@ def local_generate(request, cancel, client_ip): """Reuse the unchanged RTX API and its pinned-model, local-only safeguards.""" headers = {"Authorization": "Bearer " + Path("/vault/secrets/local-token").read_text().strip(), "X-Forwarded-For": client_ip} + schema = json.loads(encoded(SCHEMA)) + # Constrain the local decoder to aliases, excluding copied case descriptions. + schema["properties"]["groups"]["items"]["properties"]["members"]["items"]["enum"] = [ + case["alias"] for case in request["cases"]] response = post(LOCAL + "/api/generate", { "model": MODELS["local"]["model"], "prompt": SYSTEM + "\n" + prompt(request), - "stream": False, "format": SCHEMA, + "stream": False, "format": schema, "options": {"num_predict": 2048, "temperature": 0, "seed": 0}}, request["execution"]["max_seconds"], headers) if cancel.is_set(): diff --git a/services/hermes/scripts/suite_contract.py b/services/hermes/scripts/suite_contract.py index a26e54d9..dddfefe8 100644 --- a/services/hermes/scripts/suite_contract.py +++ b/services/hermes/scripts/suite_contract.py @@ -6,7 +6,7 @@ import json import re from collections import Counter -REVISION = "suite-v1-20260929" +REVISION = "suite-v6-20260929" MAX_BODY = 1 << 20 MAX_RESULT = 1 << 20 MAX_CASES = 400 @@ -156,12 +156,14 @@ def preflight(request): reasons[provider] = "unverified_output_capacity" continue reserve = output_estimate + (8192 if provider == "claude" else 0) - if reserve > model["output"] or input_bytes + model["overhead"] + model["output"] * (3 if provider == "claude" else 1) > model["context"]: + schema_extra = len(encoded([c["alias"] for c in request["cases"]])) + 16 if provider == "local" else 0 + total_input = input_bytes + schema_extra + if reserve > model["output"] or total_input + model["overhead"] + model["output"] * (3 if provider == "claude" else 1) > model["context"]: reasons[provider] = "capacity" continue return {"provider": provider, **model, "configuration_revision": REVISION, - "input_bytes": input_bytes, "input_token_count": None, - "input_token_bound": input_bytes + model["overhead"], + "input_bytes": total_input, "input_token_count": None, + "input_token_bound": total_input + model["overhead"], "input_count_method": "UTF-8 byte upper bound plus reserved harness overhead; not a tokenizer", "output_reservation_tokens": reserve, "output_reservation_verified": False, "case_count": count, "source_sha256": digest(request["cases"])} diff --git a/services/hermes/scripts/suite_synthetic.py b/services/hermes/scripts/suite_synthetic.py index 06377bfd..bdd9daa9 100644 --- a/services/hermes/scripts/suite_synthetic.py +++ b/services/hermes/scripts/suite_synthetic.py @@ -36,6 +36,7 @@ def fixture(size): "success_criteria": assertion + ". Preserve diagnostic evidence for this objective.", "preconditions": setup + ". Reset fixture state between parameter variations.", "operating_condition": "Qualified synthetic build; deterministic seed; isolated execution.", + "case_type": ("nominal", "boundary", "fault injection")[(index // 6) % 3], "verification_method": "test" if family != "static" else "analysis", "verifies": "[reference]"} cases.append(case) diff --git a/services/hermes/suite-planner-deployment.yaml b/services/hermes/suite-planner-deployment.yaml index 20b642fa..2d776929 100644 --- a/services/hermes/suite-planner-deployment.yaml +++ b/services/hermes/suite-planner-deployment.yaml @@ -36,7 +36,7 @@ spec: app: hermes-suite-planner annotations: fluentbit.io/exclude: "true" - ai.bstein.dev/config-rev: suite-v5-20260929 + ai.bstein.dev/config-rev: suite-v6-20260929 vault.hashicorp.com/agent-inject: "true" vault.hashicorp.com/agent-pre-populate-only: "true" vault.hashicorp.com/agent-init-first: "true" @@ -55,6 +55,11 @@ spec: {{- with secret "kv/data/atlas/hermes/suite-planning-api" -}} {{ .Data.data.synthetic_token }} {{- end -}} + vault.hashicorp.com/agent-inject-secret-operational-token: kv/data/atlas/hermes/suite-planning-api + vault.hashicorp.com/agent-inject-template-operational-token: | + {{- with secret "kv/data/atlas/hermes/suite-planning-api" -}} + {{ .Data.data.operational_token }} + {{- end -}} vault.hashicorp.com/agent-inject-secret-local-token: kv/data/atlas/hermes/model-gate-lan-api vault.hashicorp.com/agent-inject-template-local-token: | {{- with secret "kv/data/atlas/hermes/model-gate-lan-api" -}} @@ -67,6 +72,7 @@ spec: {{- end -}} vault.hashicorp.com/agent-inject-perms-token: "0400" vault.hashicorp.com/agent-inject-perms-synthetic-token: "0400" + vault.hashicorp.com/agent-inject-perms-operational-token: "0400" vault.hashicorp.com/agent-inject-perms-local-token: "0400" vault.hashicorp.com/agent-inject-perms-claude-token: "0400" spec: @@ -114,6 +120,8 @@ spec: env: - {name: PYTHONDONTWRITEBYTECODE, value: "1"} - {name: PYTHONUNBUFFERED, value: "1"} + # User approved generalized CASE records on this Claude account, 2026-09-29. + - {name: PLANNING_GENERALIZED_CLAUDE_APPROVED, value: "true"} - {name: PLANNING_CLAUDE_SHA256, value: 4e9bec1177ce9690e8bd988b710ac24105e70da428dd094c5adcbbe786a55555} ports: - {name: http, containerPort: 9000} diff --git a/services/vault/hermes-suite-role-bootstrap-job.yaml b/services/vault/hermes-suite-role-bootstrap-job.yaml index 66e8105e..90e04206 100644 --- a/services/vault/hermes-suite-role-bootstrap-job.yaml +++ b/services/vault/hermes-suite-role-bootstrap-job.yaml @@ -1,9 +1,9 @@ # services/vault/hermes-suite-role-bootstrap-job.yaml -# Purpose: apply the Vault read/write boundaries needed by Hermes operator OIDC. +# Configure the scoped suite-planning Vault roles. apiVersion: batch/v1 kind: Job metadata: - name: vault-k8s-auth-suite-1 + name: vault-k8s-auth-suite-2 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/hermes-suite-token-seed-job.yaml b/services/vault/hermes-suite-token-seed-job.yaml index 2390de9c..ac6ef5fd 100644 --- a/services/vault/hermes-suite-token-seed-job.yaml +++ b/services/vault/hermes-suite-token-seed-job.yaml @@ -8,7 +8,7 @@ metadata: apiVersion: batch/v1 kind: Job metadata: - name: vault-hermes-suite-token-seed-1 + name: vault-hermes-suite-token-seed-2 namespace: vault spec: backoffLimit: 2 diff --git a/services/vault/scripts/vault_hermes_suite_token_ensure.sh b/services/vault/scripts/vault_hermes_suite_token_ensure.sh index 8650e47d..4bb454b5 100644 --- a/services/vault/scripts/vault_hermes_suite_token_ensure.sh +++ b/services/vault/scripts/vault_hermes_suite_token_ensure.sh @@ -1,5 +1,5 @@ #!/usr/bin/env sh -# Seed distinct local-only and synthetic-external credentials without rotation. +# Seed distinct local-only, synthetic, and approved operational credentials. set -eu umask 077 secret_path=kv/atlas/hermes/suite-planning-api @@ -26,19 +26,38 @@ if existing="$(vault kv get -format=json "$secret_path" 2>&1)"; then test "${#value}" -eq 64 || exit 1 unset value done - printf 'Suite credentials already present; unchanged.\n' + if value="$(vault kv get -field=operational_token "$secret_path" 2>/dev/null)"; then + case "$value" in ''|*[!0-9a-f]*) exit 1 ;; esac + test "${#value}" -eq 64 || exit 1 + unset value + printf 'Suite credentials already present; unchanged.\n' + exit 0 + fi + operational_token="$(vault write -field=random_bytes sys/tools/random/32 format=hex)" + case "$operational_token" in ''|*[!0-9a-f]*) exit 1 ;; esac + test "${#operational_token}" -eq 64 || exit 1 + printf '{"operational_token":"%s"}\n' "$operational_token" > "$payload_file" + unset operational_token + attempt=0 + until vault kv patch -method=patch "$secret_path" @"$payload_file" >/dev/null 2>&1; do + attempt=$((attempt + 1)) + test "$attempt" -lt 40 || exit 1 + sleep 3 + done + printf 'Operational credential added; existing credentials unchanged.\n' exit 0 fi case "$existing" in *'No value found'*|*'Code: 404'*) ;; *) exit 1 ;; esac unset existing local_token="$(vault write -field=random_bytes sys/tools/random/32 format=hex)" synthetic_token="$(vault write -field=random_bytes sys/tools/random/32 format=hex)" -for value in "$local_token" "$synthetic_token"; do +operational_token="$(vault write -field=random_bytes sys/tools/random/32 format=hex)" +for value in "$local_token" "$synthetic_token" "$operational_token"; do case "$value" in ''|*[!0-9a-f]*) exit 1 ;; esac test "${#value}" -eq 64 || exit 1 done -printf '{"options":{"cas":0},"data":{"token":"%s","synthetic_token":"%s"}}\n' \ - "$local_token" "$synthetic_token" > "$payload_file" -unset local_token synthetic_token value +printf '{"options":{"cas":0},"data":{"token":"%s","synthetic_token":"%s","operational_token":"%s"}}\n' \ + "$local_token" "$synthetic_token" "$operational_token" > "$payload_file" +unset local_token synthetic_token operational_token value vault write kv/data/atlas/hermes/suite-planning-api @"$payload_file" >/dev/null 2>&1 printf 'Suite credentials created.\n' diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh index 1364fc55..18520569 100644 --- a/services/vault/scripts/vault_k8s_auth_configure.sh +++ b/services/vault/scripts/vault_k8s_auth_configure.sh @@ -283,7 +283,7 @@ write_policy_and_role "hermes-model-gate" "hermes" "hermes-model-gate" \ write_policy_and_role "hermes-suite-planner" "hermes" "hermes-suite-planner" \ "hermes/suite-planning-api hermes/model-gate-lan-api hermes/agent-tokens" "" hermes_suite_seed_policy=' -path "kv/data/atlas/hermes/suite-planning-api" { capabilities = ["create", "read"] } +path "kv/data/atlas/hermes/suite-planning-api" { capabilities = ["create", "read", "patch"] } path "sys/tools/random/32" { capabilities = ["update"] } ' write_raw_policy "hermes-suite-token-seed" "${hermes_suite_seed_policy}" diff --git a/testing/tests/test_suite_planning.py b/testing/tests/test_suite_planning.py index 5ee862d5..93392243 100644 --- a/testing/tests/test_suite_planning.py +++ b/testing/tests/test_suite_planning.py @@ -61,7 +61,8 @@ def test_malformed_policy_rejected(policy): suite_api.authorize(request, ["claude"]) -def test_permissions_and_external_data_scope(): +def test_permissions_and_external_data_scope(monkeypatch): + monkeypatch.delenv("PLANNING_GENERALIZED_CLAUDE_APPROVED", raising=False) with pytest.raises(Problem, match="provider_forbidden"): suite_api.authorize(external(), []) request = external() @@ -70,6 +71,37 @@ def test_permissions_and_external_data_scope(): suite_api.authorize(request, ["claude"]) +def test_generalized_approval_remains_claude_and_credential_scoped(monkeypatch): + monkeypatch.setenv("PLANNING_GENERALIZED_CLAUDE_APPROVED", "true") + request = external() + request["cases"][0]["description"] = "Synthetic example of a generalized input" + with pytest.raises(Problem, match="external_data_not_approved"): + suite_api.authorize(request, ["claude"]) + assert suite_api.authorize(request, ["claude"], operational=True)["cases"] == request["cases"] + with pytest.raises(Problem, match="provider_forbidden"): + suite_api.authorize(request, [], operational=True) + for providers in (["codex"], ["claude", "codex"]): + request["routing"]["allowed_external_providers"] = providers + with pytest.raises(Problem, match="provider_forbidden"): + suite_api.authorize(request, providers, operational=True) + + +def test_local_schema_excludes_descriptions_from_members(monkeypatch): + from types import SimpleNamespace + captured = [] + request = validate_request({"campaign": "SYNTHETIC", "suite": "PARSER", "cases": [ + {"alias": "CASE-1", "description": "Parse valid configuration text"}]}, []) + def respond(url, value, *args): + captured.append(value) + return {"done": True, "model": MODELS["local"]["model"], "response": '{"groups":[]}'} + monkeypatch.setattr(suite_backends, "post", respond) + monkeypatch.setattr(suite_backends, "Path", lambda _: SimpleNamespace(read_text=lambda: "fake")) + suite_backends.local_generate(request, threading.Event(), "192.168.22.8") + items = captured[0]["format"]["properties"]["groups"]["items"]["properties"]["members"]["items"] + assert items["enum"] == ["CASE-1"] + assert "enum" not in suite_backends.SCHEMA["properties"]["groups"]["items"]["properties"]["members"]["items"] + + def test_local_default_cannot_overflow_to_provider(): request = validate_request(fixture(14)[0], ["claude", "codex"]) assert request["routing"] == {"allow_external": False, "allowed_external_providers": []} @@ -174,6 +206,37 @@ def test_compaction_and_incomplete_detection(): suite_backends.parse_claude("\n".join(json.dumps(e) for e in events), "claude-fable-5") +@pytest.mark.parametrize("failure,code", [ + (None, None), ("model", "model_changed"), + ("context", "backend_capabilities_changed"), + ("tools", "worker_isolation_failed"), ("output", "incomplete_generation"), +]) +def test_actual_cli_envelope_and_runtime_guards(failure, code): + model = MODELS["claude"]["model"] + init = {"type": "system", "subtype": "init", "model": model + "[1m]", + "tools": ["StructuredOutput"], "mcp_servers": [], "plugins": []} + limits = {"contextWindow": 1000000, "maxOutputTokens": 64000, + "canonicalModel": model, "provider": "firstParty"} + result = {"type": "result", "subtype": "success", "is_error": False, + "modelUsage": {model + "[1m]": limits}, "usage": {}, + "structured_output": expected_result(14)} + if failure == "model": + result["modelUsage"] = {"claude-unexpected": limits} + elif failure == "context": + limits["contextWindow"] = 200000 + elif failure == "tools": + init["tools"].append("Bash") + elif failure == "output": + result["stop_reason"] = "max_tokens" + raw = "\n".join(json.dumps(e) for e in [init, result]) + if code: + with pytest.raises(Problem, match=code): + suite_backends.parse_claude(raw, model) + else: + value, metadata = suite_backends.parse_claude(raw, model) + assert value == expected_result(14) and metadata["model"] == model + + @pytest.mark.parametrize("raw", ['{"routing":{},"routing":{}}', '{"x":NaN}', '{"x":Infinity}', '{']) def test_strict_json(raw): with pytest.raises(Problem, match="invalid_json"): @@ -183,7 +246,9 @@ def test_strict_json(raw): def test_credential_permissions(tmp_path): (tmp_path / "token").write_text("local-secret") (tmp_path / "synthetic-token").write_text("synthetic-secret") + (tmp_path / "operational-token").write_text("operational-secret") assert suite_api.credential("Bearer local-secret", tmp_path)[1] == [] assert suite_api.credential("Bearer synthetic-secret", tmp_path)[1] == ["claude", "codex"] + assert suite_api.credential("Bearer operational-secret", tmp_path)[1] == ["claude"] with pytest.raises(Problem, match="authentication"): suite_api.credential("Bearer invalid", tmp_path)