atlas-iac/services/hermes/scripts/cli_lane_runner.py
jenkins 034c8372c7 hermes: enforce quota-aware fail-closed cli-auto provider routing
Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked
on the decomposed cli_lane modules.

- cli_lane_quota: soft-exclude a provider from NEW cli-auto work below
  the remaining-quota threshold (both-below prefers more remaining;
  fetch failure fails open with a metric).
- cli_lane_health: lane now writes provider health (G7) with classified
  failure reasons splitting the capacity conflation (quota/auth/
  rate-limit/transport) and cooldown hysteresis; re-admission only on
  full cooldown expiry, passed quota reset, or fresh success (G4).
- cli_lane_routing: capacity-limited health now excludes a provider
  (G3); cooldown/reset-aware re-admission.
- cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail
  closed as transient instead of switching providers (G5); fallback
  depth stays bounded at two hosted providers (G1) with effort
  preserved; Switchyard outages block transient, not capability (G9).
- cli_lane_metrics: route-decision/fallback counters, quota and
  soft-exclusion gauges, pod-local scrape server (G6).
- cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY,
  OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-17 20:31:05 -03:00

177 lines
4.3 KiB
Python

#!/usr/bin/env python3
# ruff: noqa: F401
"""Run durable Codex and Claude workers from Hermes' authoritative Kanban."""
from __future__ import annotations
# The facade keeps the deployed entry point and compatibility API stable while
# domain modules own each independently testable reliability boundary.
import os
import sqlite3
import subprocess
import uuid
import cli_lane_goal
from cli_lane_capabilities import (
KanbanCapabilities,
_explicit_keyword,
detect_kanban_capabilities,
initialize_kanban_capabilities,
kanban_capabilities,
readiness_issue,
refresh_kanban_capabilities,
runtime_health,
)
from cli_lane_board import (
_board_call,
_external,
_record_board_access_error,
_resolve_workspace,
_task_context,
_task_value,
)
from cli_lane_config import (
ARTIFACT_GC_INTERVAL_SECONDS,
AUTH_COOLDOWN_SECONDS_DEFAULT,
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
QUOTA_METRICS_URL,
QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
kanban_setting,
ARTIFACT_RETENTION_AGE_SECONDS,
ARTIFACT_RETENTION_BYTES,
ARTIFACT_RETENTION_COUNT,
BOARD_CORRUPTION_ERRORS,
CLAUDE_BIN,
CODEX_BIN,
DATA_ROOT,
DEFAULT_CLAIM_TTL,
DEFAULT_MAX_RUNTIME,
MAX_TERMINAL_RECORD_BYTES,
PROVIDER_HEALTH_PATHS,
QUARANTINE_HASH_BYTES,
RESULT_SCHEMA,
RESULT_SCHEMA_PATH,
STATE_ROOT,
ProcessResult,
Route,
TerminalFinalizationPending,
TerminalIdentity,
canonical_run_id,
utc_now,
)
from cli_lane_dispatch import claim_ready, main, recover_orphans
from cli_lane_evidence import (
_persist_conflict_evidence,
_persist_prepared_evidence,
_rename_noreplace,
_retire_terminal_entry,
_same_terminal_document,
_terminal_document_digest,
_terminal_evidence_path,
_terminal_evidence_valid,
_write_json_noreplace,
)
from cli_lane_execution import execute_claim
from cli_lane_failover import (
FailoverOutcome,
_block_transient,
_routed_or_blocked,
capacity_failover,
)
from cli_lane_health import (
classify_capacity_failure,
record_provider_failure,
record_provider_success,
)
from cli_lane_metrics import (
METRICS,
record_provider_fallback,
record_provider_quota,
record_quota_fetch_failure,
record_route_decision,
record_router_selection_failure,
record_soft_exclusion,
start_metrics_server,
)
from cli_lane_quota import (
ProviderQuota,
SelectionConstraint,
fetch_quota_snapshot,
parse_quota_metrics,
quota_soft_exclusion,
selection_constraint,
)
from cli_lane_files import (
_candidate_path,
_fsync_directory,
_result_path,
_terminal_evidence_identity,
_terminal_identity,
_terminal_path,
atomic_json,
load_json,
state_path,
)
from cli_lane_finalization import (
_discard_evidence,
_finalize_document_db,
_finalize_terminal_record,
_promote_prepared_evidence,
_recover_exact_run,
_resolve_pending_after_winner,
_retire_snapshot,
_retire_snapshot_after_db,
)
from cli_lane_prompt import (
_event_payload,
_extract_json,
build_prompt,
git_handoff,
workspace_artifacts,
)
from cli_lane_provider import (
_base_env,
_claude_command,
_codex_command,
_descendant_processes,
_process_identity_matches,
_process_record,
_signal_worker_tree,
_terminate_worker_process,
run_provider,
stream_process,
)
from cli_lane_quarantine import _quarantine_terminal
from cli_lane_records import (
_load_small_json,
_load_terminal_json,
_open_small_json_snapshot,
_open_terminal_recovery_snapshot,
_open_terminal_snapshot,
_persist_candidate,
_read_bounded,
_terminal_record_valid,
_write_terminal_record,
)
from cli_lane_recovery import (
_drain_retirement_staging,
_has_pending_finalization,
_recover_prepared_finalizations,
_recover_retirement_staging,
_restore_staged_terminal_authority,
_staged_terminal_authority,
_terminal_entry_absent,
recover_pending_finalizations,
)
from cli_lane_retention import (
_artifact_gc_candidates,
_unlink_artifact_if_same,
gc_lane_artifacts,
maybe_gc_lane_artifacts,
)
from cli_lane_routing import fresh_unavailable_provider, parse_assignee, select_route
if __name__ == "__main__":
raise SystemExit(main())