Based on PR #15 (fix/hermes-result-decomposition-reliability); stacked on the decomposed cli_lane modules. - cli_lane_quota: soft-exclude a provider from NEW cli-auto work below the remaining-quota threshold (both-below prefers more remaining; fetch failure fails open with a metric). - cli_lane_health: lane now writes provider health (G7) with classified failure reasons splitting the capacity conflation (quota/auth/ rate-limit/transport) and cooldown hysteresis; re-admission only on full cooldown expiry, passed quota reset, or fresh success (G4). - cli_lane_routing: capacity-limited health now excludes a provider (G3); cooldown/reset-aware re-admission. - cli_lane_failover: explicit cli-codex-*/cli-claude-* assignees fail closed as transient instead of switching providers (G5); fallback depth stays bounded at two hosted providers (G1) with effort preserved; Switchyard outages block transient, not capability (G9). - cli_lane_metrics: route-decision/fallback counters, quota and soft-exclusion gauges, pod-local scrape server (G6). - cli_lane_provider: worker env drops ANTHROPIC_API_KEY, CLAUDE_API_KEY, OPENAI_API_KEY, API_SERVER_KEY so no metered path exists (G10). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
177 lines
4.3 KiB
Python
177 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
# ruff: noqa: F401
|
|
"""Run durable Codex and Claude workers from Hermes' authoritative Kanban."""
|
|
|
|
from __future__ import annotations
|
|
|
|
# The facade keeps the deployed entry point and compatibility API stable while
|
|
# domain modules own each independently testable reliability boundary.
|
|
import os
|
|
import sqlite3
|
|
import subprocess
|
|
import uuid
|
|
|
|
import cli_lane_goal
|
|
from cli_lane_capabilities import (
|
|
KanbanCapabilities,
|
|
_explicit_keyword,
|
|
detect_kanban_capabilities,
|
|
initialize_kanban_capabilities,
|
|
kanban_capabilities,
|
|
readiness_issue,
|
|
refresh_kanban_capabilities,
|
|
runtime_health,
|
|
)
|
|
from cli_lane_board import (
|
|
_board_call,
|
|
_external,
|
|
_record_board_access_error,
|
|
_resolve_workspace,
|
|
_task_context,
|
|
_task_value,
|
|
)
|
|
from cli_lane_config import (
|
|
ARTIFACT_GC_INTERVAL_SECONDS,
|
|
AUTH_COOLDOWN_SECONDS_DEFAULT,
|
|
CAPACITY_COOLDOWN_SECONDS_DEFAULT,
|
|
QUOTA_METRICS_URL,
|
|
QUOTA_MIN_REMAINING_PERCENT_DEFAULT,
|
|
kanban_setting,
|
|
ARTIFACT_RETENTION_AGE_SECONDS,
|
|
ARTIFACT_RETENTION_BYTES,
|
|
ARTIFACT_RETENTION_COUNT,
|
|
BOARD_CORRUPTION_ERRORS,
|
|
CLAUDE_BIN,
|
|
CODEX_BIN,
|
|
DATA_ROOT,
|
|
DEFAULT_CLAIM_TTL,
|
|
DEFAULT_MAX_RUNTIME,
|
|
MAX_TERMINAL_RECORD_BYTES,
|
|
PROVIDER_HEALTH_PATHS,
|
|
QUARANTINE_HASH_BYTES,
|
|
RESULT_SCHEMA,
|
|
RESULT_SCHEMA_PATH,
|
|
STATE_ROOT,
|
|
ProcessResult,
|
|
Route,
|
|
TerminalFinalizationPending,
|
|
TerminalIdentity,
|
|
canonical_run_id,
|
|
utc_now,
|
|
)
|
|
from cli_lane_dispatch import claim_ready, main, recover_orphans
|
|
from cli_lane_evidence import (
|
|
_persist_conflict_evidence,
|
|
_persist_prepared_evidence,
|
|
_rename_noreplace,
|
|
_retire_terminal_entry,
|
|
_same_terminal_document,
|
|
_terminal_document_digest,
|
|
_terminal_evidence_path,
|
|
_terminal_evidence_valid,
|
|
_write_json_noreplace,
|
|
)
|
|
from cli_lane_execution import execute_claim
|
|
from cli_lane_failover import (
|
|
FailoverOutcome,
|
|
_block_transient,
|
|
_routed_or_blocked,
|
|
capacity_failover,
|
|
)
|
|
from cli_lane_health import (
|
|
classify_capacity_failure,
|
|
record_provider_failure,
|
|
record_provider_success,
|
|
)
|
|
from cli_lane_metrics import (
|
|
METRICS,
|
|
record_provider_fallback,
|
|
record_provider_quota,
|
|
record_quota_fetch_failure,
|
|
record_route_decision,
|
|
record_router_selection_failure,
|
|
record_soft_exclusion,
|
|
start_metrics_server,
|
|
)
|
|
from cli_lane_quota import (
|
|
ProviderQuota,
|
|
SelectionConstraint,
|
|
fetch_quota_snapshot,
|
|
parse_quota_metrics,
|
|
quota_soft_exclusion,
|
|
selection_constraint,
|
|
)
|
|
from cli_lane_files import (
|
|
_candidate_path,
|
|
_fsync_directory,
|
|
_result_path,
|
|
_terminal_evidence_identity,
|
|
_terminal_identity,
|
|
_terminal_path,
|
|
atomic_json,
|
|
load_json,
|
|
state_path,
|
|
)
|
|
from cli_lane_finalization import (
|
|
_discard_evidence,
|
|
_finalize_document_db,
|
|
_finalize_terminal_record,
|
|
_promote_prepared_evidence,
|
|
_recover_exact_run,
|
|
_resolve_pending_after_winner,
|
|
_retire_snapshot,
|
|
_retire_snapshot_after_db,
|
|
)
|
|
from cli_lane_prompt import (
|
|
_event_payload,
|
|
_extract_json,
|
|
build_prompt,
|
|
git_handoff,
|
|
workspace_artifacts,
|
|
)
|
|
from cli_lane_provider import (
|
|
_base_env,
|
|
_claude_command,
|
|
_codex_command,
|
|
_descendant_processes,
|
|
_process_identity_matches,
|
|
_process_record,
|
|
_signal_worker_tree,
|
|
_terminate_worker_process,
|
|
run_provider,
|
|
stream_process,
|
|
)
|
|
from cli_lane_quarantine import _quarantine_terminal
|
|
from cli_lane_records import (
|
|
_load_small_json,
|
|
_load_terminal_json,
|
|
_open_small_json_snapshot,
|
|
_open_terminal_recovery_snapshot,
|
|
_open_terminal_snapshot,
|
|
_persist_candidate,
|
|
_read_bounded,
|
|
_terminal_record_valid,
|
|
_write_terminal_record,
|
|
)
|
|
from cli_lane_recovery import (
|
|
_drain_retirement_staging,
|
|
_has_pending_finalization,
|
|
_recover_prepared_finalizations,
|
|
_recover_retirement_staging,
|
|
_restore_staged_terminal_authority,
|
|
_staged_terminal_authority,
|
|
_terminal_entry_absent,
|
|
recover_pending_finalizations,
|
|
)
|
|
from cli_lane_retention import (
|
|
_artifact_gc_candidates,
|
|
_unlink_artifact_if_same,
|
|
gc_lane_artifacts,
|
|
maybe_gc_lane_artifacts,
|
|
)
|
|
from cli_lane_routing import fresh_unavailable_provider, parse_assignee, select_route
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|