220 lines
7.4 KiB
Python
220 lines
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""Bounded, fail-closed cross-provider failover for one capacity boundary."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
from typing import Any, Callable
|
|
|
|
from cli_lane_board import _board_call
|
|
from cli_lane_config import ProcessResult, Route
|
|
from cli_lane_health import classify_capacity_failure, record_provider_failure
|
|
from cli_lane_metrics import (
|
|
record_provider_fallback,
|
|
record_route_decision,
|
|
record_router_selection_failure,
|
|
)
|
|
from cli_lane_prompt import build_prompt, git_handoff
|
|
from cli_lane_provider import run_provider
|
|
from cli_lane_records import _persist_candidate
|
|
from cli_lane_routing import select_route
|
|
|
|
|
|
@dataclass
|
|
class FailoverOutcome:
|
|
"""Where one capacity boundary left the claim's route and result."""
|
|
|
|
route: Route
|
|
result: ProcessResult
|
|
candidate_file: Path | None
|
|
unavailable_provider: str | None
|
|
blocked: bool
|
|
|
|
|
|
def _block_transient(
|
|
kanban_db: Any, board: str, task_id: str, run_id: Any, reason: str
|
|
) -> None:
|
|
"""Park one card as retryable so it re-enters when providers recover."""
|
|
_board_call(
|
|
kanban_db,
|
|
board,
|
|
lambda fresh: kanban_db.block_task(
|
|
fresh,
|
|
task_id,
|
|
reason=reason,
|
|
kind="transient",
|
|
expected_run_id=run_id,
|
|
),
|
|
)
|
|
|
|
|
|
def _routed_or_blocked(
|
|
kanban_db: Any,
|
|
board: str,
|
|
task_id: str,
|
|
run_id: Any,
|
|
select: Callable[[], Route],
|
|
) -> Route | None:
|
|
"""Turn a Switchyard outage into a transient block, never a capability one."""
|
|
try:
|
|
return select()
|
|
except RuntimeError as error:
|
|
record_router_selection_failure()
|
|
_block_transient(
|
|
kanban_db,
|
|
board,
|
|
task_id,
|
|
run_id,
|
|
"Switchyard route selection is unavailable; the card retries "
|
|
f"when the router recovers: {error}",
|
|
)
|
|
return None
|
|
|
|
|
|
def _manual_floor_assignee(provider: str, route: Route) -> str:
|
|
"""Pin the exact recovery role and effort without legacy inference."""
|
|
return f"cli-{provider}-{route.capability}-{route.effort}"
|
|
|
|
|
|
def capacity_failover(
|
|
kanban_db: Any,
|
|
*,
|
|
board: str,
|
|
task_id: str,
|
|
run_id: Any,
|
|
assignee: str,
|
|
comment: Callable[[str], None],
|
|
context: str,
|
|
workspace: Path,
|
|
state: dict[str, Any],
|
|
state_file: Path,
|
|
log_path: Path,
|
|
heartbeat: Callable[[str], bool],
|
|
deadline: float,
|
|
route: Route,
|
|
result: ProcessResult,
|
|
candidate_file: Path | None,
|
|
goal_turn: int,
|
|
) -> FailoverOutcome:
|
|
"""Apply the capacity policy once: record, fail closed, or fail over."""
|
|
failure_class = classify_capacity_failure(result.output)
|
|
record_provider_failure(route.provider, failure_class)
|
|
record_route_decision(
|
|
route.provider, route.effort, route.classifier, "capacity-failure"
|
|
)
|
|
if assignee != "cli-auto":
|
|
# Fail closed: an operator pinned this provider, so a capacity or
|
|
# auth failure must wait for that provider, never switch away.
|
|
_block_transient(
|
|
kanban_db,
|
|
board,
|
|
task_id,
|
|
run_id,
|
|
f"Manually pinned provider {route.provider} hit a "
|
|
f"{failure_class} failure; the explicit {assignee} lane never "
|
|
"switches providers. Retry after recovery or reassign to cli-auto.",
|
|
)
|
|
return FailoverOutcome(route, result, candidate_file, None, True)
|
|
retry_context = (
|
|
context
|
|
+ "\n\nRouting boundary: the first provider failed from capacity/authentication. "
|
|
+ f"Preserve capability={route.capability} and effort={route.effort}; "
|
|
+ "select the alternate hosted provider without treating infrastructure failure as a quality miss."
|
|
)
|
|
fallback = _routed_or_blocked(
|
|
kanban_db,
|
|
board,
|
|
task_id,
|
|
run_id,
|
|
lambda: select_route(
|
|
retry_context,
|
|
assignee,
|
|
exclude_provider=route.provider,
|
|
exclude_reason=f"hit a {failure_class} failure at this boundary",
|
|
),
|
|
)
|
|
if fallback is None:
|
|
return FailoverOutcome(route, result, candidate_file, route.provider, True)
|
|
if fallback.effort != route.effort or fallback.capability != route.capability:
|
|
# Infrastructure failures preserve both floors exactly. A higher paid
|
|
# role or effort is a quality escalation and must not be inferred from
|
|
# a timeout, quota, authentication, or provider-capacity incident.
|
|
# re-pin the classifier's chosen (healthy) provider at the original
|
|
# floor, whether the downgrade came from the classifier itself or
|
|
# from its own excluded-provider health guard.
|
|
preserved = _routed_or_blocked(
|
|
kanban_db,
|
|
board,
|
|
task_id,
|
|
run_id,
|
|
lambda: select_route(
|
|
retry_context,
|
|
_manual_floor_assignee(fallback.provider, route),
|
|
),
|
|
)
|
|
if preserved is None:
|
|
return FailoverOutcome(route, result, candidate_file, route.provider, True)
|
|
comment(
|
|
f"Route floors preserved: Switchyard classification chose "
|
|
f"{fallback.capability}/{fallback.effort} for {fallback.provider}; "
|
|
f"restored the original {route.capability}/{route.effort} floor so "
|
|
"infrastructure failover "
|
|
"never changes capability or effort.",
|
|
)
|
|
fallback = preserved
|
|
record_provider_fallback(route.provider, fallback.provider, failure_class)
|
|
comment(
|
|
f"Provider fallback: {route.provider} -> {fallback.provider} "
|
|
f"after a {failure_class} failure; preserved "
|
|
f"{fallback.capability}/{fallback.effort} (classifier={fallback.classifier}).",
|
|
)
|
|
fallback_result = run_provider(
|
|
fallback,
|
|
build_prompt(
|
|
context,
|
|
workspace,
|
|
git_handoff(workspace, result.output),
|
|
),
|
|
workspace,
|
|
state,
|
|
state_file,
|
|
log_path,
|
|
heartbeat,
|
|
max(1, int(deadline - time.monotonic())),
|
|
)
|
|
if fallback_result.capacity_failure:
|
|
# Bounded failover depth: two hosted providers, never a third
|
|
# metered or local path.
|
|
fallback_class = classify_capacity_failure(fallback_result.output)
|
|
record_provider_failure(fallback.provider, fallback_class)
|
|
record_route_decision(
|
|
fallback.provider, fallback.effort, fallback.classifier, "capacity-failure"
|
|
)
|
|
_block_transient(
|
|
kanban_db,
|
|
board,
|
|
task_id,
|
|
run_id,
|
|
"Both hosted providers failed at this boundary "
|
|
f"({route.provider}: {failure_class}; {fallback.provider}: "
|
|
f"{fallback_class}); the lane never falls to a metered or "
|
|
"local path. Waiting for provider recovery.",
|
|
)
|
|
return FailoverOutcome(
|
|
fallback, fallback_result, candidate_file, route.provider, True
|
|
)
|
|
if fallback_result.structured:
|
|
candidate_file = _persist_candidate(
|
|
state,
|
|
state_file,
|
|
dict(fallback_result.structured),
|
|
route=fallback,
|
|
returncode=fallback_result.returncode,
|
|
goal_turn=goal_turn,
|
|
)
|
|
return FailoverOutcome(
|
|
fallback, fallback_result, candidate_file, route.provider, False
|
|
)
|