8c65b7bd60
Second entry in the action registry, proving it is a real extension point. No cluster mutation: the action is one Jenkins rebuild. - hermes_infra_signals: reviewable marker set across DNS/connectivity, image pull, upstream 5xx and agent-channel loss; Ariadne independently confirms a marker in the evidence before any retry, and records which marker justified it. "no space left on device" is deliberately excluded because a retry lands on the same full volume. - decision: classification -> action registry (action_classifications), falling back to the previous single-classification behavior - repair: retry_build posts to /build for unparameterized real jobs and buildWithParameters for the fixture demo job - orchestrator: retry path records requested/accepted/executed and moves the incident to awaiting_rebuild so the existing success path resolves it; one action per incident still enforced, so a retry cannot loop - events layer split out of the orchestrator to stay under the LOC cap Motivated by real failures tonight: pip DNS resolution and a Gitea 443 connect timeout, plus live incident metis/271 (SCM checkout timeout). 30 new tests; 368 pass in the hermes suite. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
161 lines
7.7 KiB
Python
161 lines
7.7 KiB
Python
from __future__ import annotations
|
|
|
|
from ariadne import settings as settings_module
|
|
from ariadne.settings import Settings
|
|
|
|
|
|
def test_env_int_invalid(monkeypatch) -> None:
|
|
monkeypatch.setenv("ARIADNE_INT_TEST", "bad")
|
|
assert settings_module._env_int("ARIADNE_INT_TEST", 5) == 5
|
|
|
|
|
|
def test_env_float_invalid(monkeypatch) -> None:
|
|
monkeypatch.setenv("ARIADNE_FLOAT_TEST", "bad")
|
|
assert settings_module._env_float("ARIADNE_FLOAT_TEST", 1.5) == 1.5
|
|
|
|
|
|
def test_from_env_includes_metis_settings(monkeypatch) -> None:
|
|
monkeypatch.setenv("METIS_BASE_URL", "http://metis.maintenance.svc.cluster.local/")
|
|
monkeypatch.setenv("METIS_WATCH_URL", "http://metis.example/internal/sentinel/watch")
|
|
monkeypatch.setenv("METIS_TIMEOUT_SEC", "9.5")
|
|
monkeypatch.setenv("ARIADNE_SCHEDULE_METIS_SENTINEL_WATCH", "*/7 * * * *")
|
|
|
|
cfg = Settings.from_env()
|
|
assert cfg.metis_base_url == "http://metis.maintenance.svc.cluster.local"
|
|
assert cfg.metis_watch_url == "http://metis.example/internal/sentinel/watch"
|
|
assert cfg.metis_timeout_sec == 9.5
|
|
assert cfg.metis_sentinel_watch_cron == "*/7 * * * *"
|
|
|
|
|
|
def test_from_env_includes_jenkins_weather_settings(monkeypatch) -> None:
|
|
monkeypatch.setenv("JENKINS_BASE_URL", "https://ci.bstein.dev/")
|
|
monkeypatch.setenv("JENKINS_API_USER", "ariadne")
|
|
monkeypatch.setenv("JENKINS_API_TOKEN", "token")
|
|
monkeypatch.setenv("JENKINS_API_TIMEOUT_SEC", "8.5")
|
|
monkeypatch.setenv("ARIADNE_SCHEDULE_JENKINS_BUILD_WEATHER", "*/9 * * * *")
|
|
monkeypatch.setenv("ARIADNE_SCHEDULE_TESTING_TRIAGE", "*/11 * * * *")
|
|
monkeypatch.setenv("ARIADNE_TESTING_TRIAGE_MODEL_URL", "http://openclaw-ollama:11434/")
|
|
monkeypatch.setenv("ARIADNE_TESTING_TRIAGE_MODEL", "local-model")
|
|
monkeypatch.setenv("ARIADNE_TESTING_TRIAGE_MODEL_TIMEOUT_SEC", "33.5")
|
|
|
|
cfg = Settings.from_env()
|
|
assert cfg.jenkins_base_url == "https://ci.bstein.dev"
|
|
assert cfg.jenkins_api_user == "ariadne"
|
|
assert cfg.jenkins_api_token == "token"
|
|
assert cfg.jenkins_api_timeout_sec == 8.5
|
|
assert cfg.jenkins_build_weather_cron == "*/9 * * * *"
|
|
assert cfg.testing_triage_cron == "*/11 * * * *"
|
|
assert cfg.testing_triage_model_url == "http://openclaw-ollama:11434"
|
|
assert cfg.testing_triage_model == "local-model"
|
|
assert cfg.testing_triage_model_timeout_sec == 33.5
|
|
|
|
|
|
def test_from_env_includes_game_stream_settings(monkeypatch) -> None:
|
|
monkeypatch.setenv("GAME_MODE_NODE_NAME", "titan-24")
|
|
monkeypatch.setenv(
|
|
"GAME_MODE_DISPLACE_WORKLOADS",
|
|
'[{"kind":"StatefulSet","namespace":"hermes","name":"hermes-llm","restoreReplicas":2}]',
|
|
)
|
|
monkeypatch.setenv("GAME_MODE_HOOK_TOKEN", "hook")
|
|
monkeypatch.setenv("GAME_MODE_LEASE_NAMESPACE", "hermes")
|
|
monkeypatch.setenv("GAME_MODE_LEASE_NAME", "titan-24-gpu-owner")
|
|
monkeypatch.setenv("GAME_MODE_OLLAMA_URL", "http://hermes-ollama:11434/")
|
|
monkeypatch.setenv("GAME_MODE_OLLAMA_MODEL", "gpt-oss:20b")
|
|
monkeypatch.setenv("GAME_MODE_OLLAMA_REQUEST_TIMEOUT_SEC", "600")
|
|
monkeypatch.setenv("GAME_MODE_TRANSITION_TIMEOUT_SEC", "700")
|
|
monkeypatch.setenv("GAME_MODE_POLL_INTERVAL_SEC", "0.5")
|
|
monkeypatch.setenv("WOLF_OIDC_CLIENT_ID", "wolf")
|
|
monkeypatch.setenv("WOLF_OIDC_BASE_URL", "https://wolf.bstein.dev/")
|
|
monkeypatch.setenv("WOLF_OIDC_VAULT_PATH", "game-stream/wolf-oidc")
|
|
monkeypatch.setenv("ARIADNE_SCHEDULE_WOLF_OIDC", "*/13 * * * *")
|
|
monkeypatch.setenv("WOLF_API_URL", "http://wolf-api.game-stream.svc.cluster.local:8088/")
|
|
monkeypatch.setenv("WOLF_GATEKEEPER_URL", "http://wolf-gatekeeper.game-stream.svc.cluster.local:8087/")
|
|
monkeypatch.setenv("GAME_STREAM_FIREWALL_UNLOCK_TTL_SEC", "28800")
|
|
monkeypatch.setenv("GAME_STREAM_MOONLIGHT_HOST", "moonlight.bstein.dev")
|
|
|
|
cfg = Settings.from_env()
|
|
|
|
assert cfg.game_mode_node_name == "titan-24"
|
|
assert cfg.game_mode_displace_workloads[0]["namespace"] == "hermes"
|
|
assert cfg.game_mode_hook_token == "hook"
|
|
assert cfg.game_mode_lease_namespace == "hermes"
|
|
assert cfg.game_mode_lease_name == "titan-24-gpu-owner"
|
|
assert cfg.game_mode_ollama_url == "http://hermes-ollama:11434"
|
|
assert cfg.game_mode_ollama_model == "gpt-oss:20b"
|
|
assert cfg.game_mode_ollama_request_timeout_sec == 600
|
|
assert cfg.game_mode_transition_timeout_sec == 700
|
|
assert cfg.game_mode_poll_interval_sec == 0.5
|
|
assert cfg.wolf_oidc_client_id == "wolf"
|
|
assert cfg.wolf_oidc_base_url == "https://wolf.bstein.dev"
|
|
assert cfg.wolf_oidc_vault_path == "game-stream/wolf-oidc"
|
|
assert cfg.wolf_oidc_cron == "*/13 * * * *"
|
|
assert cfg.wolf_api_url == "http://wolf-api.game-stream.svc.cluster.local:8088"
|
|
assert cfg.wolf_gatekeeper_url == "http://wolf-gatekeeper.game-stream.svc.cluster.local:8087"
|
|
assert cfg.game_stream_firewall_unlock_ttl_sec == 28800
|
|
assert cfg.game_stream_moonlight_host == "moonlight.bstein.dev"
|
|
|
|
|
|
def test_from_env_includes_hermes_code_settings(monkeypatch) -> None:
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_ENABLED", "true")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_JOB", "hermes-code-demo")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_OWNER", "bstein")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_REPO", "hermes-code-demo")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_BASE_BRANCH", "master")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_CANDIDATE_PATH", "src/discount.py")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_ALLOWED_PREFIXES", "src/, app/")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_ALLOWED_SUFFIXES", ".py")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_MAX_PATCH_BYTES", "2000")
|
|
monkeypatch.setenv("ARIADNE_HERMES_CODE_MAX_CHANGED_LINES", "10")
|
|
monkeypatch.setenv("ARIADNE_HERMES_GITEA_BASE_URL", "https://scm.bstein.dev/")
|
|
monkeypatch.setenv("ARIADNE_HERMES_GITEA_TOKEN", "token")
|
|
|
|
cfg = Settings.from_env()
|
|
|
|
assert cfg.hermes_code_enabled is True
|
|
assert cfg.hermes_code_job == "hermes-code-demo"
|
|
assert cfg.hermes_code_owner == "bstein"
|
|
assert cfg.hermes_code_repo == "hermes-code-demo"
|
|
assert cfg.hermes_code_base_branch == "master"
|
|
assert cfg.hermes_code_candidate_path == "src/discount.py"
|
|
assert cfg.hermes_code_allowed_prefixes == ["src/", "app/"]
|
|
assert cfg.hermes_code_allowed_suffixes == [".py"]
|
|
assert cfg.hermes_code_max_patch_bytes == 2000
|
|
assert cfg.hermes_code_max_changed_lines == 10
|
|
assert cfg.hermes_gitea_base_url == "https://scm.bstein.dev"
|
|
assert cfg.hermes_gitea_token == "token"
|
|
|
|
|
|
def test_from_env_includes_hermes_action_registry(monkeypatch) -> None:
|
|
monkeypatch.setenv("ARIADNE_HERMES_ALLOWED_ACTIONS", "repair_demo_fixture, retry_transient_infra")
|
|
monkeypatch.setenv("ARIADNE_HERMES_PARAMETERIZED_JOBS", "hermes-triage-demo, seeded-demo")
|
|
|
|
cfg = Settings.from_env()
|
|
|
|
assert cfg.hermes_allowed_actions == ["repair_demo_fixture", "retry_transient_infra"]
|
|
assert cfg.hermes_parameterized_jobs == ["hermes-triage-demo", "seeded-demo"]
|
|
assert cfg.hermes_action_classifications == {
|
|
"known_demo_fixture_failure": "repair_demo_fixture",
|
|
"transient_infra_failure": "retry_transient_infra",
|
|
}
|
|
|
|
|
|
def test_hermes_action_classifications_are_overridable_and_tolerant(monkeypatch) -> None:
|
|
monkeypatch.setenv(
|
|
"ARIADNE_HERMES_ACTION_CLASSIFICATIONS",
|
|
" transient_infra_failure = retry_transient_infra , broken , = , missing_value=",
|
|
)
|
|
|
|
cfg = Settings.from_env()
|
|
|
|
assert cfg.hermes_action_classifications == {"transient_infra_failure": "retry_transient_infra"}
|
|
|
|
|
|
def test_hermes_action_defaults_stay_conservative(monkeypatch) -> None:
|
|
monkeypatch.delenv("ARIADNE_HERMES_ALLOWED_ACTIONS", raising=False)
|
|
monkeypatch.delenv("ARIADNE_HERMES_PARAMETERIZED_JOBS", raising=False)
|
|
|
|
cfg = Settings.from_env()
|
|
|
|
assert cfg.hermes_allowed_actions == ["repair_demo_fixture"]
|
|
assert cfg.hermes_parameterized_jobs == ["hermes-triage-demo"]
|