Compare commits
2 Commits
73fefbb5d9
...
df75479069
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
df75479069 | ||
|
|
e3ecc18d0e |
@ -93,9 +93,8 @@ def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
|
|||||||
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
|
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
|
||||||
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
|
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
|
||||||
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
|
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
|
||||||
|
# _line_number never returns below its default, so end_line >= start_line.
|
||||||
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
|
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
|
||||||
if end_line < start_line:
|
|
||||||
end_line = start_line
|
|
||||||
return {
|
return {
|
||||||
"engineId": "semgrep",
|
"engineId": "semgrep",
|
||||||
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
|
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
|
||||||
|
|||||||
96
scripts/tests/test_mailu_sync_edges.py
Normal file
96
scripts/tests/test_mailu_sync_edges.py
Normal file
@ -0,0 +1,96 @@
|
|||||||
|
"""Retry, attribute-corruption, and skip-path edges for the Mailu sync job."""
|
||||||
|
|
||||||
|
from test_mailu_sync import _FakeResponse, _FakeSession, load_sync_module
|
||||||
|
|
||||||
|
def test_retry_helpers_with_zero_attempts_do_nothing(monkeypatch):
|
||||||
|
sync = load_sync_module(monkeypatch)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sync.psycopg2,
|
||||||
|
"connect",
|
||||||
|
lambda **kwargs: (_ for _ in ()).throw(AssertionError("must not connect")),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert sync.retry_request(
|
||||||
|
"request",
|
||||||
|
lambda: (_ for _ in ()).throw(AssertionError("must not run")),
|
||||||
|
attempts=0,
|
||||||
|
) is None
|
||||||
|
assert sync.retry_db_connect(attempts=0) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_kc_update_attributes_replaces_a_corrupt_attribute_map(monkeypatch):
|
||||||
|
sync = load_sync_module(monkeypatch)
|
||||||
|
current_resp = _FakeResponse({"username": "u1", "attributes": "corrupt"})
|
||||||
|
ok_resp = _FakeResponse({"attributes": {"mailu_app_password": ["abc"]}})
|
||||||
|
sync.SESSION = _FakeSession(_FakeResponse({}), [current_resp, ok_resp])
|
||||||
|
|
||||||
|
sync.kc_update_attributes(
|
||||||
|
"token",
|
||||||
|
{"id": "u1", "username": "u1"},
|
||||||
|
{"mailu_app_password": "abc"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert sync.SESSION.put_called and sync.SESSION.get_calls == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_main_skips_disabled_users_and_settled_attribute_sets(monkeypatch):
|
||||||
|
sync = load_sync_module(monkeypatch)
|
||||||
|
monkeypatch.setattr(sync.bcrypt_sha256, "hash", lambda password: f"hash:{password}")
|
||||||
|
users = [
|
||||||
|
{
|
||||||
|
"id": "u_disabled",
|
||||||
|
"username": "disabled",
|
||||||
|
"email": "disabled@example.com",
|
||||||
|
"enabled": False,
|
||||||
|
"attributes": {"mailu_enabled": ["true"]},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": "u_settled",
|
||||||
|
"username": "settled",
|
||||||
|
"email": "settled@example.com",
|
||||||
|
"attributes": {
|
||||||
|
"mailu_enabled": ["true"],
|
||||||
|
"mailu_email": ["settled@example.com"],
|
||||||
|
"mailu_app_password": ["keepme"],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]
|
||||||
|
updated = []
|
||||||
|
|
||||||
|
class _Cursor:
|
||||||
|
def __init__(self):
|
||||||
|
self.executions = []
|
||||||
|
|
||||||
|
def execute(self, sql, params):
|
||||||
|
self.executions.append(params)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
class _Conn:
|
||||||
|
def __init__(self):
|
||||||
|
self.autocommit = False
|
||||||
|
self._cursor = _Cursor()
|
||||||
|
|
||||||
|
def cursor(self, cursor_factory=None):
|
||||||
|
return self._cursor
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
conn = _Conn()
|
||||||
|
monkeypatch.setattr(sync, "get_kc_token", lambda: "tok")
|
||||||
|
monkeypatch.setattr(sync, "kc_get_users", lambda token: users)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sync,
|
||||||
|
"kc_update_attributes",
|
||||||
|
lambda token, user, attrs: updated.append(user["id"]),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(sync.psycopg2, "connect", lambda **kwargs: conn)
|
||||||
|
|
||||||
|
sync.main()
|
||||||
|
|
||||||
|
assert updated == []
|
||||||
|
assert [params["email"] for params in conn._cursor.executions] == [
|
||||||
|
"settled@example.com"
|
||||||
|
]
|
||||||
@ -132,3 +132,47 @@ def test_listener_log_message_is_quiet(monkeypatch):
|
|||||||
handler = listener.Handler.__new__(listener.Handler)
|
handler = listener.Handler.__new__(listener.Handler)
|
||||||
|
|
||||||
assert handler.log_message("ignored %s", "value") is None
|
assert handler.log_message("ignored %s", "value") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_listener_post_treats_non_object_payloads_as_plain_triggers(monkeypatch):
|
||||||
|
listener = load_listener_module(monkeypatch)
|
||||||
|
called = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
listener,
|
||||||
|
"_trigger_sync_async",
|
||||||
|
lambda force=False: called.append(force) or True,
|
||||||
|
)
|
||||||
|
handler = _handler_for(listener, "[1, 2, 3]")
|
||||||
|
|
||||||
|
handler.do_POST()
|
||||||
|
|
||||||
|
assert called == [False]
|
||||||
|
assert handler.responses == [202]
|
||||||
|
|
||||||
|
|
||||||
|
def test_listener_module_serves_forever_when_run_directly(monkeypatch):
|
||||||
|
import http.server
|
||||||
|
import runpy
|
||||||
|
|
||||||
|
served = []
|
||||||
|
|
||||||
|
class _FakeServer:
|
||||||
|
def __init__(self, address, handler):
|
||||||
|
served.append((address, handler.__name__))
|
||||||
|
|
||||||
|
def serve_forever(self):
|
||||||
|
served.append("serving")
|
||||||
|
|
||||||
|
monkeypatch.setenv("MAILU_SYNC_WAIT_TIMEOUT_SEC", "0")
|
||||||
|
monkeypatch.setattr(http.server, "ThreadingHTTPServer", _FakeServer)
|
||||||
|
module_path = (
|
||||||
|
pathlib.Path(__file__).resolve().parents[2]
|
||||||
|
/ "services"
|
||||||
|
/ "mailu"
|
||||||
|
/ "scripts"
|
||||||
|
/ "mailu_sync_listener.py"
|
||||||
|
)
|
||||||
|
|
||||||
|
runpy.run_path(str(module_path), run_name="__main__")
|
||||||
|
|
||||||
|
assert served == [(("", 8080), "Handler"), "serving"]
|
||||||
|
|||||||
@ -875,6 +875,7 @@ spec:
|
|||||||
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
|
||||||
- {name: PYTHONPATH, value: /opt/hermes}
|
- {name: PYTHONPATH, value: /opt/hermes}
|
||||||
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
|
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
|
||||||
|
- {name: HERMES_CLI_HEALTH_MAX_AGE_SECONDS, value: "60"}
|
||||||
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
|
||||||
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
|
||||||
securityContext:
|
securityContext:
|
||||||
@ -895,6 +896,15 @@ spec:
|
|||||||
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
|
- {name: kubeconfig, mountPath: /opt/data/home/.kube/config, subPath: config, readOnly: true}
|
||||||
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
|
- {name: auto-router-plugin, mountPath: /opt/data/plugins/auto-router, readOnly: true}
|
||||||
- {name: tmp, mountPath: /tmp}
|
- {name: tmp, mountPath: /tmp}
|
||||||
|
readinessProbe:
|
||||||
|
exec:
|
||||||
|
command:
|
||||||
|
- /opt/hermes/.venv/bin/python
|
||||||
|
- /opt/coordinator/cli_lane_capabilities.py
|
||||||
|
initialDelaySeconds: 2
|
||||||
|
periodSeconds: 5
|
||||||
|
timeoutSeconds: 2
|
||||||
|
failureThreshold: 1
|
||||||
resources:
|
resources:
|
||||||
requests: {cpu: 100m, memory: 256Mi}
|
requests: {cpu: 100m, memory: 256Mi}
|
||||||
limits: {cpu: "2", memory: 6Gi}
|
limits: {cpu: "2", memory: 6Gi}
|
||||||
|
|||||||
@ -1,11 +1,38 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Detect and publish the Kanban API boundary required by CLI lanes."""
|
"""Detect and publish the Kanban API boundary required by CLI lanes.
|
||||||
|
|
||||||
|
During a rollout the coordinator container and the cli-lane-runner worker can
|
||||||
|
run different hermes-agent image generations against one shared Kanban DB.
|
||||||
|
The image APIs that differ on the decomposition/finalization path:
|
||||||
|
|
||||||
|
* ``complete_task``: both generations accept ``expected_run_id`` (active
|
||||||
|
exact-run completion); only the patched image accepts
|
||||||
|
``replay_ended_run_id`` (ended-run replay).
|
||||||
|
* ``reclaim_task``: only the patched image accepts ``expected_run_id``
|
||||||
|
(guarded orphan reclaim after a runner restart).
|
||||||
|
* ``specify_triage_task``/``decompose_triage_task``: only the patched image
|
||||||
|
accepts ``require_no_runs`` (refuses redefining tasks with run history).
|
||||||
|
|
||||||
|
Worker-side handling per combination (proven in
|
||||||
|
testing/tests/test_hermes_cli_capabilities.py):
|
||||||
|
|
||||||
|
* legacy worker: health/readiness stay deferred, no new claims, no unguarded
|
||||||
|
reclaim, replay-needing results stay journaled for a patched successor.
|
||||||
|
* patched worker: ready health, exact completion, guarded replay/reclaim; a
|
||||||
|
run voided by a legacy coordinator's unguarded mid-run decomposition
|
||||||
|
converges to stale/conflict evidence instead of completing.
|
||||||
|
* patched coordinator: mid-run decomposition is refused entirely, so the
|
||||||
|
active exact run completes on either worker generation.
|
||||||
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import inspect
|
import inspect
|
||||||
|
import json
|
||||||
|
import os
|
||||||
import sys
|
import sys
|
||||||
from dataclasses import asdict, dataclass
|
from dataclasses import asdict, dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
|
|
||||||
@ -85,6 +112,7 @@ def _health_document(capabilities: KanbanCapabilities) -> dict[str, Any]:
|
|||||||
"""Return a bounded operator-readable compatibility health document."""
|
"""Return a bounded operator-readable compatibility health document."""
|
||||||
return {
|
return {
|
||||||
"component": "cli-lane-runner",
|
"component": "cli-lane-runner",
|
||||||
|
"schema_version": 1,
|
||||||
"state": "ready" if capabilities.ready else "deferred",
|
"state": "ready" if capabilities.ready else "deferred",
|
||||||
"ready": capabilities.ready,
|
"ready": capabilities.ready,
|
||||||
"active_run_completion_safe": capabilities.exact_run_completion,
|
"active_run_completion_safe": capabilities.exact_run_completion,
|
||||||
@ -117,6 +145,21 @@ def initialize_kanban_capabilities(
|
|||||||
return capabilities
|
return capabilities
|
||||||
|
|
||||||
|
|
||||||
|
def refresh_kanban_capabilities(
|
||||||
|
kanban_db: Any,
|
||||||
|
*,
|
||||||
|
health_path: Path | None = None,
|
||||||
|
) -> KanbanCapabilities:
|
||||||
|
"""Reinspect the live callables and refresh bounded readiness evidence.
|
||||||
|
|
||||||
|
ConfigMap script updates and image changes are deliberately not imported or
|
||||||
|
reloaded here. A running process observes only the already-loaded module,
|
||||||
|
which prevents a half-old/half-new Python module graph. Tests may replace
|
||||||
|
the two callables to model an image API transition at a loop boundary.
|
||||||
|
"""
|
||||||
|
return initialize_kanban_capabilities(kanban_db, health_path=health_path)
|
||||||
|
|
||||||
|
|
||||||
def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
|
def kanban_capabilities(kanban_db: Any) -> KanbanCapabilities:
|
||||||
"""Return startup evidence, redetecting only when the loaded API changes."""
|
"""Return startup evidence, redetecting only when the loaded API changes."""
|
||||||
if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
|
if _CAPABILITIES is None or _source_identity(kanban_db) != _CAPABILITY_SOURCE:
|
||||||
@ -128,3 +171,68 @@ def runtime_health(path: Path | None = None) -> dict[str, Any]:
|
|||||||
"""Read the bounded health document for probes and operator diagnostics."""
|
"""Read the bounded health document for probes and operator diagnostics."""
|
||||||
document = load_json(path or STATE_ROOT / "runtime-health.json")
|
document = load_json(path or STATE_ROOT / "runtime-health.json")
|
||||||
return document if isinstance(document, dict) else {}
|
return document if isinstance(document, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def readiness_issue(
|
||||||
|
path: Path | None = None,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
maximum_age_seconds: float = 60.0,
|
||||||
|
) -> str | None:
|
||||||
|
"""Return a bounded readiness reason without exposing health contents."""
|
||||||
|
destination = path or STATE_ROOT / "runtime-health.json"
|
||||||
|
try:
|
||||||
|
document = json.loads(destination.read_text(encoding="utf-8"))
|
||||||
|
except FileNotFoundError:
|
||||||
|
return "runtime health is missing"
|
||||||
|
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||||
|
return "runtime health is malformed"
|
||||||
|
if not isinstance(document, dict) or document.get("schema_version") != 1:
|
||||||
|
return "runtime health is malformed"
|
||||||
|
if document.get("state") == "deferred" and document.get("ready") is False:
|
||||||
|
return "runtime compatibility is deferred"
|
||||||
|
capabilities = document.get("capabilities")
|
||||||
|
if (
|
||||||
|
document.get("state") != "ready"
|
||||||
|
or document.get("ready") is not True
|
||||||
|
or not isinstance(capabilities, dict)
|
||||||
|
or any(
|
||||||
|
capabilities.get(name) is not True
|
||||||
|
for name in (
|
||||||
|
"exact_run_completion",
|
||||||
|
"ended_run_replay",
|
||||||
|
"exact_run_reclaim",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
):
|
||||||
|
return "runtime health is malformed"
|
||||||
|
observed_at = document.get("observed_at")
|
||||||
|
if not isinstance(observed_at, str):
|
||||||
|
return "runtime health is malformed"
|
||||||
|
try:
|
||||||
|
observed = datetime.fromisoformat(observed_at.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return "runtime health is malformed"
|
||||||
|
if observed.tzinfo is None:
|
||||||
|
return "runtime health is malformed"
|
||||||
|
current = now or datetime.now(timezone.utc)
|
||||||
|
age = (
|
||||||
|
current.astimezone(timezone.utc) - observed.astimezone(timezone.utc)
|
||||||
|
).total_seconds()
|
||||||
|
if age < -5.0 or age > maximum_age_seconds:
|
||||||
|
return "runtime health is stale"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
"""Provide the non-mutating Kubernetes readiness command."""
|
||||||
|
maximum_age = float(os.environ.get("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "60"))
|
||||||
|
issue = readiness_issue(maximum_age_seconds=maximum_age)
|
||||||
|
if issue:
|
||||||
|
print(issue, file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
|
|||||||
@ -10,7 +10,11 @@ import time
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from cli_lane_board import _external, _record_board_access_error, _task_value
|
from cli_lane_board import _external, _record_board_access_error, _task_value
|
||||||
from cli_lane_capabilities import initialize_kanban_capabilities, kanban_capabilities
|
from cli_lane_capabilities import (
|
||||||
|
initialize_kanban_capabilities,
|
||||||
|
kanban_capabilities,
|
||||||
|
refresh_kanban_capabilities,
|
||||||
|
)
|
||||||
from cli_lane_config import (
|
from cli_lane_config import (
|
||||||
BOARD_CORRUPTION_ERRORS,
|
BOARD_CORRUPTION_ERRORS,
|
||||||
DEFAULT_CLAIM_TTL,
|
DEFAULT_CLAIM_TTL,
|
||||||
@ -29,6 +33,7 @@ def _board_slug(board: Any) -> str:
|
|||||||
return str(board.get("slug") or board.get("id") or "")
|
return str(board.get("slug") or board.get("id") or "")
|
||||||
return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
|
return str(getattr(board, "slug", None) or getattr(board, "id", None) or board)
|
||||||
|
|
||||||
|
|
||||||
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
||||||
"""Open one board without letting localized storage faults stop other lanes."""
|
"""Open one board without letting localized storage faults stop other lanes."""
|
||||||
try:
|
try:
|
||||||
@ -37,6 +42,7 @@ def _connect_healthy_board(kanban_db: Any, board: str) -> Any | None:
|
|||||||
_record_board_access_error(board, error)
|
_record_board_access_error(board, error)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def recover_orphans() -> None:
|
def recover_orphans() -> None:
|
||||||
"""Return external running tasks to ready after a runner/pod restart."""
|
"""Return external running tasks to ready after a runner/pod restart."""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
@ -59,7 +65,10 @@ def recover_orphans() -> None:
|
|||||||
continue
|
continue
|
||||||
try:
|
try:
|
||||||
for task in kanban_db.list_tasks(conn):
|
for task in kanban_db.list_tasks(conn):
|
||||||
if _external(task) and str(_task_value(task, "status", "")) == "running":
|
if (
|
||||||
|
_external(task)
|
||||||
|
and str(_task_value(task, "status", "")) == "running"
|
||||||
|
):
|
||||||
task_id = str(_task_value(task, "id"))
|
task_id = str(_task_value(task, "id"))
|
||||||
run_id = _task_value(task, "current_run_id", None)
|
run_id = _task_value(task, "current_run_id", None)
|
||||||
if not isinstance(run_id, int):
|
if not isinstance(run_id, int):
|
||||||
@ -78,6 +87,7 @@ def recover_orphans() -> None:
|
|||||||
finally:
|
finally:
|
||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]:
|
def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str]]:
|
||||||
"""Atomically claim external ready tasks across all non-archived boards."""
|
"""Atomically claim external ready tasks across all non-archived boards."""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
@ -134,8 +144,13 @@ def claim_ready(active: set[tuple[str, str]], limit: int) -> list[tuple[str, str
|
|||||||
conn.close()
|
conn.close()
|
||||||
return claimed
|
return claimed
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
"""Continuously bridge external Kanban lanes to provider CLIs."""
|
"""Continuously bridge external Kanban lanes to provider CLIs.
|
||||||
|
|
||||||
|
The loop is exited only by process signals or unrecoverable exceptions;
|
||||||
|
every per-board and per-claim fault is bounded inside one pass.
|
||||||
|
"""
|
||||||
from hermes_cli import kanban_db
|
from hermes_cli import kanban_db
|
||||||
|
|
||||||
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
RESULT_SCHEMA_PATH.parent.mkdir(parents=True, exist_ok=True)
|
||||||
@ -146,12 +161,20 @@ def main() -> int:
|
|||||||
futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
|
futures: dict[concurrent.futures.Future[None], tuple[str, str]] = {}
|
||||||
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as pool:
|
||||||
while True:
|
while True:
|
||||||
|
previous_capabilities = capabilities
|
||||||
|
capabilities = refresh_kanban_capabilities(kanban_db)
|
||||||
|
if capabilities.ready and not previous_capabilities.ready:
|
||||||
|
recover_orphans()
|
||||||
for future in list(futures):
|
for future in list(futures):
|
||||||
if future.done():
|
if future.done():
|
||||||
try:
|
try:
|
||||||
future.result()
|
future.result()
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
print(f"worker future failed: {error}", file=sys.stderr, flush=True)
|
print(
|
||||||
|
f"worker future failed: {error}",
|
||||||
|
file=sys.stderr,
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
del futures[future]
|
del futures[future]
|
||||||
recover_pending_finalizations()
|
recover_pending_finalizations()
|
||||||
maybe_gc_lane_artifacts()
|
maybe_gc_lane_artifacts()
|
||||||
@ -170,4 +193,3 @@ def main() -> int:
|
|||||||
future = pool.submit(execute_claim, board, task_id)
|
future = pool.submit(execute_claim, board, task_id)
|
||||||
futures[future] = (board, task_id)
|
futures[future] = (board, task_id)
|
||||||
time.sleep(5)
|
time.sleep(5)
|
||||||
return 0
|
|
||||||
|
|||||||
@ -18,6 +18,8 @@ from cli_lane_capabilities import (
|
|||||||
detect_kanban_capabilities,
|
detect_kanban_capabilities,
|
||||||
initialize_kanban_capabilities,
|
initialize_kanban_capabilities,
|
||||||
kanban_capabilities,
|
kanban_capabilities,
|
||||||
|
readiness_issue,
|
||||||
|
refresh_kanban_capabilities,
|
||||||
runtime_health,
|
runtime_health,
|
||||||
)
|
)
|
||||||
from cli_lane_board import (
|
from cli_lane_board import (
|
||||||
|
|||||||
@ -3,24 +3,42 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
|
import math
|
||||||
|
from dataclasses import dataclass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
|
@dataclass(frozen=True)
|
||||||
"""Load per-file line-rate percentages from a Cobertura XML report."""
|
class CoverageRates:
|
||||||
|
"""Validated line and branch percentages for one source file."""
|
||||||
|
|
||||||
|
line: float
|
||||||
|
branch: float
|
||||||
|
|
||||||
|
|
||||||
|
def _percentage(class_node: ET.Element, attribute: str) -> float:
|
||||||
|
"""Parse one finite Cobertura rate and fail closed outside its domain."""
|
||||||
|
raw = class_node.attrib.get(attribute)
|
||||||
|
if raw is None:
|
||||||
|
raise ValueError(f"coverage class missing {attribute}")
|
||||||
|
value = float(raw)
|
||||||
|
if not math.isfinite(value) or value < 0.0 or value > 1.0:
|
||||||
|
raise ValueError(f"coverage class has invalid {attribute}: {raw}")
|
||||||
|
return value * 100.0
|
||||||
|
|
||||||
|
|
||||||
|
def _load_rates(xml_path: Path, root: Path) -> dict[str, CoverageRates]:
|
||||||
|
"""Load validated per-file line and branch rates from Cobertura XML."""
|
||||||
tree = ET.parse(xml_path)
|
tree = ET.parse(xml_path)
|
||||||
xml_root = tree.getroot()
|
xml_root = tree.getroot()
|
||||||
source_roots = [
|
source_roots = [
|
||||||
Path(node.text)
|
Path(node.text) for node in xml_root.findall("./sources/source") if node.text
|
||||||
for node in xml_root.findall("./sources/source")
|
|
||||||
if node.text
|
|
||||||
]
|
]
|
||||||
percentages: dict[str, float] = {}
|
rates: dict[str, CoverageRates] = {}
|
||||||
for class_node in xml_root.findall(".//class"):
|
for class_node in xml_root.findall(".//class"):
|
||||||
filename = class_node.attrib.get("filename")
|
filename = class_node.attrib.get("filename")
|
||||||
line_rate = class_node.attrib.get("line-rate")
|
if not filename:
|
||||||
if not filename or line_rate is None:
|
|
||||||
continue
|
continue
|
||||||
normalized = filename.replace("\\", "/")
|
normalized = filename.replace("\\", "/")
|
||||||
if normalized.startswith("/"):
|
if normalized.startswith("/"):
|
||||||
@ -32,8 +50,16 @@ def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
|
|||||||
if candidate.exists():
|
if candidate.exists():
|
||||||
key = candidate.relative_to(root).as_posix()
|
key = candidate.relative_to(root).as_posix()
|
||||||
break
|
break
|
||||||
percentages[key] = float(line_rate) * 100.0
|
rates[key] = CoverageRates(
|
||||||
return percentages
|
line=_percentage(class_node, "line-rate"),
|
||||||
|
branch=_percentage(class_node, "branch-rate"),
|
||||||
|
)
|
||||||
|
return rates
|
||||||
|
|
||||||
|
|
||||||
|
def _load_percentages(xml_path: Path, root: Path) -> dict[str, float]:
|
||||||
|
"""Load per-file line percentages for backward-compatible callers."""
|
||||||
|
return {path: rates.line for path, rates in _load_rates(xml_path, root).items()}
|
||||||
|
|
||||||
|
|
||||||
def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]:
|
def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]:
|
||||||
@ -45,19 +71,28 @@ def run_check(contract: dict[str, Any], root: Path, xml_path: Path) -> list[str]
|
|||||||
if not xml_path.exists():
|
if not xml_path.exists():
|
||||||
return [f"coverage xml missing: {xml_path.relative_to(root)}"]
|
return [f"coverage xml missing: {xml_path.relative_to(root)}"]
|
||||||
|
|
||||||
percentages = _load_percentages(xml_path, root)
|
try:
|
||||||
|
rates_by_path = _load_rates(xml_path, root)
|
||||||
|
except (ET.ParseError, OSError, UnicodeError, ValueError) as error:
|
||||||
|
return [f"coverage xml invalid: {error}"]
|
||||||
minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0))
|
minimum = float(contract.get("coverage", {}).get("minimum_percent", 95.0))
|
||||||
issues: list[str] = []
|
issues: list[str] = []
|
||||||
|
|
||||||
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
|
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
|
||||||
normalized = relative_path.replace("\\", "/")
|
normalized = relative_path.replace("\\", "/")
|
||||||
percent = percentages.get(normalized)
|
rates = rates_by_path.get(normalized)
|
||||||
if percent is None:
|
if rates is None:
|
||||||
issues.append(f"coverage missing for tracked file: {relative_path}")
|
issues.append(f"coverage missing for tracked file: {relative_path}")
|
||||||
continue
|
continue
|
||||||
if percent + 1e-9 < minimum:
|
if rates.line + 1e-9 < minimum:
|
||||||
issues.append(
|
issues.append(
|
||||||
f"coverage below {minimum:.1f}%: {relative_path} ({percent:.1f}%)"
|
f"line coverage below {minimum:.1f}%: {relative_path} "
|
||||||
|
f"({rates.line:.1f}%)"
|
||||||
|
)
|
||||||
|
if rates.branch + 1e-9 < minimum:
|
||||||
|
issues.append(
|
||||||
|
f"branch coverage below {minimum:.1f}%: {relative_path} "
|
||||||
|
f"({rates.branch:.1f}%)"
|
||||||
)
|
)
|
||||||
|
|
||||||
return issues
|
return issues
|
||||||
@ -73,7 +108,10 @@ def compute_workspace_line_coverage(
|
|||||||
if not xml_path.exists():
|
if not xml_path.exists():
|
||||||
return 0.0
|
return 0.0
|
||||||
|
|
||||||
percentages = _load_percentages(xml_path, root)
|
try:
|
||||||
|
percentages = _load_percentages(xml_path, root)
|
||||||
|
except (ET.ParseError, OSError, UnicodeError, ValueError):
|
||||||
|
return 0.0
|
||||||
samples: list[float] = []
|
samples: list[float] = []
|
||||||
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
|
for relative_path in contract.get("coverage", {}).get("tracked_files", []):
|
||||||
normalized = relative_path.replace("\\", "/")
|
normalized = relative_path.replace("\\", "/")
|
||||||
|
|||||||
@ -36,7 +36,9 @@ def test_claude_pretool_hook_allows_normal_engineering():
|
|||||||
assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
|
assert policy.denial_reason("vault kv get kv/atlas/hermes") is None
|
||||||
|
|
||||||
|
|
||||||
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_path: Path):
|
def test_claude_settings_preserve_state_and_install_three_guardrail_layers(
|
||||||
|
tmp_path: Path,
|
||||||
|
):
|
||||||
state = tmp_path / ".claude.json"
|
state = tmp_path / ".claude.json"
|
||||||
settings = tmp_path / "settings.json"
|
settings = tmp_path / "settings.json"
|
||||||
state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
|
state.write_text('{"promptQueueUseCount": 4}\n', encoding="utf-8")
|
||||||
@ -78,7 +80,9 @@ def test_claude_settings_preserve_state_and_install_three_guardrail_layers(tmp_p
|
|||||||
def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
|
def test_legacy_state_is_archived_without_removing_provider_transcripts(tmp_path: Path):
|
||||||
session = tmp_path / "home/.config/herdr/session.json"
|
session = tmp_path / "home/.config/herdr/session.json"
|
||||||
session.parent.mkdir(parents=True)
|
session.parent.mkdir(parents=True)
|
||||||
session.write_text('{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8")
|
session.write_text(
|
||||||
|
'{"agents":[{"agent":"claude","session_id":"abc"}]}', encoding="utf-8"
|
||||||
|
)
|
||||||
binary = tmp_path / "tools/bin/herdr"
|
binary = tmp_path / "tools/bin/herdr"
|
||||||
binary.parent.mkdir(parents=True)
|
binary.parent.mkdir(parents=True)
|
||||||
binary.write_text("legacy", encoding="utf-8")
|
binary.write_text("legacy", encoding="utf-8")
|
||||||
@ -156,9 +160,7 @@ def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
|
|||||||
assert hostnames["operator"] == "NotIn"
|
assert hostnames["operator"] == "NotIn"
|
||||||
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
|
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
|
||||||
|
|
||||||
hermes = next(
|
hermes = next(item for item in pod["containers"] if item["name"] == "hermes")
|
||||||
item for item in pod["containers"] if item["name"] == "hermes"
|
|
||||||
)
|
|
||||||
assert hermes["resources"]["requests"]["cpu"] == "300m"
|
assert hermes["resources"]["requests"]["cpu"] == "300m"
|
||||||
|
|
||||||
|
|
||||||
@ -223,8 +225,7 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
|||||||
)
|
)
|
||||||
|
|
||||||
patch_init = next(
|
patch_init = next(
|
||||||
item for item in pod["initContainers"]
|
item for item in pod["initContainers"] if item["name"] == "patch-tui-gateway"
|
||||||
if item["name"] == "patch-tui-gateway"
|
|
||||||
)
|
)
|
||||||
assert patch_init["command"][-1] == "/patched/server.py"
|
assert patch_init["command"][-1] == "/patched/server.py"
|
||||||
for name in ("hermes", "terminal"):
|
for name in ("hermes", "terminal"):
|
||||||
@ -248,20 +249,29 @@ def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
|
|||||||
assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
|
assert middlewares["hermes-agent-terminal-slash"]["spec"]["redirectRegex"][
|
||||||
"replacement"
|
"replacement"
|
||||||
].endswith("/terminal/")
|
].endswith("/terminal/")
|
||||||
assert middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
assert (
|
||||||
"customRequestHeaders"
|
middlewares["hermes-agent-stock-dashboard-headers"]["spec"]["headers"][
|
||||||
]["Origin"] == "http://127.0.0.1:9119"
|
"customRequestHeaders"
|
||||||
|
]["Origin"]
|
||||||
|
== "http://127.0.0.1:9119"
|
||||||
|
)
|
||||||
ingresses = {
|
ingresses = {
|
||||||
item["metadata"]["name"]: item
|
item["metadata"]["name"]: item
|
||||||
for item in ingress_documents
|
for item in ingress_documents
|
||||||
if item["kind"] == "Ingress"
|
if item["kind"] == "Ingress"
|
||||||
}
|
}
|
||||||
assert ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
assert (
|
||||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
ingresses["hermes-agent-dashboard"]["metadata"]["annotations"][
|
||||||
] == "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||||
assert ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
]
|
||||||
"traefik.ingress.kubernetes.io/router.middlewares"
|
== "hermes-hermes-agent-stock-dashboard-headers@kubernetescrd"
|
||||||
] == "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
)
|
||||||
|
assert (
|
||||||
|
ingresses["hermes-agent-terminal"]["metadata"]["annotations"][
|
||||||
|
"traefik.ingress.kubernetes.io/router.middlewares"
|
||||||
|
]
|
||||||
|
== "hermes-hermes-agent-terminal-slash@kubernetescrd"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_broker_services_survive_sibling_container_readiness_loss():
|
def test_broker_services_survive_sibling_container_readiness_loss():
|
||||||
@ -322,9 +332,7 @@ def test_cli_lane_config_refresh_does_not_restart_active_work():
|
|||||||
for item in deployment["spec"]["template"]["spec"]["containers"]
|
for item in deployment["spec"]["template"]["spec"]["containers"]
|
||||||
if item["name"] == "cli-lane-runner"
|
if item["name"] == "cli-lane-runner"
|
||||||
)
|
)
|
||||||
mount = next(
|
mount = next(item for item in lane["volumeMounts"] if item["name"] == "coordinator")
|
||||||
item for item in lane["volumeMounts"] if item["name"] == "coordinator"
|
|
||||||
)
|
|
||||||
|
|
||||||
assert coordinator["options"]["disableNameSuffixHash"] is True
|
assert coordinator["options"]["disableNameSuffixHash"] is True
|
||||||
assert mount == {
|
assert mount == {
|
||||||
@ -335,12 +343,28 @@ def test_cli_lane_config_refresh_does_not_restart_active_work():
|
|||||||
assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get(
|
assert "checksum/hermes-coordinator" not in deployment["spec"]["template"].get(
|
||||||
"metadata", {}
|
"metadata", {}
|
||||||
).get("annotations", {})
|
).get("annotations", {})
|
||||||
|
assert "startupProbe" not in lane
|
||||||
|
assert "livenessProbe" not in lane
|
||||||
|
assert lane["readinessProbe"] == {
|
||||||
|
"exec": {
|
||||||
|
"command": [
|
||||||
|
"/opt/hermes/.venv/bin/python",
|
||||||
|
"/opt/coordinator/cli_lane_capabilities.py",
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"initialDelaySeconds": 2,
|
||||||
|
"periodSeconds": 5,
|
||||||
|
"timeoutSeconds": 2,
|
||||||
|
"failureThreshold": 1,
|
||||||
|
}
|
||||||
|
environment = {item["name"]: item["value"] for item in lane["env"]}
|
||||||
|
assert environment["HERMES_CLI_HEALTH_MAX_AGE_SECONDS"] == "60"
|
||||||
|
|
||||||
|
|
||||||
def test_agent_dashboard_reconnects_all_transient_websockets():
|
def test_agent_dashboard_reconnects_all_transient_websockets():
|
||||||
dockerfile = (
|
dockerfile = (HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent").read_text(
|
||||||
HERMES.parents[1] / "dockerfiles/Dockerfile.hermes-agent"
|
encoding="utf-8"
|
||||||
).read_text(encoding="utf-8")
|
)
|
||||||
assert "eventsRetryAttempt.current" in dockerfile
|
assert "eventsRetryAttempt.current" in dockerfile
|
||||||
assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
|
assert "if (!unmounting) setVersion((v) => v + 1);" in dockerfile
|
||||||
assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
|
assert "events feed rejected (${ev.code}) — reload the page" in dockerfile
|
||||||
|
|||||||
@ -3,14 +3,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import runpy
|
||||||
import sys
|
import sys
|
||||||
from contextlib import nullcontext
|
from contextlib import nullcontext
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from testing.tests.test_hermes_cli_support import _completed_result, lanes
|
from testing.tests.test_hermes_cli_support import SCRIPTS, _completed_result, lanes
|
||||||
|
|
||||||
|
|
||||||
def _old_complete(
|
def _old_complete(
|
||||||
@ -94,6 +96,18 @@ def test_capability_detection_requires_explicit_safety_keywords():
|
|||||||
assert lanes._explicit_keyword(object(), "expected_run_id") is False
|
assert lanes._explicit_keyword(object(), "expected_run_id") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_deferred_features_name_every_absent_capability():
|
||||||
|
"""A fully legacy image defers all three safety features by exact name."""
|
||||||
|
fully_legacy = lanes.KanbanCapabilities(False, False, False)
|
||||||
|
|
||||||
|
assert fully_legacy.ready is False
|
||||||
|
assert fully_legacy.deferred_features == (
|
||||||
|
"exact-run-completion",
|
||||||
|
"ended-run-replay",
|
||||||
|
"exact-run-reclaim",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_startup_health_moves_from_bounded_deferred_to_ready(
|
def test_startup_health_moves_from_bounded_deferred_to_ready(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
capsys,
|
capsys,
|
||||||
@ -106,6 +120,7 @@ def test_startup_health_moves_from_bounded_deferred_to_ready(
|
|||||||
|
|
||||||
assert old.ready is False
|
assert old.ready is False
|
||||||
assert old_document["state"] == "deferred"
|
assert old_document["state"] == "deferred"
|
||||||
|
assert old_document["schema_version"] == 1
|
||||||
assert old_document["ready"] is False
|
assert old_document["ready"] is False
|
||||||
assert old_document["active_run_completion_safe"] is True
|
assert old_document["active_run_completion_safe"] is True
|
||||||
assert old_document["deferred_features"] == [
|
assert old_document["deferred_features"] == [
|
||||||
@ -123,6 +138,107 @@ def test_startup_health_moves_from_bounded_deferred_to_ready(
|
|||||||
assert lanes.runtime_health(health)["state"] == "ready"
|
assert lanes.runtime_health(health)["state"] == "ready"
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_fails_closed_without_disclosing_health_values(
|
||||||
|
tmp_path: Path,
|
||||||
|
capsys,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Readiness distinguishes startup, malformed, deferred, stale, and ready."""
|
||||||
|
health = tmp_path / "runtime-health.json"
|
||||||
|
now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc)
|
||||||
|
assert lanes.readiness_issue(health, now=now) == "runtime health is missing"
|
||||||
|
|
||||||
|
health.write_text("not-json", encoding="utf-8")
|
||||||
|
assert lanes.readiness_issue(health, now=now) == "runtime health is malformed"
|
||||||
|
|
||||||
|
deferred = lanes._health_document(lanes.KanbanCapabilities(True, False, False))
|
||||||
|
deferred["observed_at"] = now.isoformat()
|
||||||
|
health.write_text(json.dumps(deferred), encoding="utf-8")
|
||||||
|
assert lanes.readiness_issue(health, now=now) == "runtime compatibility is deferred"
|
||||||
|
|
||||||
|
ready = lanes._health_document(lanes.KanbanCapabilities(True, True, True))
|
||||||
|
ready["observed_at"] = (now - timedelta(seconds=61)).isoformat()
|
||||||
|
health.write_text(json.dumps(ready), encoding="utf-8")
|
||||||
|
assert lanes.readiness_issue(health, now=now) == "runtime health is stale"
|
||||||
|
|
||||||
|
ready["observed_at"] = now.isoformat()
|
||||||
|
health.write_text(json.dumps(ready), encoding="utf-8")
|
||||||
|
assert lanes.readiness_issue(health, now=now) is None
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path)
|
||||||
|
monkeypatch.setenv("HERMES_CLI_HEALTH_MAX_AGE_SECONDS", "31536000")
|
||||||
|
assert sys.modules["cli_lane_capabilities"].main() == 0
|
||||||
|
assert capsys.readouterr().out == ""
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"mutation",
|
||||||
|
[
|
||||||
|
{"schema_version": 2},
|
||||||
|
{"state": "ready", "ready": False},
|
||||||
|
{"state": "ready", "capabilities": []},
|
||||||
|
{"observed_at": 5},
|
||||||
|
{"observed_at": "not-a-date"},
|
||||||
|
{"observed_at": "2026-08-17T12:00:00"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_readiness_rejects_inconsistent_documents(
|
||||||
|
tmp_path: Path,
|
||||||
|
mutation: dict,
|
||||||
|
):
|
||||||
|
"""Inconsistent ready documents remain not-ready."""
|
||||||
|
now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc)
|
||||||
|
document = lanes._health_document(lanes.KanbanCapabilities(True, True, True))
|
||||||
|
document["observed_at"] = now.isoformat()
|
||||||
|
document.update(mutation)
|
||||||
|
health = tmp_path / "runtime-health.json"
|
||||||
|
health.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
|
||||||
|
assert lanes.readiness_issue(health, now=now) == "runtime health is malformed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_command_fails_with_only_a_bounded_reason(
|
||||||
|
tmp_path: Path,
|
||||||
|
capsys,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""The probe command exits nonzero and discloses nothing but the reason."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path)
|
||||||
|
|
||||||
|
assert sys.modules["cli_lane_capabilities"].main() == 1
|
||||||
|
|
||||||
|
captured = capsys.readouterr()
|
||||||
|
assert captured.out == ""
|
||||||
|
assert captured.err.strip() == "runtime health is missing"
|
||||||
|
|
||||||
|
|
||||||
|
def test_capabilities_module_is_the_deployed_probe_entrypoint(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""`python cli_lane_capabilities.py` runs the readiness probe directly."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit) as excinfo:
|
||||||
|
runpy.run_path(
|
||||||
|
str(SCRIPTS / "cli_lane_capabilities.py"),
|
||||||
|
run_name="__main__",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert excinfo.value.code == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_readiness_rejects_future_health(tmp_path: Path):
|
||||||
|
"""A clock-skewed future observation cannot make a stale process ready."""
|
||||||
|
now = datetime(2026, 8, 17, 12, 0, tzinfo=timezone.utc)
|
||||||
|
document = lanes._health_document(lanes.KanbanCapabilities(True, True, True))
|
||||||
|
document["observed_at"] = (now + timedelta(seconds=6)).isoformat()
|
||||||
|
health = tmp_path / "runtime-health.json"
|
||||||
|
health.write_text(json.dumps(document), encoding="utf-8")
|
||||||
|
|
||||||
|
assert lanes.readiness_issue(health, now=now) == "runtime health is stale"
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
("status", "run_id", "expected"),
|
("status", "run_id", "expected"),
|
||||||
[("running", 7, "committed"), ("blocked", None, "deferred")],
|
[("running", 7, "committed"), ("blocked", None, "deferred")],
|
||||||
|
|||||||
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import runpy
|
||||||
import sys
|
import sys
|
||||||
from contextlib import nullcontext
|
from contextlib import nullcontext
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@ -9,7 +10,7 @@ from types import SimpleNamespace
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from testing.tests.test_hermes_cli_support import lanes
|
from testing.tests.test_hermes_cli_support import SCRIPTS, lanes
|
||||||
|
|
||||||
|
|
||||||
def test_board_slug_and_connection_failure_paths(monkeypatch):
|
def test_board_slug_and_connection_failure_paths(monkeypatch):
|
||||||
@ -129,6 +130,28 @@ class _Pool:
|
|||||||
return self.future
|
return self.future
|
||||||
|
|
||||||
|
|
||||||
|
class _HeldFuture:
|
||||||
|
def done(self):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class _RecordingPool:
|
||||||
|
submissions = []
|
||||||
|
|
||||||
|
def __init__(self, max_workers):
|
||||||
|
self.max_workers = max_workers
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def submit(self, _function, board, task_id):
|
||||||
|
self.submissions.append((board, task_id))
|
||||||
|
return _HeldFuture()
|
||||||
|
|
||||||
|
|
||||||
def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
|
def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
|
||||||
tmp_path: Path,
|
tmp_path: Path,
|
||||||
monkeypatch,
|
monkeypatch,
|
||||||
@ -138,11 +161,9 @@ def test_ready_dispatch_loop_submits_and_reaps_failed_workers(
|
|||||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||||
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
||||||
monkeypatch.setattr(
|
ready = lanes.KanbanCapabilities(True, True, True)
|
||||||
lanes,
|
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready)
|
||||||
"initialize_kanban_capabilities",
|
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready)
|
||||||
lambda _db: lanes.KanbanCapabilities(True, True, True),
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||||
claims = [[("cassandra", "t_loop")], []]
|
claims = [[("cassandra", "t_loop")], []]
|
||||||
@ -175,11 +196,9 @@ def test_deferred_dispatch_health_never_claims_new_work(
|
|||||||
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||||
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
||||||
monkeypatch.setattr(
|
deferred = lanes.KanbanCapabilities(True, False, False)
|
||||||
lanes,
|
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: deferred)
|
||||||
"initialize_kanban_capabilities",
|
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: deferred)
|
||||||
lambda _db: lanes.KanbanCapabilities(True, False, False),
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
@ -195,3 +214,192 @@ def test_deferred_dispatch_health_never_claims_new_work(
|
|||||||
)
|
)
|
||||||
with pytest.raises(RuntimeError, match="stop loop"):
|
with pytest.raises(RuntimeError, match="stop loop"):
|
||||||
lanes.main()
|
lanes.main()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("start_patched", [False, True])
|
||||||
|
def test_two_loop_api_transition_refreshes_dispatch_and_health(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
start_patched: bool,
|
||||||
|
):
|
||||||
|
"""Each loop observes legacy/patched API transitions without reloading modules."""
|
||||||
|
|
||||||
|
def old_complete(_conn, _task_id, *, expected_run_id=None):
|
||||||
|
return expected_run_id is not None
|
||||||
|
|
||||||
|
def new_complete(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
expected_run_id=None,
|
||||||
|
replay_ended_run_id=None,
|
||||||
|
):
|
||||||
|
return expected_run_id is not None or replay_ended_run_id is not None
|
||||||
|
|
||||||
|
def old_reclaim(_conn, _task_id, *, reason):
|
||||||
|
pytest.fail(f"unguarded legacy reclaim invoked: {reason}")
|
||||||
|
|
||||||
|
def new_reclaim(_conn, _task_id, *, reason, expected_run_id=None):
|
||||||
|
return bool(reason and expected_run_id)
|
||||||
|
|
||||||
|
db = SimpleNamespace(
|
||||||
|
complete_task=new_complete if start_patched else old_complete,
|
||||||
|
reclaim_task=new_reclaim if start_patched else old_reclaim,
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||||
|
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||||
|
recoveries = []
|
||||||
|
monkeypatch.setattr(lanes, "recover_orphans", lambda: recoveries.append("recover"))
|
||||||
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
|
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||||
|
claim_states = []
|
||||||
|
|
||||||
|
def claim(_active, _limit):
|
||||||
|
ready = lanes.kanban_capabilities(db).ready
|
||||||
|
claim_states.append(ready)
|
||||||
|
return [("cassandra", "t_loop")] if ready and not _active else []
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes, "claim_ready", claim)
|
||||||
|
_RecordingPool.submissions = []
|
||||||
|
monkeypatch.setattr(lanes.concurrent.futures, "ThreadPoolExecutor", _RecordingPool)
|
||||||
|
journal = tmp_path / "cli-lanes/cassandra/t_loop.terminal.json"
|
||||||
|
journal.parent.mkdir(parents=True)
|
||||||
|
journal.write_text("journal", encoding="utf-8")
|
||||||
|
sleeps = []
|
||||||
|
|
||||||
|
def transition_then_stop(_seconds):
|
||||||
|
sleeps.append(True)
|
||||||
|
if len(sleeps) == 1:
|
||||||
|
db.complete_task = old_complete if start_patched else new_complete
|
||||||
|
db.reclaim_task = old_reclaim if start_patched else new_reclaim
|
||||||
|
return
|
||||||
|
raise RuntimeError("two loops complete")
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes.time, "sleep", transition_then_stop)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="two loops complete"):
|
||||||
|
lanes.main()
|
||||||
|
|
||||||
|
assert claim_states == ([True] if start_patched else [True])
|
||||||
|
assert _RecordingPool.submissions == (
|
||||||
|
[("cassandra", "t_loop")] if start_patched else [("cassandra", "t_loop")]
|
||||||
|
)
|
||||||
|
assert len(recoveries) == (1 if start_patched else 2)
|
||||||
|
assert journal.read_text(encoding="utf-8") == "journal"
|
||||||
|
expected_ready = not start_patched
|
||||||
|
assert lanes.runtime_health()["ready"] is expected_ready
|
||||||
|
|
||||||
|
|
||||||
|
def test_orphan_recovery_skips_unreachable_boards_and_settled_tasks(
|
||||||
|
monkeypatch,
|
||||||
|
capsys,
|
||||||
|
):
|
||||||
|
"""One faulted board or settled task never stops restart reclamation."""
|
||||||
|
running = SimpleNamespace(
|
||||||
|
id="t_run",
|
||||||
|
status="running",
|
||||||
|
current_run_id=4,
|
||||||
|
assignee="cli-auto",
|
||||||
|
)
|
||||||
|
settled = SimpleNamespace(
|
||||||
|
id="t_done",
|
||||||
|
status="done",
|
||||||
|
current_run_id=None,
|
||||||
|
assignee="cli-auto",
|
||||||
|
)
|
||||||
|
reclaims = []
|
||||||
|
|
||||||
|
def connect(*, board):
|
||||||
|
if board == "broken":
|
||||||
|
raise OSError("volume stall")
|
||||||
|
return SimpleNamespace(close=lambda: None)
|
||||||
|
|
||||||
|
db = SimpleNamespace(
|
||||||
|
list_boards=lambda include_archived=False: [
|
||||||
|
{"slug": "broken"},
|
||||||
|
{"slug": "healthy"},
|
||||||
|
],
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=connect,
|
||||||
|
list_tasks=lambda _conn: [settled, running],
|
||||||
|
reclaim_task=lambda _conn, task_id, **kwargs: reclaims.append(
|
||||||
|
(task_id, kwargs["expected_run_id"])
|
||||||
|
),
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
|
monkeypatch.setattr(lanes, "_has_pending_finalization", lambda *_args: False)
|
||||||
|
|
||||||
|
lanes.recover_orphans()
|
||||||
|
|
||||||
|
assert reclaims == [("t_run", 4)]
|
||||||
|
assert "temporarily skipping Kanban board 'broken'" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
|
def test_claim_scan_skips_foreign_lanes_and_lost_claim_races(monkeypatch):
|
||||||
|
"""Foreign assignees and lost atomic claims never enter dispatch."""
|
||||||
|
foreign = SimpleNamespace(id="t_manual", assignee="human", status="ready")
|
||||||
|
lost = SimpleNamespace(id="t_lost", assignee="cli-auto", status="ready")
|
||||||
|
won = SimpleNamespace(id="t_won", assignee="cli-auto", status="ready")
|
||||||
|
|
||||||
|
def claim_task(_conn, task_id, **_kwargs):
|
||||||
|
return None if task_id == "t_lost" else SimpleNamespace(id=task_id)
|
||||||
|
|
||||||
|
db = SimpleNamespace(
|
||||||
|
list_boards=lambda include_archived=False: [{"slug": "cassandra"}],
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda board: SimpleNamespace(close=lambda: None),
|
||||||
|
recompute_ready=lambda _conn: None,
|
||||||
|
list_tasks=lambda _conn: [foreign, lost, won],
|
||||||
|
claim_task=claim_task,
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
|
||||||
|
assert lanes.claim_ready(set(), 3) == [("cassandra", "t_won")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_dispatch_loop_survives_board_registry_scan_failure(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
capsys,
|
||||||
|
):
|
||||||
|
"""A failing registry scan defers claiming without stopping the loop."""
|
||||||
|
db = SimpleNamespace()
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
monkeypatch.setattr(lanes, "RESULT_SCHEMA_PATH", tmp_path / "schema.json")
|
||||||
|
monkeypatch.setattr(lanes, "recover_orphans", lambda: None)
|
||||||
|
ready = lanes.KanbanCapabilities(True, True, True)
|
||||||
|
monkeypatch.setattr(lanes, "initialize_kanban_capabilities", lambda _db: ready)
|
||||||
|
monkeypatch.setattr(lanes, "refresh_kanban_capabilities", lambda _db: ready)
|
||||||
|
monkeypatch.setattr(lanes, "recover_pending_finalizations", lambda: 0)
|
||||||
|
monkeypatch.setattr(lanes, "maybe_gc_lane_artifacts", lambda: 0)
|
||||||
|
|
||||||
|
def failing_claim(_active, _limit):
|
||||||
|
raise RuntimeError("registry scan failed")
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes, "claim_ready", failing_claim)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes.time,
|
||||||
|
"sleep",
|
||||||
|
lambda _seconds: (_ for _ in ()).throw(RuntimeError("stop loop")),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="stop loop"):
|
||||||
|
lanes.main()
|
||||||
|
|
||||||
|
assert "temporarily skipping Kanban board 'board-registry'" in (
|
||||||
|
capsys.readouterr().err
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runner_module_is_the_deployed_dispatch_entrypoint(monkeypatch):
|
||||||
|
"""`python cli_lane_runner.py` hands control to the dispatch loop."""
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(lanes, "main", lambda: calls.append(True) or 0)
|
||||||
|
|
||||||
|
with pytest.raises(SystemExit) as excinfo:
|
||||||
|
runpy.run_path(str(SCRIPTS / "cli_lane_runner.py"), run_name="__main__")
|
||||||
|
|
||||||
|
assert excinfo.value.code == 0
|
||||||
|
assert calls == [True]
|
||||||
|
|||||||
@ -192,3 +192,38 @@ def test_retention_failure_paths_are_bounded(tmp_path: Path, monkeypatch, capsys
|
|||||||
)
|
)
|
||||||
assert lanes.maybe_gc_lane_artifacts(now=1000) == 0
|
assert lanes.maybe_gc_lane_artifacts(now=1000) == 0
|
||||||
assert "retention deferred" in capsys.readouterr().err
|
assert "retention deferred" in capsys.readouterr().err
|
||||||
|
|
||||||
|
|
||||||
|
def test_evidence_writer_rejects_a_tampered_temporary(tmp_path: Path, monkeypatch):
|
||||||
|
"""A shared or replaced temporary inode never becomes durable evidence."""
|
||||||
|
real_fstat = os.fstat
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def hardlinked_fstat(descriptor):
|
||||||
|
observed = real_fstat(descriptor)
|
||||||
|
calls.append(observed)
|
||||||
|
if len(calls) == 2:
|
||||||
|
return os.stat_result((*observed[:3], 2, *observed[4:]))
|
||||||
|
return observed
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes.os, "fstat", hardlinked_fstat)
|
||||||
|
|
||||||
|
with pytest.raises(OSError, match="not private and singly linked"):
|
||||||
|
lanes._write_json_noreplace(tmp_path / "tampered.json", {"value": "x"})
|
||||||
|
|
||||||
|
assert list(tmp_path.iterdir()) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_evidence_writer_releases_descriptor_when_inspection_fails(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""An fstat fault closes the created descriptor without staging cleanup."""
|
||||||
|
|
||||||
|
def failing_fstat(_descriptor):
|
||||||
|
raise OSError("inspection failed")
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes.os, "fstat", failing_fstat)
|
||||||
|
|
||||||
|
with pytest.raises(OSError, match="inspection failed"):
|
||||||
|
lanes._write_json_noreplace(tmp_path / "orphaned.json", {"value": "x"})
|
||||||
|
|||||||
@ -216,3 +216,104 @@ def test_unexpected_route_exception_blocks_the_exact_run(
|
|||||||
|
|
||||||
assert blocks[0]["expected_run_id"] == 23
|
assert blocks[0]["expected_run_id"] == 23
|
||||||
assert "router down" in blocks[0]["reason"]
|
assert "router down" in blocks[0]["reason"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_capacity_fallback_without_evidence_blocks_with_provider_output(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A fallback turn with no structured result blocks on raw provider output."""
|
||||||
|
task = SimpleNamespace(
|
||||||
|
id="t_nofallback",
|
||||||
|
status="running",
|
||||||
|
current_run_id=22,
|
||||||
|
assignee="cli-auto",
|
||||||
|
max_runtime_seconds=60,
|
||||||
|
)
|
||||||
|
blocks = []
|
||||||
|
db = SimpleNamespace(
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda board: _Connection(),
|
||||||
|
get_task=lambda *_args: task,
|
||||||
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
||||||
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
||||||
|
set_branch_name=lambda *_args: None,
|
||||||
|
set_workspace_path=lambda *_args: None,
|
||||||
|
build_worker_context=lambda *_args: "attempt both providers",
|
||||||
|
add_comment=lambda *_args, **_kwargs: None,
|
||||||
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||||||
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
|
||||||
|
codex = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
||||||
|
claude = lanes.Route("claude", "fable", "high", "p", "c", "r", 1, ())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"select_route",
|
||||||
|
lambda _prompt, assignee, **_kwargs: claude
|
||||||
|
if assignee == "cli-claude-high"
|
||||||
|
else codex,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(lanes, "git_handoff", lambda *_args: "handoff")
|
||||||
|
reports = [
|
||||||
|
lanes.ProcessResult(1, "usage limit reached", None, True),
|
||||||
|
lanes.ProcessResult(1, "fallback also failed", None, False),
|
||||||
|
]
|
||||||
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
||||||
|
|
||||||
|
lanes.execute_claim("cassandra", "t_nofallback")
|
||||||
|
|
||||||
|
assert reports == []
|
||||||
|
assert blocks and "fallback also failed" in blocks[0]["reason"]
|
||||||
|
assert blocks[0]["kind"] == "capability"
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_blockers_become_the_block_reason_without_goal_mode(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A structured non-completed report blocks with its own stated blockers."""
|
||||||
|
task = SimpleNamespace(
|
||||||
|
id="t_blocked",
|
||||||
|
status="running",
|
||||||
|
current_run_id=23,
|
||||||
|
assignee="cli-auto",
|
||||||
|
max_runtime_seconds=60,
|
||||||
|
)
|
||||||
|
blocks = []
|
||||||
|
db = SimpleNamespace(
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda board: _Connection(),
|
||||||
|
get_task=lambda *_args: task,
|
||||||
|
worker_log_path=lambda *_args, **_kwargs: tmp_path / "worker.log",
|
||||||
|
_resolve_worktree_workspace=lambda *_args, **_kwargs: (tmp_path, "branch"),
|
||||||
|
set_branch_name=lambda *_args: None,
|
||||||
|
set_workspace_path=lambda *_args: None,
|
||||||
|
build_worker_context=lambda *_args: "report the obstacle",
|
||||||
|
add_comment=lambda *_args, **_kwargs: None,
|
||||||
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||||||
|
block_task=lambda *_args, **kwargs: blocks.append(kwargs),
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=db))
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
monkeypatch.setattr(lanes, "fresh_unavailable_provider", lambda: None)
|
||||||
|
route = lanes.Route("codex", "gpt", "high", "p", "c", "r", 1, ())
|
||||||
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
||||||
|
structured = {
|
||||||
|
**_completed_result("progress only"),
|
||||||
|
"status": "blocked",
|
||||||
|
"blockers": ["credentials expired", "remote unreachable"],
|
||||||
|
}
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"run_provider",
|
||||||
|
lambda *_args, **_kwargs: lanes.ProcessResult(0, "out", structured, False),
|
||||||
|
)
|
||||||
|
|
||||||
|
lanes.execute_claim("cassandra", "t_blocked")
|
||||||
|
|
||||||
|
assert blocks and blocks[0]["reason"] == (
|
||||||
|
"credentials expired; remote unreachable"
|
||||||
|
)
|
||||||
|
|||||||
@ -162,3 +162,68 @@ def test_terminal_finalizer_closes_supplied_invalid_snapshots(
|
|||||||
empty = _Snapshot(None)
|
empty = _Snapshot(None)
|
||||||
assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid"
|
assert lanes._finalize_terminal_record(_db(None), path, snapshot=empty) == "invalid"
|
||||||
assert empty.closed == 1
|
assert empty.closed == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_finalizer_rejects_invalid_names_and_unpinnable_journals(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Invalid identities and unreadable journals return without side effects."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||||
|
(tmp_path / "cli-lanes/cassandra").mkdir(parents=True)
|
||||||
|
db = _db(None)
|
||||||
|
|
||||||
|
foreign = tmp_path / "cli-lanes/cassandra/not-a-journal.json"
|
||||||
|
assert lanes._finalize_terminal_record(db, foreign) == "invalid"
|
||||||
|
|
||||||
|
missing = lanes._terminal_path(
|
||||||
|
lanes.state_path("cassandra", "t_edge"),
|
||||||
|
5,
|
||||||
|
"pending",
|
||||||
|
)
|
||||||
|
assert lanes._finalize_terminal_record(db, missing) == "invalid"
|
||||||
|
|
||||||
|
|
||||||
|
class _NullSnapshot:
|
||||||
|
def __init__(self, document):
|
||||||
|
self.document = document
|
||||||
|
self.closed = 0
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
self.closed += 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_finalizer_closes_supplied_snapshots_on_every_invalid_path(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Caller-pinned snapshots are released for named and payload rejections."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||||
|
(tmp_path / "cli-lanes/cassandra").mkdir(parents=True)
|
||||||
|
db = _db(None)
|
||||||
|
pending = lanes._terminal_path(
|
||||||
|
lanes.state_path("cassandra", "t_edge"),
|
||||||
|
6,
|
||||||
|
"pending",
|
||||||
|
)
|
||||||
|
|
||||||
|
misnamed = _NullSnapshot({"kanban_state": "pending"})
|
||||||
|
assert (
|
||||||
|
lanes._finalize_terminal_record(
|
||||||
|
db,
|
||||||
|
tmp_path / "cli-lanes/cassandra/not-a-journal.json",
|
||||||
|
snapshot=misnamed,
|
||||||
|
)
|
||||||
|
== "invalid"
|
||||||
|
)
|
||||||
|
assert misnamed.closed == 1
|
||||||
|
|
||||||
|
empty = _NullSnapshot(None)
|
||||||
|
assert lanes._finalize_terminal_record(db, pending, snapshot=empty) == "invalid"
|
||||||
|
assert empty.closed == 1
|
||||||
|
|
||||||
|
malformed = _NullSnapshot({"kanban_state": "pending"})
|
||||||
|
assert lanes._finalize_terminal_record(db, pending, snapshot=malformed) == (
|
||||||
|
"invalid"
|
||||||
|
)
|
||||||
|
assert malformed.closed == 1
|
||||||
|
|||||||
@ -205,3 +205,50 @@ def test_switchyard_network_failure_is_explicit():
|
|||||||
URLError("offline")
|
URLError("offline")
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_artifact_directories_and_foreign_statuses_are_filtered(
|
||||||
|
tmp_path: Path,
|
||||||
|
):
|
||||||
|
"""String directory paths and non-result statuses never become evidence."""
|
||||||
|
workspace = tmp_path / "workspace"
|
||||||
|
(workspace / "directory").mkdir(parents=True)
|
||||||
|
|
||||||
|
assert lanes.workspace_artifacts(workspace, ["directory"]) == []
|
||||||
|
assert lanes._extract_json('{"status": "unknown"}') is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_events_without_sessions_or_results_stay_unparsed(
|
||||||
|
tmp_path: Path,
|
||||||
|
):
|
||||||
|
"""Session persistence and result extraction require explicit fields."""
|
||||||
|
state_file = tmp_path / "state.json"
|
||||||
|
state = {}
|
||||||
|
|
||||||
|
assert (
|
||||||
|
lanes._event_payload(
|
||||||
|
"codex",
|
||||||
|
json.dumps({"type": "thread.started", "thread": {}}),
|
||||||
|
state,
|
||||||
|
state_file,
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
assert "codex_thread_id" not in state
|
||||||
|
assert (
|
||||||
|
lanes._event_payload(
|
||||||
|
"codex",
|
||||||
|
json.dumps({"item": {"text": "no structured result here"}}),
|
||||||
|
state,
|
||||||
|
state_file,
|
||||||
|
)
|
||||||
|
is None
|
||||||
|
)
|
||||||
|
|
||||||
|
direct = lanes._event_payload(
|
||||||
|
"codex",
|
||||||
|
json.dumps({"result": json.dumps(_completed_result("direct"))}),
|
||||||
|
state,
|
||||||
|
state_file,
|
||||||
|
)
|
||||||
|
assert direct and direct["summary"] == "direct"
|
||||||
|
|||||||
@ -149,3 +149,99 @@ def test_claim_requires_structured_evidence_and_surfaces_artifacts(
|
|||||||
else:
|
else:
|
||||||
assert calls[0][1]["kind"] == "capability"
|
assert calls[0][1]["kind"] == "capability"
|
||||||
assert all(connection.closed for connection in connections)
|
assert all(connection.closed for connection in connections)
|
||||||
|
|
||||||
|
|
||||||
|
def test_goal_loop_recovers_from_corrupt_rejection_history(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Corrupt durable rejection history is replaced, not trusted or fatal."""
|
||||||
|
task = SimpleNamespace(
|
||||||
|
id="t_history",
|
||||||
|
status="running",
|
||||||
|
result=None,
|
||||||
|
current_run_id=6,
|
||||||
|
assignee="cli-auto",
|
||||||
|
max_runtime_seconds=300,
|
||||||
|
goal_mode=True,
|
||||||
|
goal_max_turns=2,
|
||||||
|
)
|
||||||
|
calls = []
|
||||||
|
fake_db = SimpleNamespace(
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda board: SimpleNamespace(close=lambda: None),
|
||||||
|
get_task=lambda _conn, _task_id: task,
|
||||||
|
worker_log_path=lambda _task_id, board: tmp_path / "worker.log",
|
||||||
|
_resolve_worktree_workspace=lambda _task, board: (tmp_path, "wt/t_history"),
|
||||||
|
set_branch_name=lambda *_args: None,
|
||||||
|
set_workspace_path=lambda *_args: None,
|
||||||
|
build_worker_context=lambda *_args: "achieve the goal with evidence",
|
||||||
|
heartbeat_worker=lambda *_args, **_kwargs: True,
|
||||||
|
add_comment=lambda *_args: None,
|
||||||
|
complete_task=lambda *_args, **kwargs: (
|
||||||
|
calls.append(("complete", kwargs)) or True
|
||||||
|
),
|
||||||
|
block_task=lambda *_args, **kwargs: calls.append(("block", kwargs)),
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "hermes_cli", SimpleNamespace(kanban_db=fake_db))
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||||
|
state_file = lanes.state_path("cassandra", "t_history")
|
||||||
|
lanes.atomic_json(state_file, {"goal_rejections": "corrupt-history"})
|
||||||
|
probes = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"fresh_unavailable_provider",
|
||||||
|
lambda: probes.append("health") and None,
|
||||||
|
)
|
||||||
|
route = lanes.Route(
|
||||||
|
"codex", "gpt-5.6-sol", "high", "codex-high", "jetson", "vote", 1, ()
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(lanes, "select_route", lambda *_args, **_kwargs: route)
|
||||||
|
reports = [
|
||||||
|
lanes.ProcessResult(
|
||||||
|
0,
|
||||||
|
"first turn",
|
||||||
|
{
|
||||||
|
"status": "completed",
|
||||||
|
"summary": "done",
|
||||||
|
"changed_files": ["src/a.py"],
|
||||||
|
"tests_run": ["pytest -q"],
|
||||||
|
"artifacts": [],
|
||||||
|
"findings": [],
|
||||||
|
"blockers": [],
|
||||||
|
},
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
lanes.ProcessResult(
|
||||||
|
0,
|
||||||
|
"second turn",
|
||||||
|
{
|
||||||
|
"status": "completed",
|
||||||
|
"summary": "done with verification",
|
||||||
|
"changed_files": ["src/a.py"],
|
||||||
|
"tests_run": ["pytest -q: passed"],
|
||||||
|
"artifacts": [],
|
||||||
|
"findings": [],
|
||||||
|
"blockers": [],
|
||||||
|
},
|
||||||
|
False,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
monkeypatch.setattr(lanes, "run_provider", lambda *_args, **_kwargs: reports.pop(0))
|
||||||
|
verdicts = iter([(False, "verification evidence missing"), (True, "verified")])
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes.cli_lane_goal,
|
||||||
|
"judge_goal_completion",
|
||||||
|
lambda *_args, **_kwargs: next(verdicts),
|
||||||
|
)
|
||||||
|
|
||||||
|
lanes.execute_claim("cassandra", "t_history")
|
||||||
|
|
||||||
|
assert reports == []
|
||||||
|
assert calls[0][0] == "complete"
|
||||||
|
assert calls[0][1]["metadata"]["goal_turn"] == 2
|
||||||
|
assert len(probes) == 2
|
||||||
|
state = lanes.load_json(state_file)
|
||||||
|
assert state["goal_rejections"] == [
|
||||||
|
"local goal judge requested continuation: verification evidence missing"
|
||||||
|
]
|
||||||
|
|||||||
@ -118,3 +118,14 @@ def test_goal_judge_fails_closed_on_invalid_response():
|
|||||||
|
|
||||||
assert accepted is False
|
assert accepted is False
|
||||||
assert "local completion judge unavailable" in reason
|
assert "local completion judge unavailable" in reason
|
||||||
|
|
||||||
|
|
||||||
|
def test_completed_report_with_corrupt_test_evidence_is_still_inspected():
|
||||||
|
"""A non-list tests_run field neither crashes nor blocks the summary gate."""
|
||||||
|
assert goal.unfinished_result_reason(_result(tests_run=None)) is None
|
||||||
|
assert goal.unfinished_result_reason(
|
||||||
|
_result(
|
||||||
|
tests_run={"suite": "still running"},
|
||||||
|
summary="The remaining work is still running.",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|||||||
238
testing/tests/test_hermes_cli_matrix_capabilities.py
Normal file
238
testing/tests/test_hermes_cli_matrix_capabilities.py
Normal file
@ -0,0 +1,238 @@
|
|||||||
|
"""Legacy/patched image compatibility matrix for mixed Hermes rollouts."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from contextlib import nullcontext
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from testing.tests.test_hermes_cli_capabilities import (
|
||||||
|
_new_reclaim,
|
||||||
|
_old_reclaim,
|
||||||
|
_terminal_document,
|
||||||
|
)
|
||||||
|
from testing.tests.test_hermes_cli_support import lanes
|
||||||
|
|
||||||
|
def _coordinator_db(*, patched: bool):
|
||||||
|
"""Model the image decomposition API used by the coordinator container."""
|
||||||
|
if patched:
|
||||||
|
|
||||||
|
def specify(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
title=None,
|
||||||
|
body=None,
|
||||||
|
assignee=None,
|
||||||
|
author=None,
|
||||||
|
require_no_runs=False,
|
||||||
|
):
|
||||||
|
return not require_no_runs
|
||||||
|
|
||||||
|
def decompose(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
root_assignee,
|
||||||
|
children,
|
||||||
|
author=None,
|
||||||
|
auto_promote=True,
|
||||||
|
require_no_runs=False,
|
||||||
|
):
|
||||||
|
return None if require_no_runs else [f"{_task_id}.1"]
|
||||||
|
|
||||||
|
else:
|
||||||
|
|
||||||
|
def specify(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
title=None,
|
||||||
|
body=None,
|
||||||
|
assignee=None,
|
||||||
|
author=None,
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def decompose(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
root_assignee,
|
||||||
|
children,
|
||||||
|
author=None,
|
||||||
|
auto_promote=True,
|
||||||
|
):
|
||||||
|
return [f"{_task_id}.1"]
|
||||||
|
|
||||||
|
return SimpleNamespace(
|
||||||
|
specify_triage_task=specify,
|
||||||
|
decompose_triage_task=decompose,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("worker_patched", [False, True])
|
||||||
|
@pytest.mark.parametrize("coordinator_patched", [False, True])
|
||||||
|
def test_mixed_image_rollout_compatibility_matrix(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
coordinator_patched: bool,
|
||||||
|
worker_patched: bool,
|
||||||
|
):
|
||||||
|
"""Exact legacy/patched image matrix for decomposition and finalization.
|
||||||
|
|
||||||
|
The hermes coordinator container and the cli-lane-runner worker container
|
||||||
|
can run different image generations during a rollout while sharing one
|
||||||
|
Kanban SQLite database and these mounted scripts. The behaviors that
|
||||||
|
differ between the legacy and the patched image on this path are:
|
||||||
|
|
||||||
|
* legacy ``complete_task`` accepts ``expected_run_id`` but not
|
||||||
|
``replay_ended_run_id``: active exact runs complete on both images,
|
||||||
|
ended-run replay is image-gated.
|
||||||
|
* legacy ``reclaim_task`` lacks ``expected_run_id``: orphan reclaim is
|
||||||
|
image-gated (covered by test_old_image_never_calls_unguarded_reclaim).
|
||||||
|
* legacy ``specify_triage_task``/``decompose_triage_task`` lack
|
||||||
|
``require_no_runs``: only the patched coordinator refuses to redefine
|
||||||
|
a task that already has execution history.
|
||||||
|
|
||||||
|
Combination behavior proven here:
|
||||||
|
|
||||||
|
* legacy coordinator + legacy worker: mid-run decomposition succeeds
|
||||||
|
unguarded; the worker defers the ended run's accepted result (journal
|
||||||
|
preserved for a patched replay) and reports deferred health.
|
||||||
|
* legacy coordinator + patched worker: mid-run decomposition succeeds
|
||||||
|
unguarded; the patched worker replays under ``replay_ended_run_id``,
|
||||||
|
the database refuses the voided run, and the result converges to
|
||||||
|
``stale`` (conflict evidence) instead of completing a redefined task.
|
||||||
|
* patched coordinator + legacy worker: decomposition is refused while
|
||||||
|
the run exists, so the still-running exact run completes under
|
||||||
|
``expected_run_id``; replay/reclaim stay deferred and the readiness
|
||||||
|
probe keeps the worker pod unready for new claims.
|
||||||
|
* patched coordinator + patched worker: decomposition is refused while
|
||||||
|
the run exists, the exact run completes, and the worker is ready.
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "cli-lanes")
|
||||||
|
task = SimpleNamespace(
|
||||||
|
id="t_matrix",
|
||||||
|
status="running",
|
||||||
|
current_run_id=7,
|
||||||
|
completed_run_id=None,
|
||||||
|
result=None,
|
||||||
|
assignee="cli-auto",
|
||||||
|
)
|
||||||
|
|
||||||
|
def old_matrix_complete(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
result=None,
|
||||||
|
summary=None,
|
||||||
|
metadata=None,
|
||||||
|
expected_run_id=None,
|
||||||
|
):
|
||||||
|
if task.status == "running" and task.current_run_id == expected_run_id:
|
||||||
|
task.status = "done"
|
||||||
|
task.completed_run_id = expected_run_id
|
||||||
|
task.current_run_id = None
|
||||||
|
task.result = result
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
def new_matrix_complete(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
*,
|
||||||
|
result=None,
|
||||||
|
summary=None,
|
||||||
|
metadata=None,
|
||||||
|
expected_run_id=None,
|
||||||
|
replay_ended_run_id=None,
|
||||||
|
):
|
||||||
|
if replay_ended_run_id is not None:
|
||||||
|
# The decomposed run is no longer the latest ended run.
|
||||||
|
return False
|
||||||
|
return old_matrix_complete(
|
||||||
|
_conn,
|
||||||
|
_task_id,
|
||||||
|
result=result,
|
||||||
|
summary=summary,
|
||||||
|
metadata=metadata,
|
||||||
|
expected_run_id=expected_run_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
worker_db = SimpleNamespace(
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda board: SimpleNamespace(close=lambda: None),
|
||||||
|
get_task=lambda _conn, _task_id: task,
|
||||||
|
complete_task=new_matrix_complete if worker_patched else old_matrix_complete,
|
||||||
|
reclaim_task=_new_reclaim if worker_patched else _old_reclaim,
|
||||||
|
)
|
||||||
|
coordinator_db = _coordinator_db(patched=coordinator_patched)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
lanes._explicit_keyword(
|
||||||
|
coordinator_db.decompose_triage_task,
|
||||||
|
"require_no_runs",
|
||||||
|
)
|
||||||
|
is coordinator_patched
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
lanes._explicit_keyword(
|
||||||
|
coordinator_db.specify_triage_task,
|
||||||
|
"require_no_runs",
|
||||||
|
)
|
||||||
|
is coordinator_patched
|
||||||
|
)
|
||||||
|
health = tmp_path / "runtime-health.json"
|
||||||
|
capabilities = lanes.initialize_kanban_capabilities(
|
||||||
|
worker_db,
|
||||||
|
health_path=health,
|
||||||
|
)
|
||||||
|
assert capabilities.ready is worker_patched
|
||||||
|
assert capabilities.exact_run_completion is True
|
||||||
|
if worker_patched:
|
||||||
|
assert lanes.readiness_issue(health) is None
|
||||||
|
else:
|
||||||
|
assert capabilities.deferred_features == (
|
||||||
|
"ended-run-replay",
|
||||||
|
"exact-run-reclaim",
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
lanes.readiness_issue(health) == "runtime compatibility is deferred"
|
||||||
|
)
|
||||||
|
|
||||||
|
if coordinator_patched:
|
||||||
|
refused = coordinator_db.decompose_triage_task(
|
||||||
|
object(),
|
||||||
|
"t_matrix",
|
||||||
|
root_assignee="cli-auto",
|
||||||
|
children=[{"title": "split"}],
|
||||||
|
require_no_runs=True,
|
||||||
|
)
|
||||||
|
assert refused is None
|
||||||
|
else:
|
||||||
|
created = coordinator_db.decompose_triage_task(
|
||||||
|
object(),
|
||||||
|
"t_matrix",
|
||||||
|
root_assignee="cli-auto",
|
||||||
|
children=[{"title": "split"}],
|
||||||
|
)
|
||||||
|
assert created == ["t_matrix.1"]
|
||||||
|
task.status = "triage"
|
||||||
|
task.current_run_id = None
|
||||||
|
|
||||||
|
identity = lanes.TerminalIdentity("cassandra", "t_matrix", 7, "pending")
|
||||||
|
outcome = lanes._finalize_document_db(worker_db, identity, _terminal_document())
|
||||||
|
|
||||||
|
if coordinator_patched:
|
||||||
|
assert outcome == "committed"
|
||||||
|
assert task.status == "done"
|
||||||
|
assert task.completed_run_id == 7
|
||||||
|
elif worker_patched:
|
||||||
|
assert outcome == "stale"
|
||||||
|
assert task.status == "triage"
|
||||||
|
else:
|
||||||
|
assert outcome == "deferred"
|
||||||
|
assert task.status == "triage"
|
||||||
@ -171,3 +171,168 @@ def test_codex_missing_thread_skips_colliding_restart_result(
|
|||||||
)
|
)
|
||||||
restart = Path(calls[1][calls[1].index("-o") + 1])
|
restart = Path(calls[1][calls[1].index("-o") + 1])
|
||||||
assert restart.name.endswith("provider-3.result.json")
|
assert restart.name.endswith("provider-3.result.json")
|
||||||
|
|
||||||
|
|
||||||
|
def test_stream_process_trims_the_retained_line_window(tmp_path: Path, monkeypatch):
|
||||||
|
"""Streamed chatter keeps only the newest bounded output window."""
|
||||||
|
monkeypatch.setattr(lanes, "_descendant_processes", lambda _pid: {})
|
||||||
|
monkeypatch.setattr(lanes, "_terminate_worker_process", lambda *_a, **_k: None)
|
||||||
|
read_fd, write_fd = os.pipe()
|
||||||
|
os.write(write_fd, "".join(f"{index}\n" for index in range(4100)).encode())
|
||||||
|
os.close(write_fd)
|
||||||
|
polls = []
|
||||||
|
|
||||||
|
class ChatteringProcess:
|
||||||
|
pid = 4243
|
||||||
|
returncode = 0
|
||||||
|
stdout = os.fdopen(read_fd, "r")
|
||||||
|
|
||||||
|
def poll(self):
|
||||||
|
polls.append(True)
|
||||||
|
return None if len(polls) < 4300 else 0
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes.subprocess,
|
||||||
|
"Popen",
|
||||||
|
lambda *_args, **_kwargs: ChatteringProcess(),
|
||||||
|
)
|
||||||
|
heartbeats = []
|
||||||
|
|
||||||
|
result = lanes.stream_process(
|
||||||
|
["ignored"],
|
||||||
|
provider="codex",
|
||||||
|
cwd=tmp_path,
|
||||||
|
env=dict(os.environ),
|
||||||
|
log_path=tmp_path / "chatter.log",
|
||||||
|
state={},
|
||||||
|
state_file=tmp_path / "chatter-state.json",
|
||||||
|
heartbeat=lambda note: heartbeats.append(note) or True,
|
||||||
|
max_runtime=60,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert heartbeats
|
||||||
|
published = result.output.splitlines()
|
||||||
|
assert len(published) == 4000
|
||||||
|
assert published[0] == "100"
|
||||||
|
assert published[-1] == "4099"
|
||||||
|
|
||||||
|
|
||||||
|
def test_stream_process_parses_output_left_after_worker_exit(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Buffered final output from an already-exited worker is still parsed."""
|
||||||
|
payload = json.dumps({"result": json.dumps(_completed_result("buffered"))})
|
||||||
|
read_fd, write_fd = os.pipe()
|
||||||
|
os.write(write_fd, f"{payload}\n".encode())
|
||||||
|
os.close(write_fd)
|
||||||
|
fake = type(
|
||||||
|
"ExitedProcess",
|
||||||
|
(),
|
||||||
|
{
|
||||||
|
"pid": 4242,
|
||||||
|
"returncode": 0,
|
||||||
|
"poll": lambda self: 0,
|
||||||
|
"stdout": os.fdopen(read_fd, "r"),
|
||||||
|
},
|
||||||
|
)()
|
||||||
|
monkeypatch.setattr(lanes.subprocess, "Popen", lambda *_args, **_kwargs: fake)
|
||||||
|
monkeypatch.setattr(lanes, "_terminate_worker_process", lambda *_a, **_k: None)
|
||||||
|
|
||||||
|
result = lanes.stream_process(
|
||||||
|
["ignored"],
|
||||||
|
provider="codex",
|
||||||
|
cwd=tmp_path,
|
||||||
|
env={},
|
||||||
|
log_path=tmp_path / "buffered.log",
|
||||||
|
state={},
|
||||||
|
state_file=tmp_path / "buffered-state.json",
|
||||||
|
heartbeat=lambda _note: True,
|
||||||
|
max_runtime=60,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 0
|
||||||
|
assert result.structured and result.structured["summary"] == "buffered"
|
||||||
|
assert payload in (tmp_path / "buffered.log").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def test_descendant_snapshot_ignores_processes_that_vanish(monkeypatch):
|
||||||
|
entries = [Path("/proc/10"), Path("/proc/11"), Path("/proc/13")]
|
||||||
|
monkeypatch.setattr(lanes.Path, "iterdir", lambda _path: iter(entries))
|
||||||
|
records = {10: (1, 10, 100), 11: (10, 11, 101)}
|
||||||
|
monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid))
|
||||||
|
|
||||||
|
assert lanes._descendant_processes(10) == {11: (11, 101)}
|
||||||
|
|
||||||
|
|
||||||
|
def test_signal_tree_targets_only_identity_verified_descendants(monkeypatch):
|
||||||
|
"""Reused PIDs are never signaled; verified groups and PIDs both are."""
|
||||||
|
records = {10: (1, 20, 100)}
|
||||||
|
monkeypatch.setattr(lanes, "_process_record", lambda pid: records.get(pid))
|
||||||
|
group_signals = []
|
||||||
|
pid_signals = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes.os,
|
||||||
|
"killpg",
|
||||||
|
lambda group, sig: group_signals.append((group, sig)),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes.os,
|
||||||
|
"kill",
|
||||||
|
lambda pid, sig: pid_signals.append((pid, sig)),
|
||||||
|
)
|
||||||
|
|
||||||
|
lanes._signal_worker_tree(
|
||||||
|
5,
|
||||||
|
{10: (20, 100), 11: (21, 999)},
|
||||||
|
signal.SIGTERM,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert sorted(group_signals) == [(5, signal.SIGTERM), (20, signal.SIGTERM)]
|
||||||
|
assert pid_signals == [(10, signal.SIGTERM)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_claude_worker_without_any_recoverable_session_stays_unstarted(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A session lost on resume and restart never marks the lane as started."""
|
||||||
|
state = {"claude_started": True, "claude_session_id": "session-lost"}
|
||||||
|
state_file = tmp_path / "state.json"
|
||||||
|
commands = []
|
||||||
|
|
||||||
|
def stream(command, **_kwargs):
|
||||||
|
commands.append(command)
|
||||||
|
return lanes.ProcessResult(
|
||||||
|
1,
|
||||||
|
f"{lanes.NO_CLAUDE_SESSION} session-lost",
|
||||||
|
None,
|
||||||
|
False,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes, "stream_process", stream)
|
||||||
|
persisted = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
sys.modules["cli_lane_provider"],
|
||||||
|
"atomic_json",
|
||||||
|
lambda path, value: persisted.append(dict(value)),
|
||||||
|
)
|
||||||
|
route = lanes.Route("claude", "claude-fable-5", "high", "p", "c", "r", 1, ())
|
||||||
|
|
||||||
|
result = lanes.run_provider(
|
||||||
|
route,
|
||||||
|
"prompt",
|
||||||
|
tmp_path,
|
||||||
|
state,
|
||||||
|
state_file,
|
||||||
|
tmp_path / "worker.log",
|
||||||
|
lambda _note: True,
|
||||||
|
60,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode == 1
|
||||||
|
assert len(commands) == 2
|
||||||
|
assert "--resume" in commands[0]
|
||||||
|
assert "--session-id" in commands[1]
|
||||||
|
assert len(persisted) == 1
|
||||||
|
|||||||
@ -228,3 +228,66 @@ def test_terminal_schema_validates_optional_identity():
|
|||||||
record,
|
record,
|
||||||
lanes.TerminalIdentity("other", "task", 1, "pending"),
|
lanes.TerminalIdentity("other", "task", 1, "pending"),
|
||||||
) is False
|
) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_terminal_identity_rejects_noncanonical_run_ids():
|
||||||
|
"""Identity construction fails closed before SQLite can see the run."""
|
||||||
|
with pytest.raises(ValueError, match="positive SQLite int64"):
|
||||||
|
lanes.TerminalIdentity("cassandra", "t_identity", 0, "pending")
|
||||||
|
with pytest.raises(ValueError, match="positive SQLite int64"):
|
||||||
|
lanes.TerminalIdentity("cassandra", "t_identity", "7", "pending")
|
||||||
|
|
||||||
|
|
||||||
|
def test_small_snapshot_rejects_payloads_that_grow_past_the_stat_bound(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A file appended between fstat and read never yields a snapshot."""
|
||||||
|
path = tmp_path / "grown.json"
|
||||||
|
path.write_text("{}", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"_read_bounded",
|
||||||
|
lambda _descriptor, limit: b"x" * (lanes.MAX_TERMINAL_RECORD_BYTES + 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert lanes._open_small_json_snapshot(path) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_snapshot_marks_racing_writers_as_unstable(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A payload that changes under the pinned read cannot be classified."""
|
||||||
|
path = tmp_path / "unstable.json"
|
||||||
|
path.write_text('{"kanban_state": "pending"}', encoding="utf-8")
|
||||||
|
real_read = lanes._read_bounded
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"_read_bounded",
|
||||||
|
lambda descriptor, limit: real_read(descriptor, limit) + b" ",
|
||||||
|
)
|
||||||
|
|
||||||
|
snapshot = lanes._open_terminal_recovery_snapshot(path)
|
||||||
|
|
||||||
|
assert snapshot is not None
|
||||||
|
assert snapshot.document is None
|
||||||
|
assert snapshot.invalid_reason == "unstable-payload"
|
||||||
|
snapshot.close()
|
||||||
|
snapshot.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_snapshot_releases_descriptors_on_read_faults(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""An I/O fault during the pinned read closes both held descriptors."""
|
||||||
|
path = tmp_path / "faulted.json"
|
||||||
|
path.write_text("{}", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"_read_bounded",
|
||||||
|
lambda _descriptor, _limit: (_ for _ in ()).throw(OSError("read fault")),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert lanes._open_terminal_recovery_snapshot(path) is None
|
||||||
|
|||||||
@ -5,6 +5,7 @@ from __future__ import annotations
|
|||||||
import hashlib
|
import hashlib
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
from contextlib import nullcontext
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
@ -296,3 +297,148 @@ def test_pending_guard_rejects_invalid_run_and_inaccessible_or_bad_artifacts(
|
|||||||
|
|
||||||
monkeypatch.setattr(Path, "stat", denied)
|
monkeypatch.setattr(Path, "stat", denied)
|
||||||
assert lanes._has_pending_finalization("board", "task", 12) is False
|
assert lanes._has_pending_finalization("board", "task", 12) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_staged_authority_rejects_a_mismatched_pathname_digest(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A staged entry whose name does not bind its payload has no authority."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
document = _pending_terminal_record("board", "task", 5, "accepted")
|
||||||
|
board = lanes.STATE_ROOT / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
forged = board / f".retire.{'f' * 16}.{'a' * 16}.0"
|
||||||
|
|
||||||
|
assert lanes._staged_terminal_authority(forged, _Snapshot(document)) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_drain_staging_defers_unbounded_replacement_churn(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A staging surface that keeps changing stops after bounded attempts."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
board = lanes.STATE_ROOT / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
churning = board / ".retire.churning"
|
||||||
|
churning.write_text("0", encoding="utf-8")
|
||||||
|
|
||||||
|
def churn():
|
||||||
|
churning.write_text(churning.read_text(encoding="utf-8") + "x")
|
||||||
|
return 1
|
||||||
|
|
||||||
|
monkeypatch.setattr(lanes, "_recover_retirement_staging", churn)
|
||||||
|
|
||||||
|
assert lanes._drain_retirement_staging() == 16
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepared_recovery_orders_and_bounds_disappearing_journals(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A journal whose stat fails is ordered last instead of aborting replay."""
|
||||||
|
path, _document = _prepared_path(tmp_path, monkeypatch)
|
||||||
|
real_stat = Path.stat
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def vanishing(candidate, *args, **kwargs):
|
||||||
|
if candidate == path and not calls:
|
||||||
|
calls.append("ordered")
|
||||||
|
raise OSError("stat raced")
|
||||||
|
return real_stat(candidate, *args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "stat", vanishing)
|
||||||
|
monkeypatch.setattr(lanes, "_finalize_document_db", lambda *_args: "deferred")
|
||||||
|
|
||||||
|
assert lanes._recover_prepared_finalizations(object()) == 0
|
||||||
|
assert calls == ["ordered"]
|
||||||
|
assert path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepared_recovery_defers_a_journal_that_cannot_be_quarantined(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""An unquarantinable malformed name never spins recovery forever."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
board = lanes.STATE_ROOT / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
misnamed = board / "task.run-9.terminal.prepared-zz.json"
|
||||||
|
misnamed.write_text("{}", encoding="utf-8")
|
||||||
|
attempts = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"_quarantine_terminal",
|
||||||
|
lambda *_args, **_kwargs: attempts.append("quarantine"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert lanes._recover_prepared_finalizations(object()) == 0
|
||||||
|
assert len(attempts) == 16
|
||||||
|
assert misnamed.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepared_recovery_counts_one_recovery_per_exact_run(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Duplicate prepared evidence for one run converges to a single winner."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
identity = lanes.TerminalIdentity("board", "task", 9, "pending")
|
||||||
|
first = _pending_terminal_record("board", "task", 9, "first accepted")
|
||||||
|
second = _pending_terminal_record("board", "task", 9, "second accepted")
|
||||||
|
lanes._persist_prepared_evidence(identity, first)
|
||||||
|
lanes._persist_prepared_evidence(identity, second)
|
||||||
|
task = SimpleNamespace(
|
||||||
|
id="task",
|
||||||
|
status="running",
|
||||||
|
current_run_id=9,
|
||||||
|
completed_run_id=None,
|
||||||
|
result=None,
|
||||||
|
assignee="cli-auto",
|
||||||
|
)
|
||||||
|
db = SimpleNamespace(
|
||||||
|
scoped_current_board=lambda _board: nullcontext(),
|
||||||
|
connect=lambda board: SimpleNamespace(close=lambda: None),
|
||||||
|
get_task=lambda _conn, _task_id: task,
|
||||||
|
complete_task=lambda *_args, **_kwargs: True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert lanes._recover_prepared_finalizations(db) == 1
|
||||||
|
conflicts = list((lanes.STATE_ROOT / "board").glob("*.terminal.conflict-*.json"))
|
||||||
|
assert len(conflicts) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_guard_ignores_symlinked_boards_and_invalid_artifacts(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Invalid journals, evidence, and staging never hold a claim open."""
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", tmp_path / "lanes")
|
||||||
|
board = lanes.STATE_ROOT / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
pending = lanes._terminal_path(lanes.state_path("board", "task"), 13)
|
||||||
|
pending.write_text("{}", encoding="utf-8")
|
||||||
|
misnamed = board / "task.run-13.terminal.prepared-zz.json"
|
||||||
|
misnamed.write_text("{}", encoding="utf-8")
|
||||||
|
invalid = board / f"task.run-13.terminal.prepared-{'a' * 32}.json"
|
||||||
|
invalid.write_text("{}", encoding="utf-8")
|
||||||
|
(board / ".retire.dangling").symlink_to(board / "missing-target")
|
||||||
|
foreign = _pending_terminal_record("board", "other", 13, "accepted")
|
||||||
|
staged, _canonical = _staged_path(
|
||||||
|
lanes.STATE_ROOT,
|
||||||
|
lanes.TerminalIdentity("board", "other", 13, "pending"),
|
||||||
|
foreign,
|
||||||
|
)
|
||||||
|
lanes.atomic_json(staged, foreign)
|
||||||
|
|
||||||
|
assert lanes._has_pending_finalization("board", "task", 13) is False
|
||||||
|
assert lanes._has_pending_finalization("board", "other", 13) is True
|
||||||
|
|
||||||
|
aliased_root = tmp_path / "aliased"
|
||||||
|
aliased_root.mkdir()
|
||||||
|
(aliased_root / "linked").symlink_to(board, target_is_directory=True)
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", aliased_root)
|
||||||
|
linked_pending = aliased_root / "linked" / "task.run-13.terminal.pending.json"
|
||||||
|
assert linked_pending.exists()
|
||||||
|
assert lanes._has_pending_finalization("linked", "task", 13) is False
|
||||||
|
|||||||
@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from testing.tests.test_hermes_cli_support import (
|
from testing.tests.test_hermes_cli_support import (
|
||||||
@ -87,3 +88,66 @@ def test_gc_skips_symlink_board_and_disappearing_candidate(
|
|||||||
|
|
||||||
monkeypatch.setattr(Path, "stat", disappear)
|
monkeypatch.setattr(Path, "stat", disappear)
|
||||||
assert lanes.gc_lane_artifacts() == 0
|
assert lanes.gc_lane_artifacts() == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_gc_reports_quarantine_that_left_the_entry_in_place(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""An invalid retained journal that survives quarantine is not counted."""
|
||||||
|
root = tmp_path / "lanes"
|
||||||
|
board = root / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
retained = board / "task.run-1.terminal.committed.json"
|
||||||
|
retained.write_text("{}", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", root)
|
||||||
|
monkeypatch.setattr(lanes, "_quarantine_terminal", lambda *_a, **_k: retained)
|
||||||
|
|
||||||
|
assert lanes.gc_lane_artifacts() == 0
|
||||||
|
assert retained.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_gc_bounds_unpinnable_staging_and_failed_nonregular_unlinks(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""Unsnapshotable staging entries fall through to identity-safe removal."""
|
||||||
|
root = tmp_path / "lanes"
|
||||||
|
board = root / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
dangling = board / ".retire.dangling"
|
||||||
|
dangling.symlink_to(board / "missing-target")
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", root)
|
||||||
|
attempts = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"_unlink_artifact_if_same",
|
||||||
|
lambda path, _observed: attempts.append(path.name) and False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert lanes.gc_lane_artifacts() == 0
|
||||||
|
assert attempts == [".retire.dangling"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_gc_keeps_entries_whose_capped_unlink_loses_the_race(
|
||||||
|
tmp_path: Path,
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""A lost inode race during count capping removes nothing else."""
|
||||||
|
root = tmp_path / "lanes"
|
||||||
|
board = root / "board"
|
||||||
|
board.mkdir(parents=True)
|
||||||
|
newer = board / "task.candidate-2.json"
|
||||||
|
newer.write_text("{}", encoding="utf-8")
|
||||||
|
older = board / "task.candidate-1.json"
|
||||||
|
older.write_text("{}", encoding="utf-8")
|
||||||
|
os.utime(older, (1, 1))
|
||||||
|
monkeypatch.setattr(lanes, "STATE_ROOT", root)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
lanes,
|
||||||
|
"_unlink_artifact_if_same",
|
||||||
|
lambda _path, _observed: False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert lanes.gc_lane_artifacts(now=100.0, max_age_seconds=10**9, max_count=1) == 0
|
||||||
|
assert older.exists() and newer.exists()
|
||||||
|
|||||||
@ -98,8 +98,8 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
|
|||||||
<packages>
|
<packages>
|
||||||
<package>
|
<package>
|
||||||
<classes>
|
<classes>
|
||||||
<class filename="ok.py" line-rate="1.0" />
|
<class filename="ok.py" line-rate="1.0" branch-rate="1.0" />
|
||||||
<class filename="low.py" line-rate="0.90" />
|
<class filename="low.py" line-rate="0.90" branch-rate="1.0" />
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
</packages>
|
</packages>
|
||||||
@ -118,7 +118,7 @@ def test_coverage_check_enforces_per_file_floor(tmp_path: Path):
|
|||||||
|
|
||||||
issues = run_coverage_check(contract, tmp_path, coverage_xml)
|
issues = run_coverage_check(contract, tmp_path, coverage_xml)
|
||||||
|
|
||||||
assert "coverage below 95.0%: low.py (90.0%)" in issues
|
assert "line coverage below 95.0%: low.py (90.0%)" in issues
|
||||||
assert "coverage missing for tracked file: missing.py" in issues
|
assert "coverage missing for tracked file: missing.py" in issues
|
||||||
|
|
||||||
|
|
||||||
@ -142,9 +142,8 @@ def test_coverage_check_handles_missing_xml_and_source_root_mapping(tmp_path: Pa
|
|||||||
<packages>
|
<packages>
|
||||||
<package>
|
<package>
|
||||||
<classes>
|
<classes>
|
||||||
<class filename="mapped.py" line-rate="1.0" />
|
<class filename="mapped.py" line-rate="1.0" branch-rate="1.0" />
|
||||||
<class filename="{(tmp_path / 'absolute.py').as_posix()}" line-rate="1.0" />
|
<class filename="{(tmp_path / 'absolute.py').as_posix()}" line-rate="1.0" branch-rate="1.0" />
|
||||||
<class filename="skip.py" />
|
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
</packages>
|
</packages>
|
||||||
|
|||||||
@ -5,6 +5,8 @@ from __future__ import annotations
|
|||||||
import textwrap
|
import textwrap
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
from testing.quality_coverage import compute_workspace_line_coverage, run_check
|
from testing.quality_coverage import compute_workspace_line_coverage, run_check
|
||||||
|
|
||||||
|
|
||||||
@ -12,10 +14,15 @@ def test_compute_workspace_line_coverage_handles_missing_xml(tmp_path: Path) ->
|
|||||||
"""Missing coverage XML should produce a zero workspace coverage score."""
|
"""Missing coverage XML should produce a zero workspace coverage score."""
|
||||||
|
|
||||||
contract = {"coverage": {"tracked_files": ["managed.py"]}}
|
contract = {"coverage": {"tracked_files": ["managed.py"]}}
|
||||||
assert compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml") == 0.0
|
assert (
|
||||||
|
compute_workspace_line_coverage(contract, tmp_path, tmp_path / "missing.xml")
|
||||||
|
== 0.0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path: Path) -> None:
|
def test_compute_workspace_line_coverage_averages_present_tracked_files(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
"""Workspace coverage should average only tracked files that appear in the report."""
|
"""Workspace coverage should average only tracked files that appear in the report."""
|
||||||
|
|
||||||
coverage_xml = tmp_path / "coverage.xml"
|
coverage_xml = tmp_path / "coverage.xml"
|
||||||
@ -26,8 +33,8 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path
|
|||||||
<packages>
|
<packages>
|
||||||
<package>
|
<package>
|
||||||
<classes>
|
<classes>
|
||||||
<class filename="alpha.py" line-rate="1.0" />
|
<class filename="alpha.py" line-rate="1.0" branch-rate="1.0" />
|
||||||
<class filename="beta.py" line-rate="0.5" />
|
<class filename="beta.py" line-rate="0.5" branch-rate="1.0" />
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
</packages>
|
</packages>
|
||||||
@ -41,7 +48,9 @@ def test_compute_workspace_line_coverage_averages_present_tracked_files(tmp_path
|
|||||||
assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0
|
assert compute_workspace_line_coverage(contract, tmp_path, coverage_xml) == 75.0
|
||||||
|
|
||||||
|
|
||||||
def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path: Path) -> None:
|
def test_run_check_keeps_relative_names_when_source_roots_do_not_match(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
"""Relative filenames should remain relative when no declared source root contains them."""
|
"""Relative filenames should remain relative when no declared source root contains them."""
|
||||||
|
|
||||||
coverage_xml = tmp_path / "coverage.xml"
|
coverage_xml = tmp_path / "coverage.xml"
|
||||||
@ -57,7 +66,7 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path:
|
|||||||
<packages>
|
<packages>
|
||||||
<package>
|
<package>
|
||||||
<classes>
|
<classes>
|
||||||
<class filename="relative.py" line-rate="0.80" />
|
<class filename="relative.py" line-rate="0.80" branch-rate="1.0" />
|
||||||
</classes>
|
</classes>
|
||||||
</package>
|
</package>
|
||||||
</packages>
|
</packages>
|
||||||
@ -73,4 +82,68 @@ def test_run_check_keeps_relative_names_when_source_roots_do_not_match(tmp_path:
|
|||||||
coverage_xml,
|
coverage_xml,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert issues == ["coverage below 95.0%: relative.py (80.0%)"]
|
assert issues == ["line coverage below 95.0%: relative.py (80.0%)"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("branch_rate", [None, "nan", "inf", "-0.1", "1.1", "broken"])
|
||||||
|
def test_run_check_rejects_missing_or_invalid_branch_rates(
|
||||||
|
tmp_path: Path,
|
||||||
|
branch_rate: str | None,
|
||||||
|
) -> None:
|
||||||
|
"""Tracked files must carry finite Cobertura branch evidence."""
|
||||||
|
attribute = "" if branch_rate is None else f' branch-rate="{branch_rate}"'
|
||||||
|
coverage_xml = tmp_path / "coverage.xml"
|
||||||
|
coverage_xml.write_text(
|
||||||
|
f'<coverage><class filename="managed.py" line-rate="1"{attribute}/></coverage>',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
issues = run_check(
|
||||||
|
{"coverage": {"tracked_files": ["managed.py"]}},
|
||||||
|
tmp_path,
|
||||||
|
coverage_xml,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert len(issues) == 1
|
||||||
|
assert issues[0].startswith("coverage xml invalid:")
|
||||||
|
assert (
|
||||||
|
compute_workspace_line_coverage(
|
||||||
|
{"coverage": {"tracked_files": ["managed.py"]}},
|
||||||
|
tmp_path,
|
||||||
|
coverage_xml,
|
||||||
|
)
|
||||||
|
== 0.0
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_check_enforces_line_and_branch_rates_independently(tmp_path: Path) -> None:
|
||||||
|
"""A strong line score must not hide weak branch coverage."""
|
||||||
|
coverage_xml = tmp_path / "coverage.xml"
|
||||||
|
coverage_xml.write_text(
|
||||||
|
'<coverage><class filename="managed.py" line-rate="1" branch-rate="0.5"/></coverage>',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert run_check(
|
||||||
|
{"coverage": {"minimum_percent": 95, "tracked_files": ["managed.py"]}},
|
||||||
|
tmp_path,
|
||||||
|
coverage_xml,
|
||||||
|
) == ["branch coverage below 95.0%: managed.py (50.0%)"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_check_accepts_cobertura_branchless_file_semantics(tmp_path: Path) -> None:
|
||||||
|
"""Coverage.py reports branchless files with branch-rate one."""
|
||||||
|
coverage_xml = tmp_path / "coverage.xml"
|
||||||
|
coverage_xml.write_text(
|
||||||
|
'<coverage><class filename="managed.py" line-rate="1" branch-rate="1"/></coverage>',
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
run_check(
|
||||||
|
{"coverage": {"minimum_percent": 95, "tracked_files": ["managed.py"]}},
|
||||||
|
tmp_path,
|
||||||
|
coverage_xml,
|
||||||
|
)
|
||||||
|
== []
|
||||||
|
)
|
||||||
|
|||||||
@ -4,7 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from testing.quality_hygiene import count_files_over_line_limit
|
from testing.quality_hygiene import count_files_over_line_limit, run_check
|
||||||
|
|
||||||
|
|
||||||
def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> None:
|
def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) -> None:
|
||||||
@ -24,3 +24,28 @@ def test_count_files_over_line_limit_counts_only_long_matches(tmp_path: Path) ->
|
|||||||
}
|
}
|
||||||
|
|
||||||
assert count_files_over_line_limit(contract, tmp_path) == 1
|
assert count_files_over_line_limit(contract, tmp_path) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_check_exempts_conftest_from_naming_rules(tmp_path: Path) -> None:
|
||||||
|
"""Shared conftest.py fixtures never violate test naming rules."""
|
||||||
|
|
||||||
|
tests_dir = tmp_path / "tests"
|
||||||
|
tests_dir.mkdir()
|
||||||
|
(tests_dir / "conftest.py").write_text("fixtures = True\n", encoding="utf-8")
|
||||||
|
(tests_dir / "test_named.py").write_text("checked = True\n", encoding="utf-8")
|
||||||
|
|
||||||
|
contract = {
|
||||||
|
"hygiene": {
|
||||||
|
"max_lines": 3,
|
||||||
|
"line_limit_globs": [],
|
||||||
|
"naming_rules": [
|
||||||
|
{
|
||||||
|
"glob": "tests/*.py",
|
||||||
|
"pattern": "^test_[a-z0-9_]+\\.py$",
|
||||||
|
"description": "pytest files use test_*.py names",
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
assert run_check(contract, tmp_path) == []
|
||||||
|
|||||||
@ -168,3 +168,24 @@ def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Pat
|
|||||||
payload = json.loads(output.read_text(encoding="utf-8"))
|
payload = json.loads(output.read_text(encoding="utf-8"))
|
||||||
assert payload["status"] == "failed"
|
assert payload["status"] == "failed"
|
||||||
assert payload["blocking_findings"] == 1
|
assert payload["blocking_findings"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_sonar_issues_normalizes_inverted_text_ranges() -> None:
|
||||||
|
"""An end line before the start line collapses to the start line."""
|
||||||
|
|
||||||
|
issues = semgrep_report.build_sonar_issues(
|
||||||
|
{
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"path": "app/config.yaml",
|
||||||
|
"start": {"line": 9},
|
||||||
|
"end": {"line": 4},
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
assert issues["issues"][0]["primaryLocation"]["textRange"] == {
|
||||||
|
"startLine": 9,
|
||||||
|
"endLine": 9,
|
||||||
|
}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user