Compare commits

..

7 Commits

Author SHA1 Message Date
jenkins
5ab0eba619 hermes: add daemonless agent image release lane 2026-08-17 06:50:57 -03:00
ab346f5550 Merge pull request 'hermes: harden worker isolation and blocked-task semantics' (#13) from wt/t_cca008de into main
Reviewed-on: atlas/titan-iac#13
2026-08-17 08:33:41 +00:00
flux-bot
d9a88713a7 chore(maintenance): automated image update 2026-08-17 05:31:09 +00:00
flux-bot
9ada3ba6d0 chore(maintenance): automated image update 2026-08-17 02:00:37 +00:00
flux-bot
b4f78a615b chore(maintenance): automated image update 2026-08-17 01:51:36 +00:00
Hermes Agent
750dfa241f hermes: fail closed on kanban created-event producer drift
The sticky-block gate added in the previous commit classifies a task from
the `created` event payload that upstream `create_task` writes. That
producer is code we do not own, so trusting it silently was the gap: if
upstream renamed the key, dropped it, or stopped deriving it from
`initial_status`, the image would still build and ship a consumer that
mis-classifies every task it reads.

Anchor the producer contract at build time, before the regression suite
runs, with three assert-only preconditions: the `initial_status="blocked"`
park resolves `task_status` to `"blocked"`, every non-park creation
resolves it to something else, and the `created` event carries that same
variable under `"status"`. None of them rewrite the producer.

Textual anchors cannot see dataflow, so add the runtime net the reviewer
asked for. The suite now drives the real API: create + claim an ordinary
task, trip the circuit breaker once at failure_limit=1 so it parks with a
`gave_up` event (leaving its own `created` event as the most recent
create/block/unblock row), then recompute at failure_limit=2 and require
promotion to ready. That case is red under an unconditional-true created
predicate and red under producer drift that labels every created event
blocked, while the explicit block/unblock, dependency-promotion and
circuit-breaker-at-current-limit cases stay green. Non-blocked and
malformed created payloads are pinned as controls, and the gate now
rejects non-dict payloads rather than trusting `.get`.

Also make the live placement correction durable: titan-04 is cordoned
after repeated kernel undervoltage and kubelet failure and titan-19 was
probe/Longhorn unstable under worker load, so both join the hard NotIn
list; titan-05 is healthy but sits at 3592m/3600m requested CPU, so the
main hermes container gives back 50m (350m -> 300m) to schedule there.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-16 22:42:18 +00:00
Hermes Agent
4f8dcfbbf7 hermes: harden worker isolation and blocked-task semantics
Three narrowly scoped Hermes reliability fixes backed by live evidence
from the Cassandra/titan-iac proof run.

Worker concurrency. Three simultaneous direct CLI workers on the 4-core
hermes-agent node drove load to ~45 and made the hermes and oauth2-proxy
containers fail their probes, leaving the pod 8/10 Ready; two workers
stayed at 10/10. Cap HERMES_CLI_LANE_CONCURRENCY at 2 and lower the
cli-lane-runner CPU limit from 3 to 2 so the dashboard and auth sidecars
keep a guaranteed share of the node. Requests are unchanged: the pod
still asks for 745m total, so placement does not move.

Service links. Kubernetes injects a service-link variable pair for every
service in the namespace, and hermes-claude-broker produces
HERMES_CLAUDE_BROKER_PORT=tcp://10.43.31.76:9006 — a value the broker
parses as an int. That contaminated worker and test environments even
though the deployment already addresses every service by DNS name. Set
enableServiceLinks: false on the hermes-agent pod spec.

Blocked-task scheduling. create_task(initial_status="blocked") records a
created event carrying status=blocked but never a blocked event, while
_has_sticky_block() only inspects blocked/unblocked events. recompute_ready()
considers blocked tasks, so an explicitly parked task with no incomplete
parent auto-promoted on the next dispatcher cycle. Teach _has_sticky_block()
to also recognize a created event whose payload status is blocked, which
covers tasks created before this image patch without adding a persisted
field. Dependency-driven promotion and the circuit-breaker failure-limit
guard are untouched; unblock_task() still releases either kind of block.

hermes-kanban-blocked-regression.py runs against the real upstream
kanban_db API during the image build, so the build fails if any of these
semantics regress.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-16 20:53:22 +00:00
13 changed files with 586 additions and 51 deletions

View File

@ -15,6 +15,10 @@ RUN apt-get update \
# Paid/provider-backed search remains selectable through normal Hermes config.
RUN uv pip install --python /opt/hermes/.venv/bin/python ddgs==9.14.4
ARG HERMES_KANIKO_HEREDOC_COMPAT=0
COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
# Keep dashboard chat sockets tied to the intended React mount and conversation.
# A resumed conversation needs a different PTY attachment key from a fresh chat;
# reconnects to that same conversation must keep using the same key.
@ -252,6 +256,7 @@ for (const [before, after, label] of [
}
fs.writeFileSync(sidebarPath, sidebar);
NODE
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 1 ;; *) exit 2 ;; esac
# The upstream OIDC gate authenticates users but deliberately treats the
# dashboard as one shared workstation. Allow a deployment to narrow that
@ -324,6 +329,7 @@ for before, after, label in (
source = source.replace(before, after, 1)
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 2 ;; *) exit 2 ;; esac
# Give trusted plugins a pre-turn routing hook. It runs after fallback runtime
# restoration but before Hermes builds its provider-specific system prompt.
@ -493,6 +499,7 @@ if oneshot.count(oneshot_before) != 1:
)
oneshot_path.write_text(oneshot.replace(oneshot_before, oneshot_after, 1))
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 3 ;; *) exit 2 ;; esac
# Hermes WebUI sends its model/provider/reasoning selection on /v1/runs.
# Upstream currently applies only statically declared model_routes there, so
@ -628,17 +635,127 @@ runs_source = runs_source.replace(runs_agent_before, runs_agent_after, 1)
source = source[:runs_start] + runs_source
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 4 ;; *) exit 2 ;; esac
# ``create_task(initial_status="blocked")`` is an explicit operator park, but
# upstream only treats later block_task() events as sticky. Recognize the
# existing created(status=blocked) event too. This also protects tasks created
# before this image patch without making dependency or circuit-breaker blocks
# depend on a new persisted field.
RUN python - <<'PY'
from pathlib import Path
db_path = Path("/opt/hermes/hermes_cli/kanban_db.py")
db_source = db_path.read_text()
# The rewritten sticky-block gate below classifies a task from the ``created``
# event payload that ``create_task`` writes. That producer is upstream code we
# do not own, so anchor the exact contract the consumer depends on instead of
# trusting it: an ``initial_status="blocked"`` park must resolve ``task_status``
# to ``"blocked"``, every other creation must resolve it to something else, and
# the ``created`` event must carry that same variable under the ``"status"``
# key. If upstream renames the key, drops it, hardcodes it, or stops deriving it
# from ``initial_status``, fail the build here rather than ship a consumer that
# silently mis-classifies every task it reads. These assert only -- they
# deliberately do not rewrite the producer. The regression suite copied in below
# is the second net: it proves the same contract against the real create_task,
# for the drift these textual anchors cannot see.
producer_park_anchor = ''' if initial_status == "blocked":
task_status = "blocked"
'''
if db_source.count(producer_park_anchor) != 1:
raise SystemExit(
"Hermes Kanban create_task park semantics changed: expected 1, "
f"found {db_source.count(producer_park_anchor)}"
)
# The mirror of the park branch: every non-park creation must resolve to a
# status the gate does *not* read as a park. Without this, drift that widened
# the ladder to park ordinary work would leave both other anchors intact.
producer_unparked_anchor = ''' elif triage:
task_status = "triage"
else:
task_status = "ready"
'''
if db_source.count(producer_unparked_anchor) != 1:
raise SystemExit(
"Hermes Kanban create_task non-park status ladder changed: expected 1, "
f"found {db_source.count(producer_unparked_anchor)}"
)
producer_event_anchor = ''' _append_event(
conn,
task_id,
"created",
{
"assignee": assignee,
"status": task_status,
'''
if db_source.count(producer_event_anchor) != 1:
raise SystemExit(
"Hermes Kanban created-event status payload changed: expected 1, "
f"found {db_source.count(producer_event_anchor)}"
)
sticky_doc_before = ''' The cheapest signal that distinguishes the two is the most recent
``"blocked"`` / ``"unblocked"`` event for the task. If the most
recent one is ``"blocked"`` (or there is a ``"blocked"`` event and
no ``"unblocked"`` event has fired since), the task is sticky and
``recompute_ready`` must *not* auto-promote it.
Returns ``False`` when there is no such event at all (e.g. the task
'''
sticky_doc_after = ''' The cheapest signal that distinguishes the two is the most recent
``"created"`` / ``"blocked"`` / ``"unblocked"`` event for the task.
A ``"created"`` event whose payload has ``status="blocked"`` is the
explicit park requested by ``create_task(initial_status="blocked")``.
A later ``"unblocked"`` event clears either kind of sticky block.
Returns ``False`` when there is no such event at all (e.g. the task
'''
if db_source.count(sticky_doc_before) != 1:
raise SystemExit(
"Hermes Kanban sticky-block documentation changed: expected 1, "
f"found {db_source.count(sticky_doc_before)}"
)
db_source = db_source.replace(sticky_doc_before, sticky_doc_after, 1)
sticky_before = ''' row = conn.execute(
"SELECT kind FROM task_events "
"WHERE task_id = ? AND kind IN ('blocked', 'unblocked') "
"ORDER BY id DESC LIMIT 1",
(task_id,),
).fetchone()
return bool(row) and row["kind"] == "blocked"
'''
sticky_after = ''' row = conn.execute(
"SELECT kind, payload FROM task_events "
"WHERE task_id = ? AND kind IN ('created', 'blocked', 'unblocked') "
"ORDER BY id DESC LIMIT 1",
(task_id,),
).fetchone()
if not row or row["kind"] == "unblocked":
return False
if row["kind"] == "blocked":
return True
try:
payload = json.loads(row["payload"] or "{}")
except (TypeError, ValueError):
return False
if not isinstance(payload, dict):
return False
return payload.get("status") == "blocked"
'''
if db_source.count(sticky_before) != 1:
raise SystemExit(
"Hermes Kanban sticky-block gate changed: expected 1, "
f"found {db_source.count(sticky_before)}"
)
db_source = db_source.replace(sticky_before, sticky_after, 1)
# A manual evidence-based close must not expose a parked task to the dispatcher
# between separate `unblock` and `complete` commands. Upstream already permits
# direct completion from `blocked`; extend the same atomic path to `scheduled`
# and make that contract visible in CLI help so agents do not create a ready
# window that can launch redundant work.
RUN python - <<'PY'
from pathlib import Path
db_path = Path("/opt/hermes/hermes_cli/kanban_db.py")
db_source = db_path.read_text()
state_before = ''' WHERE id = ?
AND status IN ('running', 'ready', 'blocked')
'''
@ -670,6 +787,11 @@ if cli_source.count(help_before) != 1:
)
cli_path.write_text(cli_source.replace(help_before, help_after, 1))
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 5 ;; *) exit 2 ;; esac
COPY dockerfiles/hermes-kanban-blocked-regression.py /tmp/hermes-kanban-blocked-regression.py
RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py \
&& rm /tmp/hermes-kanban-blocked-regression.py
COPY dockerfiles/hermes-python-sandbox-tool.py /opt/hermes/tools/python_sandbox_tool.py
COPY dockerfiles/hermes-public-extract/__init__.py /opt/hermes/plugins/web/public_extract/__init__.py
@ -703,6 +825,7 @@ if source.count(before) != 1:
)
path.write_text(source.replace(before, after, 1))
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 6 ;; *) exit 2 ;; esac
# Keep the dashboard's account cards aligned with the managed provider lanes.
# Switchyard receives Claude Code subscription OAuth through the environment;
@ -769,6 +892,7 @@ for before, after, label in replacements:
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 7 ;; *) exit 2 ;; esac
# Make parent-linked API workers first-class live dashboard sessions. They stay
# active until the API runner closes them, even during a long final model call;
@ -862,6 +986,7 @@ source = source.replace(row_before, row_after)
path.write_text(source)
PY
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 8 ;; *) exit 2 ;; esac
# Keep the browser terminal reliable across GPU context loss, make durable
# worker lineage visible, expose an accessible API-worker transcript, and
@ -1430,18 +1555,11 @@ function RootRedirect() {
fs.writeFileSync(path, source);
}
NODE
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0) ;; 1) python /tmp/hermes-kaniko-heredoc-runner.py --dockerfile /tmp/hermes-agent.Dockerfile --block-index 9 ;; *) exit 2 ;; esac
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
ARG HERMES_KANIKO_HEREDOC_COMPAT=0
COPY dockerfiles/Dockerfile.hermes-agent /tmp/hermes-agent.Dockerfile
COPY dockerfiles/hermes-kaniko-heredoc-runner.py /tmp/hermes-kaniko-heredoc-runner.py
RUN case "${HERMES_KANIKO_HEREDOC_COMPAT}" in 0|1) ;; *) exit 2 ;; esac \
&& if [ "${HERMES_KANIKO_HEREDOC_COMPAT}" = 1 ]; then \
python /tmp/hermes-kaniko-heredoc-runner.py \
--dockerfile /tmp/hermes-agent.Dockerfile; \
fi \
&& rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
RUN rm -f /tmp/hermes-agent.Dockerfile /tmp/hermes-kaniko-heredoc-runner.py
RUN cd /opt/hermes/web \
&& npm run build \

View File

@ -1,4 +1,5 @@
**
!dockerfiles/hermes-kanban-blocked-regression.py
!dockerfiles/hermes-python-sandbox-tool.py
!dockerfiles/hermes-public-extract/
!dockerfiles/hermes-public-extract/**

View File

@ -0,0 +1,249 @@
"""Regression tests for Hermes Kanban blocked-task scheduling semantics."""
from __future__ import annotations
import json
import os
import tempfile
import unittest
from pathlib import Path
_HERMES_HOME = tempfile.TemporaryDirectory(prefix="hermes-kanban-test-home-")
os.environ["HERMES_HOME"] = _HERMES_HOME.name
from hermes_cli import kanban_db # noqa: E402
class BlockedTaskSchedulingTests(unittest.TestCase):
"""Exercise the real upstream database API against isolated databases."""
def setUp(self) -> None:
self._database_dir = tempfile.TemporaryDirectory(
prefix="hermes-kanban-test-db-"
)
self.connection = kanban_db.connect(
Path(self._database_dir.name) / "kanban.db"
)
def tearDown(self) -> None:
self.connection.close()
self._database_dir.cleanup()
def create_task(self, title: str, **kwargs: object) -> str:
return kanban_db.create_task(self.connection, title=title, **kwargs)
def status(self, task_id: str) -> str:
task = kanban_db.get_task(self.connection, task_id)
self.assertIsNotNone(task)
return task.status
def created_payload(self, task_id: str) -> dict:
"""Return the decoded payload of the task's ``created`` event."""
row = self.connection.execute(
"SELECT payload FROM task_events "
"WHERE task_id = ? AND kind = 'created' ORDER BY id LIMIT 1",
(task_id,),
).fetchone()
self.assertIsNotNone(row, "create_task emitted no 'created' event")
return json.loads(row["payload"] or "{}")
def set_created_payload(self, task_id: str, payload: object) -> None:
"""Overwrite a ``created`` event payload to simulate producer drift."""
with self.connection:
self.connection.execute(
"UPDATE task_events SET payload = ? "
"WHERE task_id = ? AND kind = 'created'",
(payload, task_id),
)
def block_via_circuit_breaker(self, title: str) -> str:
"""Create + claim a normal task, then trip the breaker once on it.
The resulting task is ``blocked`` with exactly one recorded failure and
-- critically -- no ``blocked`` event, because the breaker emits
``gave_up``. Its most recent created/blocked/unblocked event is
therefore its own ``created`` event, which is what makes it the right
probe for the created-event branch of the sticky gate.
"""
task_id = self.create_task(title)
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
self.assertTrue(
kanban_db._record_spawn_failure(
self.connection,
task_id,
"worker failed",
failure_limit=1,
)
)
self.assertEqual(self.status(task_id), "blocked")
return task_id
def test_initial_block_without_parents_is_sticky(self) -> None:
task_id = self.create_task("operator parked", initial_status="blocked")
self.assertEqual(kanban_db.recompute_ready(self.connection), 0)
self.assertEqual(self.status(task_id), "blocked")
def test_initial_block_with_complete_parents_is_sticky(self) -> None:
parent_id = self.create_task("completed prerequisite")
self.assertTrue(kanban_db.complete_task(self.connection, parent_id))
task_id = self.create_task(
"operator parked after prerequisite",
initial_status="blocked",
parents=[parent_id],
)
self.assertEqual(kanban_db.recompute_ready(self.connection), 0)
self.assertEqual(self.status(task_id), "blocked")
def test_initial_block_remains_sticky_when_parent_later_completes(self) -> None:
parent_id = self.create_task("incomplete prerequisite")
task_id = self.create_task(
"operator parked before prerequisite completes",
initial_status="blocked",
parents=[parent_id],
)
self.assertEqual(self.status(task_id), "blocked")
self.assertTrue(kanban_db.complete_task(self.connection, parent_id))
self.assertEqual(self.status(task_id), "blocked")
def test_initial_block_is_released_by_an_explicit_unblock(self) -> None:
task_id = self.create_task("operator parked", initial_status="blocked")
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertEqual(self.status(task_id), "ready")
def test_dependency_block_promotes_after_parent_completes(self) -> None:
parent_id = self.create_task("incomplete prerequisite")
task_id = self.create_task("dependency gated", parents=[parent_id])
self.assertEqual(self.status(task_id), "todo")
self.assertTrue(kanban_db.complete_task(self.connection, parent_id))
self.assertEqual(self.status(task_id), "ready")
def test_explicit_block_and_unblock_remain_sticky_and_reversible(self) -> None:
task_id = self.create_task("worker handoff")
self.assertTrue(
kanban_db.block_task(
self.connection,
task_id,
reason="human review required",
)
)
self.assertEqual(kanban_db.recompute_ready(self.connection), 0)
self.assertEqual(self.status(task_id), "blocked")
self.assertTrue(kanban_db.unblock_task(self.connection, task_id))
self.assertEqual(self.status(task_id), "ready")
def test_circuit_breaker_block_is_not_auto_promoted(self) -> None:
task_id = self.create_task("repeated worker failure")
self.assertIsNotNone(kanban_db.claim_task(self.connection, task_id))
# Upstream has no public circuit-breaker entry point; exercise the
# dispatcher helper that owns its persisted failure-limit semantics.
self.assertTrue(
kanban_db._record_spawn_failure(
self.connection,
task_id,
"worker failed",
failure_limit=1,
)
)
self.assertEqual(self.status(task_id), "blocked")
self.assertEqual(
kanban_db.recompute_ready(self.connection, failure_limit=1),
0,
)
self.assertEqual(self.status(task_id), "blocked")
def test_circuit_breaker_block_is_promoted_below_the_failure_limit(self) -> None:
"""Non-sticky oracle for the created-event branch of the sticky gate.
A circuit-breaker block must still auto-recover once the effective
failure limit rises above the recorded failure count -- that is the
pre-existing upstream contract the sticky-block patch must not widen.
This task's most recent created/blocked/unblocked event is its own
``created`` event carrying ``status="ready"``, so it is exactly the
case that a mis-classifying gate strands. It fails if the gate treats
any ``created`` event as a park (over-broad consumer) and it fails if
``create_task`` drifts to labelling every ``created`` event
``"blocked"`` (over-broad producer), while the
circuit-breaker-at-current-limit case above still holds.
"""
task_id = self.block_via_circuit_breaker("recoverable worker failure")
self.assertFalse(
kanban_db._has_sticky_block(self.connection, task_id),
"a circuit-breaker block must never be classified as a park",
)
self.assertEqual(
kanban_db.recompute_ready(self.connection, failure_limit=2),
1,
)
self.assertEqual(self.status(task_id), "ready")
def test_created_event_carries_the_status_the_sticky_gate_reads(self) -> None:
"""Producer contract asserted directly against the real create_task.
The sticky gate reads ``created`` payload ``status``. Pin both poles of
that contract so producer drift is a test failure here as well as a
build failure at the Dockerfile source anchor.
"""
parked_id = self.create_task("operator parked", initial_status="blocked")
normal_id = self.create_task("ordinary work")
self.assertEqual(self.created_payload(parked_id).get("status"), "blocked")
self.assertEqual(self.created_payload(normal_id).get("status"), "ready")
def test_non_blocked_created_payloads_are_not_treated_as_a_park(self) -> None:
"""Only ``status="blocked"`` parks; every other value must promote."""
for payload_status in ("ready", "todo", "running", "triage"):
with self.subTest(payload_status=payload_status):
task_id = self.block_via_circuit_breaker(
f"breaker trip with {payload_status} created payload"
)
self.set_created_payload(
task_id, json.dumps({"status": payload_status})
)
self.assertFalse(
kanban_db._has_sticky_block(self.connection, task_id)
)
self.assertEqual(
kanban_db.recompute_ready(self.connection, failure_limit=2),
1,
)
self.assertEqual(self.status(task_id), "ready")
def test_malformed_created_payloads_fail_closed_to_not_parked(self) -> None:
"""Undecodable or non-object payloads must not park -- or crash.
``recompute_ready`` walks every blocked task in one pass, so an
exception escaping the gate would stall promotion board-wide. Falling
back to "not parked" preserves upstream's pre-patch auto-recover
default for rows the gate cannot interpret.
"""
malformed = (None, "", " ", "{not json", "[]", "null", '"blocked"', "17")
for payload in malformed:
with self.subTest(payload=payload):
task_id = self.block_via_circuit_breaker(
f"breaker trip with payload {payload!r}"
)
self.set_created_payload(task_id, payload)
self.assertFalse(
kanban_db._has_sticky_block(self.connection, task_id)
)
self.assertEqual(
kanban_db.recompute_ready(self.connection, failure_limit=2),
1,
)
self.assertEqual(self.status(task_id), "ready")
if __name__ == "__main__":
unittest.main(verbosity=2)

View File

@ -10,6 +10,7 @@ normally, so this compatibility runner is enabled only by the Kaniko pipeline.
from __future__ import annotations
import argparse
import re
import subprocess
from pathlib import Path
@ -20,32 +21,75 @@ BLOCKS = (
("RUN python - <<'PY'", "PY", ("python", "-")),
)
EXPECTED_COMMANDS = ("node", *("python",) * 7, "node")
PARSER_DIRECTIVE = re.compile(
r"^\s*#\s*([a-z]+)\s*=\s*(.*?)\s*$", re.IGNORECASE
)
def _continued_run_header(lines: list[str], index: int) -> tuple[str, int]:
"""Return one logical RUN header, including backslash continuations."""
header = lines[index]
while header.rstrip().endswith("\\") and index + 1 < len(lines):
def _escape_character(lines: list[str]) -> str:
"""Return the one Dockerfile parser escape directive, or its default."""
escape = "\\"
directives: set[str] = set()
for line in lines:
if not line.strip():
break
match = PARSER_DIRECTIVE.fullmatch(line)
if not match:
break
name, value = match.group(1).lower(), match.group(2)
if name in directives:
raise ValueError(f"multiple Dockerfile {name} directives")
directives.add(name)
if name == "escape":
if value not in {"\\", "`"}:
raise ValueError("unsupported Dockerfile escape directive")
escape = value
return escape
def _continued(line: str, escape: str) -> bool:
"""Match Docker's unescaped continuation marker at physical line end."""
stripped = line.rstrip()
count = len(stripped) - len(stripped.rstrip(escape))
return count % 2 == 1
def _logical_instruction(
lines: list[str], index: int, escape: str
) -> tuple[str, int]:
"""Normalize escape-newline pairs, including split opcodes/operators."""
logical = lines[index]
while _continued(logical.split("\n")[-1], escape):
logical = logical.rstrip()
logical = logical[:-1]
index += 1
header = f"{header}\n{lines[index]}"
return header, index
while index < len(lines) and lines[index].lstrip().startswith("#"):
index += 1
if index >= len(lines) or not lines[index].strip():
raise ValueError("unterminated Dockerfile line continuation")
logical += lines[index].lstrip()
return logical, index
def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
"""Inventory every RUN heredoc, then return only the reviewed contract."""
markers = {start: (end, command) for start, end, command in BLOCKS}
lines = source.splitlines()
escape = _escape_character(lines)
blocks: list[tuple[tuple[str, ...], str]] = []
index = 0
while index < len(lines):
marker = markers.get(lines[index])
if marker is None:
stripped = lines[index].lstrip()
if not lines[index].strip() or lines[index].lstrip().startswith("#"):
index += 1
continue
logical, index = _logical_instruction(lines, index, escape)
stripped = logical.lstrip()
if stripped[:3].upper() == "RUN" and (
len(stripped) == 3 or stripped[3].isspace()
):
header, index = _continued_run_header(lines, index)
if "<<" in header:
if "<<" in stripped:
raise ValueError(
"unsupported RUN heredoc outside the exact reviewed contract"
)
@ -73,23 +117,27 @@ def extract_blocks(source: str) -> list[tuple[tuple[str, ...], str]]:
return blocks
def replay(dockerfile: Path) -> None:
"""Execute each exact heredoc in an isolated interpreter process."""
def replay(dockerfile: Path, block_index: int) -> None:
"""Execute one exact heredoc after inventorying the complete Dockerfile."""
size = dockerfile.stat().st_size
if size < 1 or size > MAX_DOCKERFILE_BYTES:
raise ValueError("Hermes Dockerfile size is outside the reviewed boundary")
source = dockerfile.read_text(encoding="utf-8")
for index, (command, body) in enumerate(extract_blocks(source), start=1):
print(f"replaying reviewed Dockerfile heredoc {index}/{len(EXPECTED_COMMANDS)}")
subprocess.run(command, input=body, text=True, check=True)
blocks = extract_blocks(source)
if block_index < 1 or block_index > len(blocks):
raise ValueError("heredoc block index is outside the reviewed contract")
command, body = blocks[block_index - 1]
print(f"replaying reviewed Dockerfile heredoc {block_index}/{len(blocks)}")
subprocess.run(command, input=body, text=True, check=True)
def main() -> int:
"""Parse the one explicit Dockerfile path and replay its reviewed patches."""
parser = argparse.ArgumentParser()
parser.add_argument("--dockerfile", required=True, type=Path)
parser.add_argument("--block-index", required=True, type=int)
args = parser.parse_args()
replay(args.dockerfile)
replay(args.dockerfile, args.block_index)
return 0

View File

@ -124,16 +124,19 @@ root-in-the-build-pod risk, bounded to disposable `emptyDir` storage and a
human-reviewed `main` revision; Dockerfile changes require the same scrutiny as
executable cluster code.
Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize their
inline files when executing a command. The Jenkins lane therefore enables a
bounded compatibility replay after copying the reviewed Dockerfile and runner
from the same checked-out commit. The runner inventories all `RUN` heredoc
headers before launching an interpreter, rejects every form outside the exact
nine Node/Python blocks, and launches each accepted body in a separate process.
Docker and BuildKit keep their native behavior because the compatibility
argument defaults off. The final TypeScript build, Python compile, and source
assertions remain mandatory, so an omitted or drifted replay fails before any
image can pass release verification.
Pinned Kaniko v1.23.2 parses Dockerfile `RUN` heredocs but does not materialize
their inline files when executing a command. The Jenkins lane therefore enables
a bounded compatibility replay after copying the reviewed Dockerfile and runner
from the same checked-out commit. Before every replay, the runner inventories
the entire Dockerfile as logical instructions, including split opcodes,
operators, and either Dockerfile escape character. It rejects every heredoc
form outside the exact nine Node/Python blocks before launching an interpreter.
Each accepted body is then replayed immediately after its corresponding Kaniko
`RUN`, preserving source order before any dependent regression. Docker and
BuildKit keep their native behavior because the compatibility argument defaults
off. The final TypeScript build, Python compile, and source assertions remain
mandatory, so an omitted or drifted replay fails before any image can pass
release verification.
From agent.hermes, trigger that one fixed job with:

View File

@ -96,6 +96,7 @@ spec:
vault.hashicorp.com/agent-limits-mem: 128Mi
spec:
serviceAccountName: hermes-agent
enableServiceLinks: false
automountServiceAccountToken: true
securityContext:
fsGroup: 10000
@ -115,7 +116,7 @@ spec:
values: ["true"]
- key: kubernetes.io/hostname
operator: NotIn
values: [titan-08, titan-13, titan-14, titan-17, titan-18]
values: [titan-04, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
@ -694,7 +695,7 @@ spec:
seccompProfile:
type: RuntimeDefault
resources:
requests: {cpu: 350m, memory: 768Mi}
requests: {cpu: 300m, memory: 768Mi}
limits: {cpu: "3", memory: 6Gi}
- name: oauth2-proxy
image: quay.io/oauth2-proxy/oauth2-proxy:v7.15.3@sha256:10a1165743a192e1940b4708fb9647027185ce11a681a1c5519b442ff7f1f561
@ -878,7 +879,7 @@ spec:
- {name: CLAUDE_CONFIG_DIR, value: /runtime-access/claude}
- {name: KUBECONFIG, value: /opt/data/home/.kube/config}
- {name: PYTHONPATH, value: /opt/hermes}
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "4"}
- {name: HERMES_CLI_LANE_CONCURRENCY, value: "2"}
- {name: HERMES_AUTO_ROUTER_PROFILE, value: agent}
- {name: PATH, value: /opt/coordinator:/opt/data/tools/bin:/opt/data/home/.local/bin:/opt/hermes/.venv/bin:/usr/local/bin:/usr/bin:/bin}
securityContext:
@ -901,7 +902,7 @@ spec:
- {name: tmp, mountPath: /tmp}
resources:
requests: {cpu: 100m, memory: 256Mi}
limits: {cpu: "3", memory: 6Gi}
limits: {cpu: "2", memory: 6Gi}
- name: model-steward
image: registry.bstein.dev/bstein/hermes-agent@sha256:81970563e542f0720773e72297810b3a844b83e381e278f25c0916c78d930107
imagePullPolicy: IfNotPresent

View File

@ -15,7 +15,7 @@ data:
METIS_MAX_DEVICE_BYTES: "1000000000000"
METIS_NAMESPACE: maintenance
METIS_REMOTE_POD_TIMEOUT_SEC: "14400"
METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-295-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
METIS_RUNNER_IMAGE_AMD64: registry.bstein.dev/bstein/metis:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-amd64"}
METIS_RUNNER_IMAGE_ARM64: registry.bstein.dev/bstein/metis:0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64"}
METIS_HARBOR_REGISTRY: registry.bstein.dev
METIS_HARBOR_PROJECT: metis

View File

@ -11,7 +11,7 @@ resources:
- networking
images:
- name: registry.bstein.dev/bstein/ariadne
newTag: 0.1.0-460 # {"$imagepolicy": "maintenance:ariadne:tag"}
newTag: 0.1.0-461 # {"$imagepolicy": "maintenance:ariadne:tag"}
- name: registry.bstein.dev/bstein/metis
newTag: 0.1.0-295-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"}
- name: registry.bstein.dev/bstein/soteria

View File

@ -32,7 +32,7 @@ spec:
kubernetes.io/arch: amd64
containers:
- name: metis-sentinel
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-295-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
image: registry.bstein.dev/bstein/metis-sentinel:0.1.0-296-amd64 # {"$imagepolicy": "maintenance:metis-sentinel-amd64"}
imagePullPolicy: Always
envFrom:
- configMapRef:

View File

@ -120,6 +120,9 @@ def test_agent_image_completes_parked_kanban_tasks_atomically():
"Atomically mark running, ready, blocked, or scheduled tasks done"
in dockerfile
)
assert "kind IN ('created', 'blocked', 'unblocked')" in dockerfile
assert "payload.get(\"status\") == \"blocked\"" in dockerfile
assert "hermes-kanban-blocked-regression.py" in dockerfile
def test_sandbox_shares_only_the_tenant_workspace_without_credentials():

View File

@ -1074,6 +1074,7 @@ def test_agent_uses_one_native_kanban_control_plane():
deployment = _agent_deployment()
pod = deployment["spec"]["template"]["spec"]
assert pod["enableServiceLinks"] is False
names = {item["name"] for item in pod["containers"]}
assert "cli-lane-runner" in names
assert "terminal" in names
@ -1083,6 +1084,49 @@ def test_agent_uses_one_native_kanban_control_plane():
assert "herdr-dispatch" not in rendered
def test_cli_lane_reserves_cpu_headroom_for_ui_and_auth():
deployment = _agent_deployment()
containers = {
item["name"]: item
for item in deployment["spec"]["template"]["spec"]["containers"]
}
lane = containers["cli-lane-runner"]
environment = {item["name"]: item["value"] for item in lane["env"]}
assert environment["HERMES_CLI_LANE_CONCURRENCY"] == "2"
assert lane["resources"] == {
"requests": {"cpu": "100m", "memory": "256Mi"},
"limits": {"cpu": "2", "memory": "6Gi"},
}
def test_agent_avoids_unhealthy_nodes_and_fits_its_remaining_capacity():
"""Placement correction: keep the agent off nodes that cannot hold it.
titan-04 is cordoned after repeated kernel undervoltage and kubelet
failure, and titan-19 was probe/Longhorn unstable under worker load, so
both must join the existing hard exclusions. That leaves titan-05 as the
healthy candidate, which is tight enough on requested CPU that the main
container has to give back 50m to schedule there.
"""
pod = _agent_deployment()["spec"]["template"]["spec"]
hostnames = next(
item
for item in pod["affinity"]["nodeAffinity"][
"requiredDuringSchedulingIgnoredDuringExecution"
]["nodeSelectorTerms"][0]["matchExpressions"]
if item["key"] == "kubernetes.io/hostname"
)
assert hostnames["operator"] == "NotIn"
assert set(hostnames["values"]) >= {"titan-04", "titan-19"}
hermes = next(
item for item in pod["containers"] if item["name"] == "hermes"
)
assert hermes["resources"]["requests"]["cpu"] == "300m"
def test_agent_root_is_stock_dashboard_and_terminal_is_a_separate_path():
deployment = _agent_deployment()
pod = deployment["spec"]["template"]["spec"]

View File

@ -5,6 +5,7 @@ from __future__ import annotations
import importlib.util
import io
import json
import re
import urllib.parse
from pathlib import Path
@ -150,8 +151,10 @@ def test_kaniko_replays_only_the_exact_reviewed_heredoc_contract() -> None:
)
assert "session message total" in blocks[-1][1]
compat = dockerfile.split("ARG HERMES_KANIKO_HEREDOC_COMPAT=0", 1)[1]
assert "case \"${HERMES_KANIKO_HEREDOC_COMPAT}\" in 0|1)" in compat
assert "python /tmp/hermes-kaniko-heredoc-runner.py" in compat
assert compat.count("python /tmp/hermes-kaniko-heredoc-runner.py") == 9
assert [int(value) for value in re.findall(r"--block-index (\d+)", compat)] == list(
range(1, 10)
)
ignored = (
REPO_ROOT / "dockerfiles/Dockerfile.hermes-agent.dockerignore"
).read_text()
@ -176,7 +179,8 @@ def test_kaniko_heredoc_runner_rejects_drift_and_executes_separately(
monkeypatch.setattr(module.subprocess, "run", run)
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(source, encoding="utf-8")
module.replay(dockerfile)
for block_index in range(1, 10):
module.replay(dockerfile, block_index)
assert len(calls) == 9
assert all(call[1]["check"] is True for call in calls)
assert all(call[1]["text"] is True for call in calls)

View File

@ -57,7 +57,7 @@ def test_all_unsupported_run_heredoc_forms_reject_before_execution(
calls = []
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
with pytest.raises(ValueError, match="unsupported RUN heredoc"):
module.replay(dockerfile)
module.replay(dockerfile, 1)
assert calls == []
@ -86,10 +86,74 @@ def test_appended_tenth_reviewed_form_rejects_before_execution(
calls = []
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
with pytest.raises(ValueError, match="contract changed"):
module.replay(dockerfile)
module.replay(dockerfile, 1)
assert calls == []
@pytest.mark.parametrize(
("prefix", "unsupported"),
[
("", "R\\\nUN cat <<EOF\nbody\nEOF"),
("", "RUN cat <\\\n<EOF\nbody\nEOF"),
("", "R\\\nUN cat <\\\n<EOF\nbody\nEOF"),
("# escape=`\n", "R`\nUN cat <<EOF\nbody\nEOF"),
("# escape=`\n", "RUN cat <`\n<EOF\nbody\nEOF"),
("# escape=`\n", "R`\nUN cat <`\n<EOF\nbody\nEOF"),
("", "R\\\n# removed comment\nUN cat <<EOF\nbody\nEOF"),
(
"# ordinary comment\n# escape=`\n",
"R\\\nUN cat <<EOF\nbody\nEOF",
),
],
)
def test_logical_instruction_normalization_rejects_split_heredocs_before_execution(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
prefix: str,
unsupported: str,
) -> None:
"""Split RUN/opcode operators and backtick escapes cannot evade inventory."""
module = _load(RUNNER, f"heredoc_logical_{abs(hash((prefix, unsupported)))}")
dockerfile = tmp_path / "Dockerfile"
dockerfile.write_text(
prefix + DOCKERFILE.read_text(encoding="utf-8") + "\n" + unsupported + "\n",
encoding="utf-8",
)
calls = []
monkeypatch.setattr(module.subprocess, "run", lambda *_args, **_kwargs: calls.append(1))
with pytest.raises(ValueError, match="unsupported RUN heredoc"):
module.replay(dockerfile, 1)
assert calls == []
def test_merged_main_replays_each_block_before_dependent_work() -> None:
"""PR13's blocked-task regression runs only after its source patch replay."""
source = DOCKERFILE.read_text(encoding="utf-8")
replay_positions = [
source.index(f"--block-index {index}") for index in range(1, 10)
]
# Locate repeated markers rather than trusting one synthetic occurrence.
block_positions = []
for marker in ("RUN node <<'NODE'", "RUN python - <<'PY'"):
start = 0
while True:
position = source.find(marker, start)
if position < 0:
break
block_positions.append(position)
start = position + len(marker)
block_positions.sort()
assert len(block_positions) == len(replay_positions) == 9
for index, (block, replay) in enumerate(zip(block_positions, replay_positions)):
assert block < replay
if index + 1 < len(block_positions):
assert replay < block_positions[index + 1]
regression = source.index(
"RUN /opt/hermes/.venv/bin/python /tmp/hermes-kanban-blocked-regression.py"
)
assert replay_positions[4] < regression < replay_positions[5]
def test_builder_service_account_is_explicit_tokenless_and_unbound() -> None:
"""The build Pod selects one tokenless SA that no tracked RBAC grants bind."""
account = yaml.safe_load(BUILDER_SA.read_text(encoding="utf-8"))