titan/atlas-iac
3
0
Fork 0
Code Issues Pull Requests 9 Actions Packages Projects Releases Wiki Activity
atlas-iac/testing/quality_contract.json
Hermes Agent 5d963c9354 hermes: make pool lease recovery and release isolation safe 
Independent review t_5975c06a blocked this branch on a P1: a Kanban write that
failed while a lease expired left a `lease_failed` row that was invisible to
every pass, immortal to garbage collection, and fatal to the coordinator. It
poisoned `reconcile()` forever with a conflicting-duplicate primary key,
produced a spurious capability `block_task` from `dispatch()`, and -- because
startup maintenance ran unguarded before the port bound, against a store on a
PVC -- crash-looped the coordinator with no automatic recovery.

`lease_failed` is now a retryable state that every maintenance pass drains, and
a row only reaches a terminal state on authoritative evidence about its exact
Kanban run, so nothing is collected before its outcome is known and nothing is
silently dropped. Each row, task, and board is processed in isolation, and a
coordinator-side fault is never converted into a Kanban mutation. Startup runs
through the same guarded cycle as the steady-state loop.

The wire protocol and the durable store are now separate modules, and the
maintenance passes moved out of the coordinator, so each file stays under the
managed line ceiling with room for the recovery logic.

Also closes three consequential handoff risks the same review raised:

* mediator-N pinned itself hard to worker-N while sharing a ReadWriteOnce
  claim, so a drain or preemption that moved only the lower-priority worker
  deadlocked the ordinal on Multi-Attach until an operator deleted a Pod. The
  shared workspace is now ReadWriteMany (as the hermes-chat tenant workspaces
  already are on the same class), colocation is a preference, and the mediator
  shares the worker's preemption priority, so each Pod reschedules on its own.
* the broker permits only branch creation, so a retry that added commits could
  never submit and the run's work was discarded with the failure. Submission
  now targets a fresh attempt- or content-scoped ref in the same reviewed
  namespace -- never an update -- and is idempotent under replay. A refused
  submission downgrades the result and says why instead of unwinding the run.
* the provider CLIs were reinstalled into an emptyDir on every Pod start inside
  the 10m Flux health window for the whole hermes app. They now install once
  per pinned version onto a durable volume, re-verified against the real
  binaries and time-bounded, and the best-effort pool no longer gates the
  health of the app its dependents wait on.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-08-17 16:31:15 +00:00

348 lines
14 KiB
JSON
Raw Blame History

{
"required_docs": [
{
"path": "README.md",
"description": "Top-level repository handbook."
},
{
"path": "Jenkinsfile",
"description": "Top-level Jenkins mirror for multibranch discovery."
},
{
"path": "ci/Jenkinsfile.titan-iac",
"description": "Canonical titan-iac Jenkins pipeline definition."
}
],
"managed_modules": [
"ci/scripts/publish_test_metrics.py",
"ci/scripts/publish_test_metrics_quality.py",
"ci/scripts/semgrep_report.py",
"ci/scripts/supply_chain_report.py",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py",
"services/gitea/scripts/gitea_branch_protection_check.py",
"services/hermes/scm-common/scripts/gitea_api.py",
"services/hermes/scm-common/scripts/gitea_api_policy.py",
"services/hermes/scm-common/scripts/scm_broker.py",
"services/hermes/scm-common/scripts/scm_broker_client.py",
"services/hermes/scm-common/scripts/scm_broker_io.py",
"services/hermes/scm-common/scripts/scm_broker_server.py",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_maintenance.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_store.py",
"services/hermes/scripts/execution_pool_worker.py",
"services/hermes/scripts/hermes_coordinator.py",
"services/hermes/scripts/node_account_audit.py",
"services/hermes/scripts/node_account_hardening.py",
"services/hermes/scripts/node_account_io.py",
"services/hermes/scripts/stage_runtime_access.py",
"services/hermes/scripts/cli_lane_board.py",
"services/hermes/scripts/cli_lane_capabilities.py",
"services/hermes/scripts/cli_lane_config.py",
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",
"services/hermes/scripts/cli_lane_routing.py",
"services/hermes/scripts/cli_lane_runner.py",
"testing/__init__.py",
"testing/quality_contract.py",
"testing/quality_docs.py",
"testing/quality_hygiene.py",
"testing/quality_coverage.py",
"testing/quality_gate.py",
"ci/tests/glue/test_ariadne_schedules.py",
"ci/tests/glue/test_glue_metrics.py",
"testing/tests/test_publish_test_metrics.py",
"testing/tests/test_supply_chain_report.py",
"testing/tests/test_semgrep_report.py",
"testing/tests/test_quality_contract.py",
"testing/tests/test_quality_gate.py"
],
"lint_paths": [
"ci/scripts/publish_test_metrics.py",
"ci/scripts/publish_test_metrics_quality.py",
"ci/scripts/semgrep_report.py",
"ci/tests/glue",
"scripts/tests",
"services/comms/scripts/tests",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py",
"services/gitea/scripts/gitea_branch_protection_check.py",
"services/hermes/scm-common/scripts",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_maintenance.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_store.py",
"services/hermes/scripts/execution_pool_worker.py",
"services/hermes/scripts/hermes_coordinator.py",
"services/hermes/scripts/node_account_audit.py",
"services/hermes/scripts/node_account_hardening.py",
"services/hermes/scripts/node_account_io.py",
"services/hermes/scripts/stage_runtime_access.py",
"services/hermes/scripts/cli_lane_board.py",
"services/hermes/scripts/cli_lane_capabilities.py",
"services/hermes/scripts/cli_lane_config.py",
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",
"services/hermes/scripts/cli_lane_routing.py",
"services/hermes/scripts/cli_lane_runner.py",
"testing/tests",
"testing"
],
"pytest_suites": {
"unit": {
"description": "Fast unit and contract tests for repo automation.",
"paths": [
"scripts/tests",
"services/comms/scripts/tests",
"testing/tests"
],
"junit": "build/junit-unit.xml",
"coverage_sources": [
"ci/scripts",
"scripts.render.dashboards_render_atlas",
"services/gitea/scripts",
"services/hermes/scm-common/scripts",
"services/hermes/scripts",
"services/mailu/scripts",
"testing"
],
"coverage_xml": "build/coverage-unit.xml"
},
"glue": {
"description": "Cluster-live glue checks that validate CronJobs and exported metrics.",
"paths": [
"ci/tests/glue"
],
"junit": "build/junit-glue.xml"
}
},
"profiles": {
"local": [
"docs",
"smell",
"hygiene",
"unit",
"coverage"
],
"jenkins": [
"docs",
"smell",
"hygiene",
"unit",
"coverage",
"sonarqube",
"ironbank",
"glue"
]
},
"manual_scripts": [
{
"path": "scripts/manual-tests/test_atlas_user_cleanup.py",
"description": "Manual cleanup validation for Atlas user lifecycle automation."
},
{
"path": "scripts/manual-tests/test_user_cleanup.py",
"description": "Manual cleanup validation for shared user lifecycle automation."
},
{
"path": "scripts/manual-tests/test_vaultwarden_user_cleanup.py",
"description": "Manual cleanup validation for Vaultwarden user lifecycle automation."
},
{
"path": "services/bstein-dev-home/scripts/test_portal_onboarding_flow.py",
"description": "Portal onboarding end-to-end flow validation with mail delivery checks."
},
{
"path": "services/keycloak/scripts/tests/test_keycloak_execute_actions_email.py",
"description": "Standalone Keycloak SMTP execute-actions-email validation script."
},
{
"path": "services/keycloak/scripts/tests/test_portal_token_exchange.py",
"description": "Standalone Keycloak token-exchange validation script."
}
],
"hygiene": {
"max_lines": 500,
"line_limit_globs": [
"testing/**/*.py",
"ci/scripts/*.py",
"ci/tests/**/*.py",
"scripts/tests/**/*.py",
"services/*/scripts/tests/**/*.py",
"services/gitea/scripts/gitea_branch_protection_check.py",
"services/hermes/scm-common/scripts/*.py",
"services/hermes/scripts/execution_pool_*.py",
"services/hermes/scripts/hermes_coordinator.py",
"services/hermes/scripts/node_account_audit.py",
"services/hermes/scripts/node_account_hardening.py",
"services/hermes/scripts/node_account_io.py",
"services/hermes/scripts/stage_runtime_access.py",
"services/hermes/scripts/cli_lane_*.py",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py"
],
"naming_rules": [
{
"glob": "testing/*.py",
"pattern": "^(?:__init__|quality_[a-z0-9_]+)\\.py$",
"description": "Top-level testing helpers use quality_* module names."
},
{
"glob": "testing/tests/*.py",
"pattern": "^test_[a-z0-9_]+\\.py$",
"description": "Top-level pytest files use test_*.py names."
},
{
"glob": "ci/tests/**/*.py",
"pattern": "^test_[a-z0-9_]+\\.py$",
"description": "CI pytest files use test_*.py names."
},
{
"glob": "scripts/tests/**/*.py",
"pattern": "^test_[a-z0-9_]+\\.py$",
"description": "Script pytest files use test_*.py names."
},
{
"glob": "scripts/test_*.py",
"pattern": "^test_[a-z0-9_]+\\.py$",
"description": "Standalone script tests use test_*.py names."
},
{
"glob": "services/*/scripts/tests/**/*.py",
"pattern": "^test_[a-z0-9_]+\\.py$",
"description": "Service pytest files use test_*.py names."
},
{
"glob": "services/*/scripts/test_*.py",
"pattern": "^test_[a-z0-9_]+\\.py$",
"description": "Standalone service test scripts use test_*.py names."
}
]
},
"coverage": {
"minimum_percent": 95.0,
"minimum_branch_percent": 95.0,
"branch_tracked_files": [
"services/gitea/scripts/gitea_branch_protection_check.py",
"services/hermes/scm-common/scripts/gitea_api.py",
"services/hermes/scm-common/scripts/gitea_api_policy.py",
"services/hermes/scm-common/scripts/scm_broker.py",
"services/hermes/scm-common/scripts/scm_broker_client.py",
"services/hermes/scm-common/scripts/scm_broker_io.py",
"services/hermes/scm-common/scripts/scm_broker_server.py",
"services/hermes/scripts/hermes_coordinator.py",
"services/hermes/scripts/node_account_audit.py",
"services/hermes/scripts/node_account_hardening.py",
"services/hermes/scripts/node_account_io.py",
"services/hermes/scripts/stage_runtime_access.py",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_maintenance.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_store.py",
"services/hermes/scripts/execution_pool_worker.py",
"services/hermes/scripts/cli_lane_board.py",
"services/hermes/scripts/cli_lane_capabilities.py",
"services/hermes/scripts/cli_lane_config.py",
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",
"services/hermes/scripts/cli_lane_routing.py",
"services/hermes/scripts/cli_lane_runner.py",
"testing/quality_coverage.py"
],
"tracked_files": [
"ci/scripts/publish_test_metrics.py",
"ci/scripts/publish_test_metrics_quality.py",
"ci/scripts/semgrep_report.py",
"ci/scripts/supply_chain_report.py",
"services/mailu/scripts/mailu_sync.py",
"services/mailu/scripts/mailu_sync_listener.py",
"services/gitea/scripts/gitea_branch_protection_check.py",
"services/hermes/scm-common/scripts/gitea_api.py",
"services/hermes/scm-common/scripts/gitea_api_policy.py",
"services/hermes/scm-common/scripts/scm_broker.py",
"services/hermes/scm-common/scripts/scm_broker_client.py",
"services/hermes/scm-common/scripts/scm_broker_io.py",
"services/hermes/scm-common/scripts/scm_broker_server.py",
"services/hermes/scripts/execution_pool_client.py",
"services/hermes/scripts/execution_pool_coordinator.py",
"services/hermes/scripts/execution_pool_maintenance.py",
"services/hermes/scripts/execution_pool_project.py",
"services/hermes/scripts/execution_pool_protocol.py",
"services/hermes/scripts/execution_pool_scm.py",
"services/hermes/scripts/execution_pool_server.py",
"services/hermes/scripts/execution_pool_store.py",
"services/hermes/scripts/execution_pool_worker.py",
"services/hermes/scripts/hermes_coordinator.py",
"services/hermes/scripts/node_account_audit.py",
"services/hermes/scripts/node_account_hardening.py",
"services/hermes/scripts/node_account_io.py",
"services/hermes/scripts/stage_runtime_access.py",
"services/hermes/scripts/cli_lane_board.py",
"services/hermes/scripts/cli_lane_capabilities.py",
"services/hermes/scripts/cli_lane_config.py",
"services/hermes/scripts/cli_lane_dispatch.py",
"services/hermes/scripts/cli_lane_evidence.py",
"services/hermes/scripts/cli_lane_execution.py",
"services/hermes/scripts/cli_lane_files.py",
"services/hermes/scripts/cli_lane_finalization.py",
"services/hermes/scripts/cli_lane_goal.py",
"services/hermes/scripts/cli_lane_prompt.py",
"services/hermes/scripts/cli_lane_provider.py",
"services/hermes/scripts/cli_lane_quarantine.py",
"services/hermes/scripts/cli_lane_records.py",
"services/hermes/scripts/cli_lane_recovery.py",
"services/hermes/scripts/cli_lane_retention.py",
"services/hermes/scripts/cli_lane_routing.py",
"services/hermes/scripts/cli_lane_runner.py",
"testing/quality_contract.py",
"testing/quality_docs.py",
"testing/quality_hygiene.py",
"testing/quality_coverage.py",
"testing/quality_gate.py"
]
}
}
Reference in New Issue View Git Blame Copy Permalink