Compare commits

..

1 Commits

Author SHA1 Message Date
jenkins
fe6495cfb9 Deploy Cassandra 0.8.72 2026-07-29 21:57:15 -03:00
364 changed files with 1989 additions and 21206 deletions

6
.gitignore vendored
View File

@ -5,7 +5,6 @@
__pycache__/ __pycache__/
*.py[cod] *.py[cod]
.pytest_cache .pytest_cache
.ruff_cache/
.coverage .coverage
build/ build/
test-results/ test-results/
@ -13,7 +12,6 @@ artifacts/
.venv .venv
.venv-ci .venv-ci
tmp/ tmp/
.mainfix/
.terraform/ .terraform/
**/.terraform/ **/.terraform/
*.tfvars *.tfvars
@ -21,7 +19,3 @@ tmp/
*.tfstate.* *.tfstate.*
crash.log crash.log
terraform/atlas/generated/ terraform/atlas/generated/
# Local demo credentials (never commit)
scripts/ops/hermes_demo.env
scripts/ops/hermes_triage_demo.env

55
Jenkinsfile vendored
View File

@ -56,11 +56,6 @@ spec:
command: command:
- cat - cat
tty: true tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command:
- cat
tty: true
""" """
} }
} }
@ -75,8 +70,6 @@ spec:
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428' VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0' QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1' QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0' QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
@ -117,26 +110,6 @@ spec:
''' '''
} }
} }
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') { stage('Collect SonarQube evidence') {
steps { steps {
container('quality-tools') { container('quality-tools') {
@ -153,7 +126,6 @@ spec:
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx" "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
) )
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml") [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]} rc=${PIPESTATUS[0]}
@ -354,33 +326,6 @@ PY
esac esac
fi fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}" ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1 ironbank_required=1

105
Makefile
View File

@ -1,105 +0,0 @@
# Makefile
SHELL := /usr/bin/env bash
KUSTOMIZE ?= kustomize
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
TERRAFORM ?= terraform
TRIVY ?= trivy
BUILD_DIR ?= build
ATLAS_FLUX_ROOT := clusters/atlas/flux-system
AETHER_FLUX_ROOT := clusters/aether/flux-system
AETHER_TF_DIR := terraform/aether
FLUX_KUSTOMIZATION ?= flux-system
FLUX_NAMESPACE ?= flux-system
FLUX_PATH ?= $(ATLAS_FLUX_ROOT)
TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache
TRIVY_DB_FLAGS ?= --skip-db-update
.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan
help:
@printf '%s\n' \
'Targets:' \
' render Render Atlas, Aether, services, and infrastructure.' \
' render-atlas Render the Atlas Flux root.' \
' render-aether Render the Aether Flux root.' \
' render-services Render every service kustomization.' \
' render-infrastructure Render infrastructure kustomizations.' \
' validate Run render checks plus the local quality gate.' \
' dashboards Regenerate Grafana dashboard artifacts.' \
' knowledge Regenerate Atlas knowledge artifacts and comms mirror.' \
' test Run the local quality gate.' \
' flux-diff Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \
' flux-inventory Print Flux Kustomization inventory.' \
' trivy-scan Run the local Trivy filesystem scan.' \
' security-report Build IronBank report from build/trivy-fs.json.' \
' security Run Trivy scan and build the IronBank report.' \
' aether-fmt Check Terraform formatting for Aether.' \
' aether-init Initialize Aether Terraform providers.' \
' aether-validate Validate Aether Terraform after init.' \
' aether-plan Plan Aether Terraform from local variables/env.'
render: render-atlas render-aether render-services render-infrastructure
render-atlas:
$(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml
render-aether:
$(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml
render-services:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find services -name kustomization.yaml | sort)
render-infrastructure:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find infrastructure -name kustomization.yaml | sort)
validate: render test
dashboards:
$(PYTHON) scripts/render/dashboards_render_atlas.py --build
$(PYTHON) scripts/render/dashboards_render_logs.py --build
$(PYTHON) scripts/render/logging_render_observability.py --build
knowledge:
$(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms
test:
$(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR)
flux-diff:
flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH)
flux-inventory:
$(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT)
trivy-scan:
mkdir -p $(BUILD_DIR)
$(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
security-report:
@test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; }
$(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json
security: trivy-scan security-report
aether-fmt:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check
aether-init:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) init
aether-validate:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate
aether-plan:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan

View File

@ -55,11 +55,6 @@ spec:
command: command:
- cat - cat
tty: true tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command:
- cat
tty: true
""" """
} }
} }
@ -74,8 +69,6 @@ spec:
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428' VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0' QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1' QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0' QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
@ -116,26 +109,6 @@ spec:
''' '''
} }
} }
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') { stage('Collect SonarQube evidence') {
steps { steps {
container('quality-tools') { container('quality-tools') {
@ -152,7 +125,6 @@ spec:
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx" "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
) )
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml") [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]} rc=${PIPESTATUS[0]}
@ -353,33 +325,6 @@ PY
esac esac
fi fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}" ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1 ironbank_required=1

View File

@ -20,7 +20,6 @@ CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
_build_check_statuses = _quality_helpers._build_check_statuses _build_check_statuses = _quality_helpers._build_check_statuses
_combine_statuses = _quality_helpers._combine_statuses _combine_statuses = _quality_helpers._combine_statuses
_infer_sonarqube_status = _quality_helpers._infer_sonarqube_status _infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
_infer_semgrep_status = _quality_helpers._infer_semgrep_status
_infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500 _infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
_infer_supply_chain_status = _quality_helpers._infer_supply_chain_status _infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
_infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent _infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
@ -287,7 +286,6 @@ def main() -> int:
if source_lines_over_500 <= 0: if source_lines_over_500 <= 0:
source_lines_over_500 = _infer_source_lines_over_500(summary) source_lines_over_500 = _infer_source_lines_over_500(summary)
sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json")) sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json"))
supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json")) supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
truthy = {"1", "true", "yes", "on"} truthy = {"1", "true", "yes", "on"}
supply_chain_required = ( supply_chain_required = (
@ -300,7 +298,6 @@ def main() -> int:
workspace_line_coverage_percent=workspace_line_coverage_percent, workspace_line_coverage_percent=workspace_line_coverage_percent,
source_lines_over_500=source_lines_over_500, source_lines_over_500=source_lines_over_500,
sonarqube_report=sonarqube_report, sonarqube_report=sonarqube_report,
semgrep_report=semgrep_report,
supply_chain_report=supply_chain_report, supply_chain_report=supply_chain_report,
supply_chain_required=supply_chain_required, supply_chain_required=supply_chain_required,
) )

View File

@ -18,7 +18,6 @@ CANONICAL_CHECKS = [
"docs_naming", "docs_naming",
"gate_glue", "gate_glue",
"sonarqube", "sonarqube",
"semgrep",
"supply_chain", "supply_chain",
] ]
@ -136,26 +135,12 @@ def _infer_supply_chain_status(report: dict, required: bool) -> str:
return normalized return normalized
def _infer_semgrep_status(report: dict) -> str:
"""Infer canonical Semgrep check status from its JSON report payload."""
if not report:
return "not_applicable"
status = report.get("status")
if status is None:
blocking_findings = report.get("blocking_findings")
errors_total = report.get("errors_total")
if isinstance(blocking_findings, int) and isinstance(errors_total, int):
return "failed" if blocking_findings > 0 or errors_total > 0 else "ok"
return _normalize_result_status(str(status) if status is not None else None, default="failed")
def _build_check_statuses( def _build_check_statuses(
summary: dict | None, summary: dict | None,
tests: dict[str, int], tests: dict[str, int],
workspace_line_coverage_percent: float, workspace_line_coverage_percent: float,
source_lines_over_500: int, source_lines_over_500: int,
sonarqube_report: dict, sonarqube_report: dict,
semgrep_report: dict,
supply_chain_report: dict, supply_chain_report: dict,
supply_chain_required: bool, supply_chain_required: bool,
) -> dict[str, str]: ) -> dict[str, str]:
@ -203,7 +188,6 @@ def _build_check_statuses(
gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable" gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report) sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report)
supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status( supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
supply_chain_report, supply_chain_report,
required=supply_chain_required, required=supply_chain_required,
@ -216,6 +200,5 @@ def _build_check_statuses(
"docs_naming": docs_naming_status, "docs_naming": docs_naming_status,
"gate_glue": gate_glue_status, "gate_glue": gate_glue_status,
"sonarqube": sonarqube_status, "sonarqube": sonarqube_status,
"semgrep": semgrep_status,
"supply_chain": supply_chain_status, "supply_chain": supply_chain_status,
} }

View File

@ -1,159 +0,0 @@
#!/usr/bin/env python3
"""Build a compact Semgrep gate report from Semgrep JSON output."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
BLOCKING_SEVERITIES = {"ERROR"}
SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN")
SONAR_SEVERITY_BY_SEMGREP = {
"ERROR": "CRITICAL",
"WARNING": "MAJOR",
"INFO": "MINOR",
"UNKNOWN": "INFO",
}
def _read_json(path: Path) -> dict[str, Any]:
"""Read a JSON object from disk, returning an error-shaped payload on failure."""
if not path.exists():
return {"errors": [{"message": f"report missing: {path}"}], "results": []}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
if not isinstance(payload, dict):
return {"errors": [{"message": "report payload is not an object"}], "results": []}
return payload
def _finding_severity(finding: dict[str, Any]) -> str:
"""Return a normalized Semgrep finding severity."""
extra = finding.get("extra")
severity = extra.get("severity") if isinstance(extra, dict) else None
normalized = str(severity or "UNKNOWN").strip().upper()
return normalized if normalized in SEVERITIES else "UNKNOWN"
def build_report(
semgrep_payload: dict[str, Any],
*,
semgrep_exit_code: int,
blocking_severities: set[str] | None = None,
) -> dict[str, Any]:
"""Summarize Semgrep evidence into the quality-gate report contract."""
blocking = blocking_severities or BLOCKING_SEVERITIES
raw_results = semgrep_payload.get("results", [])
raw_errors = semgrep_payload.get("errors", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
severity_counts = {severity: 0 for severity in SEVERITIES}
blocking_findings = 0
for finding in results:
severity = _finding_severity(finding)
severity_counts[severity] += 1
if severity in blocking:
blocking_findings += 1
engine_error = semgrep_exit_code not in {0, 1} or bool(errors)
status = "failed" if engine_error or blocking_findings else "ok"
return {
"status": status,
"scanner": "semgrep",
"semgrep_rc": semgrep_exit_code,
"findings_total": len(results),
"blocking_findings": blocking_findings,
"errors_total": len(errors),
"severity_counts": severity_counts,
"blocking_severities": sorted(blocking),
}
def _line_number(value: Any, default: int = 1) -> int:
"""Return a Sonar-compatible one-indexed line number."""
try:
line = int(value)
except (TypeError, ValueError):
return default
return max(line, default)
def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
"""Convert one Semgrep finding into SonarQube generic issue format."""
path = str(finding.get("path") or "").strip()
if not path:
return None
extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {}
severity = _finding_severity(finding)
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
if end_line < start_line:
end_line = start_line
return {
"engineId": "semgrep",
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
"type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
"severity": SONAR_SEVERITY_BY_SEMGREP[severity],
"primaryLocation": {
"message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
"filePath": path,
"textRange": {
"startLine": start_line,
"endLine": end_line,
},
},
}
def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
"""Build SonarQube generic external issues from Semgrep JSON output."""
raw_results = semgrep_payload.get("results", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
return {"issues": issues}
def main(argv: list[str] | None = None) -> int:
"""CLI entrypoint used by Jenkins after Semgrep finishes."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--semgrep-json", required=True)
parser.add_argument("--exit-code", type=int, default=0)
parser.add_argument("--output", required=True)
parser.add_argument("--sonar-issues-output")
parser.add_argument(
"--blocking-severity",
action="append",
default=[],
help="Severity that should mark the report failed. Defaults to ERROR.",
)
args = parser.parse_args(argv)
blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()}
payload = _read_json(Path(args.semgrep_json))
report = build_report(
payload,
semgrep_exit_code=args.exit_code,
blocking_severities=blocking or BLOCKING_SEVERITIES,
)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.sonar_issues_output:
sonar_output = Path(args.sonar_issues_output)
sonar_output.parent.mkdir(parents=True, exist_ok=True)
sonar_output.write_text(
json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

View File

@ -131,11 +131,7 @@ def build_report(
critical = _count_vulnerabilities(trivy_payload, "CRITICAL") critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
high = _count_vulnerabilities(trivy_payload, "HIGH") high = _count_vulnerabilities(trivy_payload, "HIGH")
secrets = _count_secrets(trivy_payload) secrets = _count_secrets(trivy_payload)
status = ( status = "ok" if critical == 0 and secrets == 0 and not open_misconfigs else "failed"
"ok"
if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0
else "failed"
)
return { return {
"status": status, "status": status,

View File

@ -1,7 +1,7 @@
{ {
"version": 1, "version": 1,
"generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan", "generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
"default_expires_at": "2026-09-30", "default_expires_at": "2026-05-22",
"ticket": "atlas-quality-wave-k8s-hardening", "ticket": "atlas-quality-wave-k8s-hardening",
"default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.", "default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
"misconfigurations": [ "misconfigurations": [
@ -15,15 +15,15 @@
"id": "KSV-0009", "id": "KSV-0009",
"targets": [ "targets": [
"services/mailu/vip-controller.yaml", "services/mailu/vip-controller.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml" "services/maintenance/k3s-agent-restart-daemonset.yaml"
] ]
}, },
{ {
"id": "KSV-0010", "id": "KSV-0010",
"targets": [ "targets": [
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", "services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml" "services/monitoring/jetson-tegrastats-exporter.yaml"
] ]
}, },
@ -47,8 +47,8 @@
"services/bstein-dev-home/backend-deployment.yaml", "services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml", "services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml", "services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml", "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml", "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml", "services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml", "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml", "services/comms/atlasbot-deployment.yaml",
@ -59,16 +59,16 @@
"services/comms/livekit-token-deployment.yaml", "services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml", "services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml", "services/comms/mas-deployment.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml", "services/comms/oneoffs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml", "services/comms/oneoffs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml", "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml", "services/comms/oneoffs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml", "services/comms/oneoffs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml", "services/comms/oneoffs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml", "services/comms/oneoffs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml", "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml", "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml", "services/comms/oneoffs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml", "services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml", "services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml", "services/comms/seed-othrys-room.yaml",
@ -83,7 +83,7 @@
"services/finance/firefly-cronjob.yaml", "services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml", "services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml", "services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml", "services/finance/oneoffs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml", "services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml", "services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml", "services/health/wger-admin-ensure-cronjob.yaml",
@ -94,62 +94,63 @@
"services/jenkins/deployment.yaml", "services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml", "services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/deployment.yaml", "services/keycloak/deployment.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml", "services/keycloak/oneoffs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml", "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml", "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml", "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml", "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml", "services/keycloak/oneoffs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml", "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml", "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml", "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml", "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml", "services/keycloak/oneoffs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml", "services/keycloak/oneoffs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml", "services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml", "services/logging/oauth2-proxy.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml", "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml", "services/logging/oneoffs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml", "services/logging/oneoffs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml", "services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml", "services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml", "services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml", "services/mailu/mailu-sync-listener.yaml",
"services/mailu/oneoffs/mailu-sync-job.yaml",
"services/mailu/vault-sync-deployment.yaml", "services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml", "services/mailu/vip-controller.yaml",
"services/maintenance/apps/ariadne-deployment.yaml", "services/maintenance/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", "services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml", "services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", "services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml", "services/maintenance/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml", "services/maintenance/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", "services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml", "services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml", "services/maintenance/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml", "services/maintenance/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml", "services/maintenance/oneoffs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml", "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml", "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml", "services/maintenance/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml", "services/maintenance/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml", "services/maintenance/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml", "services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml", "services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml", "services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml", "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml", "services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml", "services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml", "services/monitoring/postmark-exporter-deployment.yaml",
@ -187,15 +188,15 @@
"services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", "services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml", "services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", "services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml", "services/maintenance/metis-deployment.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", "services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml", "services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml", "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
"services/monitoring/dcgm-exporter.yaml", "services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml" "services/monitoring/jetson-tegrastats-exporter.yaml"
] ]
@ -210,7 +211,7 @@
"services/comms/comms-secrets-ensure-rbac.yaml", "services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml", "services/comms/mas-db-ensure-rbac.yaml",
"services/comms/mas-secrets-ensure-rbac.yaml", "services/comms/mas-secrets-ensure-rbac.yaml",
"services/maintenance/apps/soteria-rbac.yaml" "services/maintenance/soteria-rbac.yaml"
] ]
}, },
{ {
@ -225,7 +226,7 @@
"services/comms/comms-secrets-ensure-rbac.yaml", "services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml", "services/comms/mas-db-ensure-rbac.yaml",
"services/jenkins/serviceaccount.yaml", "services/jenkins/serviceaccount.yaml",
"services/maintenance/apps/ariadne-rbac.yaml" "services/maintenance/ariadne-rbac.yaml"
] ]
}, },
{ {
@ -234,8 +235,8 @@
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml", "infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml", "infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
"services/jenkins/serviceaccount.yaml", "services/jenkins/serviceaccount.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml", "services/maintenance/disable-k3s-traefik-rbac.yaml",
"services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml" "services/maintenance/k3s-traefik-cleanup-rbac.yaml"
] ]
}, },
{ {
@ -265,8 +266,8 @@
"services/bstein-dev-home/backend-deployment.yaml", "services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml", "services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml", "services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml", "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml", "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml", "services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml", "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml", "services/comms/atlasbot-deployment.yaml",
@ -276,16 +277,16 @@
"services/comms/livekit-token-deployment.yaml", "services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml", "services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml", "services/comms/mas-deployment.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml", "services/comms/oneoffs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml", "services/comms/oneoffs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml", "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml", "services/comms/oneoffs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml", "services/comms/oneoffs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml", "services/comms/oneoffs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml", "services/comms/oneoffs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml", "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml", "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml", "services/comms/oneoffs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml", "services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml", "services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml", "services/comms/seed-othrys-room.yaml",
@ -299,7 +300,7 @@
"services/finance/firefly-cronjob.yaml", "services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml", "services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml", "services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml", "services/finance/oneoffs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml", "services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml", "services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml", "services/health/wger-admin-ensure-cronjob.yaml",
@ -308,62 +309,63 @@
"services/jellyfin/loader.yaml", "services/jellyfin/loader.yaml",
"services/jenkins/deployment.yaml", "services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml", "services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml", "services/keycloak/oneoffs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml", "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml", "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml", "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml", "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml", "services/keycloak/oneoffs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml", "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml", "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml", "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml", "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml", "services/keycloak/oneoffs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml", "services/keycloak/oneoffs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml", "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml", "services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml", "services/logging/oauth2-proxy.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml", "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml", "services/logging/oneoffs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml", "services/logging/oneoffs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml", "services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml", "services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml", "services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml", "services/mailu/mailu-sync-listener.yaml",
"services/mailu/oneoffs/mailu-sync-job.yaml",
"services/mailu/vault-sync-deployment.yaml", "services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml", "services/mailu/vip-controller.yaml",
"services/maintenance/apps/ariadne-deployment.yaml", "services/maintenance/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", "services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml", "services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", "services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml", "services/maintenance/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml", "services/maintenance/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", "services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", "services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", "services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml", "services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml", "services/maintenance/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml", "services/maintenance/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml", "services/maintenance/oneoffs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml", "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml", "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml", "services/maintenance/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml", "services/maintenance/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml", "services/maintenance/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml", "services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml", "services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml", "services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml", "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml", "services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml", "services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml", "services/monitoring/postmark-exporter-deployment.yaml",
@ -393,12 +395,12 @@
"services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", "services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml", "services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/apps/metis-deployment.yaml", "services/maintenance/metis-deployment.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", "services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml", "services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml" "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml"
] ]
} }
] ]

View File

@ -1,6 +0,0 @@
# clusters/aether/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- platform
- applications

View File

@ -6,10 +6,9 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes."
spec: spec:
interval: 10m interval: 10m
path: ./services/bstein-dev-home/migration-jobs path: ./services/bstein-dev-home/oneoffs/migrations
prune: true prune: true
force: true force: true
sourceRef: sourceRef:

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -1,28 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-chat/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-chat
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-chat
targetNamespace: hermes-chat
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: true
timeout: 30m
healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-chat
namespace: hermes-chat
dependsOn:
- name: cert-manager
- name: core
- name: hermes
- name: keycloak
- name: longhorn

View File

@ -1,16 +0,0 @@
# clusters/atlas/flux-system/applications/hermes-triage-demo/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: hermes-triage-demo
namespace: flux-system
spec:
interval: 10m
path: ./services/hermes-triage-demo
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
wait: false
timeout: 5m

View File

@ -18,10 +18,6 @@ spec:
wait: true wait: true
timeout: 45m timeout: 45m
healthChecks: healthChecks:
- apiVersion: apps/v1
kind: Deployment
name: hermes-model-gate
namespace: hermes
- apiVersion: apps/v1 - apiVersion: apps/v1
kind: Deployment kind: Deployment
name: hermes-ollama name: hermes-ollama
@ -30,10 +26,6 @@ spec:
kind: Deployment kind: Deployment
name: hermes name: hermes
namespace: hermes namespace: hermes
- apiVersion: apps/v1
kind: Deployment
name: oauth2-proxy-hermes
namespace: hermes
dependsOn: dependsOn:
- name: cert-manager - name: cert-manager
- name: core - name: core

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -28,8 +28,6 @@ resources:
- ai-llm/kustomization.yaml - ai-llm/kustomization.yaml
- openclaw/kustomization.yaml - openclaw/kustomization.yaml
- hermes/kustomization.yaml - hermes/kustomization.yaml
- hermes-chat/kustomization.yaml
- hermes-triage-demo/kustomization.yaml
- game-stream/kustomization.yaml - game-stream/kustomization.yaml
- cassandra-auth/kustomization.yaml - cassandra-auth/kustomization.yaml
- cassandra/kustomization.yaml - cassandra/kustomization.yaml

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization."
spec: spec:
interval: 30m interval: 30m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -6,7 +6,6 @@ metadata:
namespace: flux-system namespace: flux-system
annotations: annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized."
spec: spec:
interval: 10m interval: 10m
suspend: true suspend: true

View File

@ -1,4 +1,4 @@
# clusters/aether/flux-system/platform/kustomization.yaml # clusters/oceanus/applications/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1 apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: [] resources: []

View File

@ -0,0 +1,9 @@
# clusters/oceanus/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Populate when oceanus cluster is bootstrapped with Flux.
# - gotk-components.yaml
# - gotk-sync.yaml
- ../platform
- ../applications

View File

@ -0,0 +1,6 @@
# clusters/oceanus/platform/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../infrastructure/modules/base
- ../../infrastructure/modules/profiles/oceanus-validator

View File

@ -1,122 +0,0 @@
# syntax=docker/dockerfile:1
# dockerfiles/Dockerfile.hermes-agent
FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973
USER root
# Keep dashboard chat sockets tied to the intended React mount and conversation.
# A resumed conversation needs a different PTY attachment key from a fresh chat;
# reconnects to that same conversation must keep using the same key.
RUN node <<'NODE'
const fs = require("node:fs");
const path = "/opt/hermes/web/src/pages/ChatPage.tsx";
let source = fs.readFileSync(path, "utf8");
const socketBefore = [
' const url = await api.buildWsUrl("/api/pty", params);',
' const ws = new WebSocket(url);',
].join("\n");
const socketAfter = [
' const url = await api.buildWsUrl("/api/pty", params);',
' if (unmounting) return;',
' const ws = new WebSocket(url);',
].join("\n");
const attachBefore = ' params.attach = ptyAttachToken(forceFresh);';
const attachAfter = [
' const attachScope = resumeParam',
' ? `resume:${resumeParam}:${scopedProfile ?? ""}`',
' : `fresh:${scopedProfile ?? ""}`;',
' params.attach = `${ptyAttachToken(forceFresh)}:${attachScope}`;',
].join("\n");
if (!source.includes(socketBefore)) {
throw new Error("Hermes ChatPage WebSocket patch context changed");
}
if (!source.includes(attachBefore)) {
throw new Error("Hermes ChatPage PTY attachment patch context changed");
}
source = source.replace(socketBefore, socketAfter);
source = source.replace(attachBefore, attachAfter);
fs.writeFileSync(path, source);
NODE
# The upstream OIDC gate authenticates users but deliberately treats the
# dashboard as one shared workstation. Allow a deployment to narrow that
# workstation to explicit OIDC subjects. Enforce this after normal provider
# verification so a denied account is a 403, not a misleading provider 503.
RUN python - <<'PY'
from pathlib import Path
path = Path("/opt/hermes/hermes_cli/dashboard_auth/middleware.py")
source = path.read_text()
helper_before = '''def _client_ip(request: Request) -> str:
fwd = request.headers.get("x-forwarded-for", "")
if fwd:
return fwd.split(",")[0].strip()
return request.client.host if request.client else ""
'''
helper_after = helper_before + '''def _dashboard_user_allowed(session) -> bool:
"""Apply an optional deployment-level OIDC-subject allowlist."""
import os
allowed = {
value.strip()
for value in os.environ.get(
"HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS", ""
).split(",")
if value.strip()
}
return not allowed or session.user_id in allowed
def _user_forbidden_response() -> Response:
"""Return an authorization failure without exposing identities."""
return JSONResponse(
{
"error": "forbidden",
"detail": "This Atlas account is not authorized for this dashboard.",
},
status_code=403,
)
'''
refresh_before = ''' new_session, refreshing_provider = refreshed
request.state.session = new_session
response = await call_next(request)
'''
refresh_after = ''' new_session, refreshing_provider = refreshed
if not _dashboard_user_allowed(new_session):
return _user_forbidden_response()
request.state.session = new_session
response = await call_next(request)
'''
final_before = ''' request.state.session = session
return await call_next(request)
'''
final_after = ''' if not _dashboard_user_allowed(session):
return _user_forbidden_response()
request.state.session = session
return await call_next(request)
'''
for before, after, label in (
(helper_before, helper_after, "allowlist helper"),
(refresh_before, refresh_after, "refreshed session"),
(final_before, final_after, "verified session"),
):
if before not in source:
raise SystemExit(f"Hermes dashboard auth {label} patch context changed")
source = source.replace(before, after, 1)
path.write_text(source)
PY
COPY dockerfiles/hermes-session-migrate.py /opt/hermes/bin/hermes-session-migrate
RUN cd /opt/hermes/web \
&& npm run build \
&& grep -Fq 'if (unmounting) return;' src/pages/ChatPage.tsx \
&& grep -Fq 'resume:${resumeParam}' src/pages/ChatPage.tsx \
&& grep -Fq 'HERMES_DASHBOARD_OIDC_ALLOWED_USER_IDS' \
/opt/hermes/hermes_cli/dashboard_auth/middleware.py \
&& chmod 0755 /opt/hermes/bin/hermes-session-migrate

View File

@ -1,101 +0,0 @@
#!/usr/bin/env python3
"""Copy selected legacy Hermes sessions into an isolated user home."""
from __future__ import annotations
import os
from pathlib import Path
import shutil
import sqlite3
from hermes_state import SessionDB
def _copy_if_missing(source: Path, target: Path) -> None:
"""Copy one credential/config file without overwriting user state."""
if source.is_file() and not target.exists():
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(source, target)
def _copy_session(
source: sqlite3.Connection,
target: sqlite3.Connection,
session_id: str,
user_id: str,
) -> bool:
"""Copy a session and its messages, preserving original timestamps."""
source.row_factory = sqlite3.Row
session = source.execute(
"SELECT * FROM sessions WHERE id = ?", (session_id,)
).fetchone()
if session is None:
return False
columns = list(session.keys())
values = [user_id if column == "user_id" else session[column] for column in columns]
placeholders = ", ".join("?" for _ in columns)
target.execute(
f"INSERT OR IGNORE INTO sessions ({', '.join(columns)}) "
f"VALUES ({placeholders})",
values,
)
message_columns = [
row[1] for row in target.execute("PRAGMA table_info(messages)").fetchall()
]
selected_columns = [
column
for column in message_columns
if column in {
row[1]
for row in source.execute("PRAGMA table_info(messages)").fetchall()
}
]
column_sql = ", ".join(selected_columns)
target.execute(
f"INSERT OR IGNORE INTO messages ({column_sql}) "
f"SELECT {column_sql} FROM source_db.messages WHERE session_id = ?",
(session_id,),
)
return True
def main() -> None:
"""Create the isolated home and perform the idempotent session copy."""
source_home = Path(os.environ["HERMES_MIGRATE_SOURCE_HOME"])
target_home = Path(os.environ["HERMES_HOME"])
user_id = os.environ["HERMES_MIGRATE_USER_ID"].strip()
session_ids = [
value.strip()
for value in os.environ.get("HERMES_MIGRATE_SESSION_IDS", "").split(",")
if value.strip()
]
target_home.mkdir(parents=True, exist_ok=True)
(target_home / "home" / ".local" / "bin").mkdir(parents=True, exist_ok=True)
(target_home / "workspace" / "skills").mkdir(parents=True, exist_ok=True)
(target_home / "logs").mkdir(parents=True, exist_ok=True)
for filename in (".env", "auth.json"):
_copy_if_missing(source_home / filename, target_home / filename)
source_db = source_home / "state.db"
target_db = target_home / "state.db"
SessionDB(db_path=target_db).close()
copied = 0
if source_db.is_file() and session_ids:
source = sqlite3.connect(f"file:{source_db}?mode=ro", uri=True)
target = sqlite3.connect(target_db)
target.execute("ATTACH DATABASE ? AS source_db", (str(source_db),))
try:
with target:
for session_id in session_ids:
copied += int(_copy_session(source, target, session_id, user_id))
finally:
target.close()
source.close()
print(f"isolated Hermes home ready; migrated_sessions={copied}")
if __name__ == "__main__":
main()

View File

@ -12,7 +12,7 @@ all:
ansible_host: REPLACE_ME ansible_host: REPLACE_ME
ansible_user: debian ansible_user: debian
roleset: minipc_gpu roleset: minipc_gpu
dedicated_hosts: baremetal:
hosts: hosts:
titan-db: titan-db:
ansible_host: REPLACE_ME ansible_host: REPLACE_ME
@ -22,3 +22,7 @@ all:
ansible_host: REPLACE_ME ansible_host: REPLACE_ME
ansible_user: jump ansible_user: jump
roleset: jumphost roleset: jumphost
oceanus:
ansible_host: REPLACE_ME
ansible_user: validator
roleset: validator

View File

@ -14,6 +14,13 @@
- common - common
- titan_jh - titan_jh
- name: Configure oceanus validator host
hosts: oceanus
gather_facts: true
roles:
- common
- oceanus_base
- name: Prepare hybrid tethys node - name: Prepare hybrid tethys node
hosts: titan-24 hosts: titan-24
gather_facts: true gather_facts: true

View File

@ -0,0 +1,6 @@
# hosts/roles/oceanus_base/tasks/main.yaml
---
- name: Placeholder for oceanus base configuration
ansible.builtin.debug:
msg: "Install validator prerequisites and monitoring exporters here."
tags: ['oceanus']

View File

@ -11,5 +11,5 @@ resources:
- coredns-deployment.yaml - coredns-deployment.yaml
- ntp-sync-daemonset.yaml - ntp-sync-daemonset.yaml
- workload-profiles.yaml - workload-profiles.yaml
- cert-manager/letsencrypt.yaml - ../sources/cert-manager/letsencrypt.yaml
- cert-manager/letsencrypt-prod.yaml - ../sources/cert-manager/letsencrypt-prod.yaml

View File

@ -2,4 +2,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1 apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: resources:
- ../../clusters/atlas/flux-system - ../clusters/atlas/flux-system

View File

@ -1,4 +1,4 @@
# clusters/aether/flux-system/applications/kustomization.yaml # infrastructure/modules/profiles/oceanus-validator/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1 apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: [] resources: []

View File

@ -1,4 +1,4 @@
# infrastructure/core/cert-manager/letsencrypt-prod.yaml # infrastructure/sources/cert-manager/letsencrypt-prod.yaml
apiVersion: cert-manager.io/v1 apiVersion: cert-manager.io/v1
kind: ClusterIssuer kind: ClusterIssuer
metadata: metadata:

View File

@ -1,4 +1,4 @@
# infrastructure/core/cert-manager/letsencrypt.yaml # infrastructure/sources/cert-manager/letsencrypt.yaml
apiVersion: cert-manager.io/v1 apiVersion: cert-manager.io/v1
kind: ClusterIssuer kind: ClusterIssuer
metadata: metadata:

View File

@ -4,6 +4,8 @@ items:
- apiVersion: apps/v1 - apiVersion: apps/v1
kind: Deployment kind: Deployment
metadata: metadata:
annotations:
deployment.kubernetes.io/revision: "4"
name: traefik name: traefik
namespace: traefik namespace: traefik
spec: spec:
@ -71,14 +73,6 @@ items:
node-role.kubernetes.io/worker: "true" node-role.kubernetes.io/worker: "true"
affinity: affinity:
nodeAffinity: nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: hardware
operator: In
values:
- rpi5
- rpi4
preferredDuringSchedulingIgnoredDuringExecution: preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100 - weight: 100
preference: preference:
@ -109,12 +103,6 @@ items:
operator: In operator: In
values: values:
- rpi4 - rpi4
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchLabels:
app: traefik
topologyKey: kubernetes.io/hostname
restartPolicy: Always restartPolicy: Always
schedulerName: default-scheduler schedulerName: default-scheduler
serviceAccount: atlas-traefik-ingress-controller serviceAccount: atlas-traefik-ingress-controller

View File

@ -33,12 +33,7 @@ spec:
enabled: false enabled: false
injector: injector:
enabled: true enabled: true
# Two replicas because the webhook is failurePolicy: Ignore. With a replicas: 1
# single replica, any pod created while the injector restarts is
# admitted unmutated: it comes up without its Vault agent sidecar,
# never finds /vault/secrets, and crash-loops indefinitely with no
# indication that injection was skipped. Observed twice on ariadne.
replicas: 2
agentImage: agentImage:
repository: hashicorp/vault repository: hashicorp/vault
tag: "1.17.6" tag: "1.17.6"

View File

@ -12,7 +12,7 @@ Layout
Regeneration Regeneration
- Update manifests/docs, then regenerate generated artifacts: - Update manifests/docs, then regenerate generated artifacts:
- `python scripts/render/knowledge_render_atlas.py --write` - `python scripts/knowledge_render_atlas.py --write`
Authoring rules Authoring rules
- Never include secret values. Prefer `secretRef` names or Vault paths like `kv/atlas/...`. - Never include secret values. Prefer `secretRef` names or Vault paths like `kv/atlas/...`.

View File

@ -1,8 +1,8 @@
{ {
"counts": { "counts": {
"helmrelease_host_hints": 22, "helmrelease_host_hints": 19,
"http_endpoints": 54, "http_endpoints": 45,
"services": 70, "services": 47,
"workloads": 100 "workloads": 74
} }
} }

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

View File

@ -1,19 +1,9 @@
flowchart LR flowchart LR
host_agent_bstein_dev["agent.bstein.dev"]
svc_hermes_hermes["hermes/hermes (Service)"]
host_agent_bstein_dev --> svc_hermes_hermes
wl_hermes_hermes["hermes/hermes (Deployment)"]
svc_hermes_hermes --> wl_hermes_hermes
host_auth_bstein_dev["auth.bstein.dev"] host_auth_bstein_dev["auth.bstein.dev"]
svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"] svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"]
host_auth_bstein_dev --> svc_sso_oauth2_proxy host_auth_bstein_dev --> svc_sso_oauth2_proxy
wl_sso_oauth2_proxy["sso/oauth2-proxy (Deployment)"] wl_sso_oauth2_proxy["sso/oauth2-proxy (Deployment)"]
svc_sso_oauth2_proxy --> wl_sso_oauth2_proxy svc_sso_oauth2_proxy --> wl_sso_oauth2_proxy
host_backup_bstein_dev["backup.bstein.dev"]
svc_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Service)"]
host_backup_bstein_dev --> svc_maintenance_oauth2_proxy_soteria
wl_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Deployment)"]
svc_maintenance_oauth2_proxy_soteria --> wl_maintenance_oauth2_proxy_soteria
host_bstein_dev["bstein.dev"] host_bstein_dev["bstein.dev"]
svc_bstein_dev_home_bstein_dev_home_frontend["bstein-dev-home/bstein-dev-home-frontend (Service)"] svc_bstein_dev_home_bstein_dev_home_frontend["bstein-dev-home/bstein-dev-home-frontend (Service)"]
host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_frontend host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_frontend
@ -121,16 +111,6 @@ flowchart LR
host_pegasus_bstein_dev --> svc_jellyfin_pegasus host_pegasus_bstein_dev --> svc_jellyfin_pegasus
wl_jellyfin_pegasus["jellyfin/pegasus (Deployment)"] wl_jellyfin_pegasus["jellyfin/pegasus (Deployment)"]
svc_jellyfin_pegasus --> wl_jellyfin_pegasus svc_jellyfin_pegasus --> wl_jellyfin_pegasus
host_quality_bstein_dev["quality.bstein.dev"]
svc_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Service)"]
host_quality_bstein_dev --> svc_quality_oauth2_proxy_sonarqube
wl_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Deployment)"]
svc_quality_oauth2_proxy_sonarqube --> wl_quality_oauth2_proxy_sonarqube
host_recovery_bstein_dev["recovery.bstein.dev"]
svc_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Service)"]
host_recovery_bstein_dev --> svc_maintenance_oauth2_proxy_metis
wl_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Deployment)"]
svc_maintenance_oauth2_proxy_metis --> wl_maintenance_oauth2_proxy_metis
host_scm_bstein_dev["scm.bstein.dev"] host_scm_bstein_dev["scm.bstein.dev"]
svc_gitea_gitea["gitea/gitea (Service)"] svc_gitea_gitea["gitea/gitea (Service)"]
host_scm_bstein_dev --> svc_gitea_gitea host_scm_bstein_dev --> svc_gitea_gitea
@ -161,20 +141,6 @@ flowchart LR
host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service
wl_vaultwarden_vaultwarden["vaultwarden/vaultwarden (Deployment)"] wl_vaultwarden_vaultwarden["vaultwarden/vaultwarden (Deployment)"]
svc_vaultwarden_vaultwarden_service --> wl_vaultwarden_vaultwarden svc_vaultwarden_vaultwarden_service --> wl_vaultwarden_vaultwarden
host_veles_bstein_dev["veles.bstein.dev"]
svc_veles_veles_frontend["veles/veles-frontend (Service)"]
host_veles_bstein_dev --> svc_veles_veles_frontend
wl_veles_veles_frontend["veles/veles-frontend (Deployment)"]
svc_veles_veles_frontend --> wl_veles_veles_frontend
svc_veles_veles_backend["veles/veles-backend (Service)"]
host_veles_bstein_dev --> svc_veles_veles_backend
wl_veles_veles_backend["veles/veles-backend (Deployment)"]
svc_veles_veles_backend --> wl_veles_veles_backend
host_wolf_bstein_dev["wolf.bstein.dev"]
svc_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Service)"]
host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf
wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"]
svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf
subgraph bstein_dev_home[bstein-dev-home] subgraph bstein_dev_home[bstein-dev-home]
svc_bstein_dev_home_bstein_dev_home_frontend svc_bstein_dev_home_bstein_dev_home_frontend
@ -209,10 +175,6 @@ flowchart LR
svc_finance_firefly svc_finance_firefly
wl_finance_firefly wl_finance_firefly
end end
subgraph game_stream[game-stream]
svc_game_stream_oauth2_proxy_wolf
wl_game_stream_oauth2_proxy_wolf
end
subgraph gitea[gitea] subgraph gitea[gitea]
svc_gitea_gitea svc_gitea_gitea
wl_gitea_gitea wl_gitea_gitea
@ -221,10 +183,6 @@ flowchart LR
svc_health_wger svc_health_wger
wl_health_wger wl_health_wger
end end
subgraph hermes[hermes]
svc_hermes_hermes
wl_hermes_hermes
end
subgraph jellyfin[jellyfin] subgraph jellyfin[jellyfin]
svc_jellyfin_pegasus svc_jellyfin_pegasus
wl_jellyfin_pegasus wl_jellyfin_pegasus
@ -246,12 +204,6 @@ flowchart LR
subgraph mailu_mailserver[mailu-mailserver] subgraph mailu_mailserver[mailu-mailserver]
svc_mailu_mailserver_mailu_front svc_mailu_mailserver_mailu_front
end end
subgraph maintenance[maintenance]
svc_maintenance_oauth2_proxy_soteria
wl_maintenance_oauth2_proxy_soteria
svc_maintenance_oauth2_proxy_metis
wl_maintenance_oauth2_proxy_metis
end
subgraph nextcloud[nextcloud] subgraph nextcloud[nextcloud]
svc_nextcloud_nextcloud svc_nextcloud_nextcloud
wl_nextcloud_nextcloud wl_nextcloud_nextcloud
@ -266,10 +218,6 @@ flowchart LR
svc_planka_planka svc_planka_planka
wl_planka_planka wl_planka_planka
end end
subgraph quality[quality]
svc_quality_oauth2_proxy_sonarqube
wl_quality_oauth2_proxy_sonarqube
end
subgraph sso[sso] subgraph sso[sso]
svc_sso_oauth2_proxy svc_sso_oauth2_proxy
wl_sso_oauth2_proxy wl_sso_oauth2_proxy
@ -284,9 +232,3 @@ flowchart LR
svc_vaultwarden_vaultwarden_service svc_vaultwarden_vaultwarden_service
wl_vaultwarden_vaultwarden wl_vaultwarden_vaultwarden
end end
subgraph veles[veles]
svc_veles_veles_frontend
wl_veles_veles_frontend
svc_veles_veles_backend
wl_veles_veles_backend
end

File diff suppressed because it is too large Load Diff

View File

@ -1,74 +0,0 @@
# Hermes Investigative Triage Demonstration
- Date: 2026-08-05
- Run ID: run_d2f61902c00044948a3912a94e895ab9
- Duration: 128 seconds, status completed
- Usage: {"input_tokens": 200961, "output_tokens": 4370, "total_tokens": 205331}
- Mode: supervised, read-only; approvals `smart` with mutation deny-list; zero mutations, zero denied approvals
- Tool audit (from the run's SSE event stream): 8 skill loads (triage-titan-test-failures pack: orchestrator, jenkins-retained-evidence, platform-quality-metrics, kubernetes-readonly-failure-classifier, flux-git-change-correlation, ...), 4 read-only terminal batches (Ariadne internal API sweep, kubectl logs/jobs/configmap in hermes-triage-demo, VictoriaMetrics queries incl. last_over_time fallback, Flux/Gitea revision correlation)
Unlike the automated loop (where Ariadne supplies a sanitized bundle), this run
was given ONLY an incident ID and gathered all evidence itself — the operator's
triage path executed by the agent. Notable behaviors: honest dead-end reporting
(Jenkins 403 -> pivot to Kubernetes-retained logs; Ariadne audit endpoints 404),
instant-query lookback miss -> switched to last_over_time, causal chain
reconstruction across five sources, and an unprompted, legitimate hardening
suggestion (add a build label to ariadne_hermes_triage_action_total).
## Prompt
```text
Use $triage-titan-test-failures.
Investigate Jenkins incident hermes-triage-demo/10. No evidence bundle is attached this time: gather the evidence yourself with your read-only tools, following the operator's standard triage path:
1. Ariadne is the source of truth: query its internal API (base URL in ARIADNE_BASE_URL, e.g. GET /api/internal/testing/triage/latest and any internal audit endpoints you can reach) for records about this incident.
2. Quality metrics: query VictoriaMetrics (VICTORIA_METRICS_URL) for the series ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo",build="10"} and ariadne_hermes_triage_action_total, and note what they show.
3. Service health: use read-only kubectl to inspect the hermes-triage-demo namespace (jobs, pods, recent events) and confirm what ran there.
4. Change correlation: check Flux state (kubectl get kustomizations -n flux-system; the flux-system GitRepository revision) for recent changes relevant to this incident.
Then return a concise operator-style report with EXACTLY these sections:
FACTS (each with source and reference), INFERENCES, LIKELY CAUSE, WHAT HAPPENED (was it repaired, by what mechanism, final state), NEXT CHECKS (what a human should verify next), and REPO-SIDE FIXES (if any are warranted).
Read-only throughout. Do not mutate anything. Do not use kubectl apply/delete/patch/scale.
```
## Full agent report
```text
FACTS
Ariadne diagnosis/latest, generated 2026-08-05T21:31:06Z, references only the broad testing bundle and does not identify hermes-triage-demo/10. Ariadnes incident/audit candidate endpoints returned HTTP 404.
Kubernetes logs, hermes-triage-demo/hermes-demo-test-10-dgvpj: build 10 emitted incident_id="hermes-triage-demo/10", classification_hint="demo_fixture_unhealthy", and “expected fixture state healthy; found unhealthy.”
Kubernetes Job status: hermes-demo-test-10 was created 2026-08-05T20:59:47Z and failed; its pod ran on titan-06 and had no restarts or image-pull events.
Kubernetes logs, hermes-demo-repair-10-nkrzz: repair job patched hermes-triage-demo-fixture and reported “fixture state reset to healthy”; the Job completed successfully at 2026-08-05T21:01:06Z.
Kubernetes logs/status, hermes-demo-test-11-hv5mh: follow-up test completed at 2026-08-05T21:01:35Z and reported “fixture state healthy.”
Kubernetes ConfigMap hermes-triage-demo-fixture: current data.state is healthy.
VictoriaMetrics, last_over_time(ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo",build="10"}[24h]): sample at 2026-08-05T21:41:01Z has resolved=1 and every other lifecycle state, including failed and human_required, at 0. The instantaneous selector returned no result because its samples were outside the default instant-query lookback.
VictoriaMetrics, ariadne_hermes_triage_action_total over 24h: repair_demo_fixture has requested=1, accepted=1, success=1 for instance 10.42.19.143; no build label exists on this counter, so it is correlated by the incident instance and timeline rather than proven per-build.
Flux Kustomization hermes-triage-demo: Ready=True at revision main@sha1:aa36b06b2bc43eda21b36e54a7d312e55d27c370. Flux source GitRepository is Ready=True at that same revision.
Gitea commit 49aab8da3298ebfc1b9edab5ce2e6938ad033a23, “feat(hermes-triage): enable autoremediation for the demo loop,” predates this incident and modifies services/maintenance/apps/ariadne-deployment.yaml. The current revision aa36b06 is an unrelated maintenance image update.
Direct Jenkins GET for /job/hermes-triage-demo/10 API metadata and console returned HTTP 403. The Kubernetes Job logs are therefore the retained direct execution evidence used here.
INFERENCES
The failure was an intentional demo-fixture health assertion failure, not an image, scheduling, node, or storage failure. The failed test, repair, and successful follow-up all ran in sequence; this is a strong causal chain.
The repair was automatically accepted under the previously deployed allowlisted autoremediation path. The repair Job itself performed the ConfigMap patch; Flux did not repair the fixture during this incident.
The unrelated cluster concerns in the Ariadne bundle, including Longhorn probe failures and pending Veles pods, have no evidenced link to this ConfigMap-based test path.
LIKELY CAUSE
The fixture was deliberately or otherwise pre-seeded unhealthy before Jenkins incident hermes-triage-demo/10. The test correctly failed on that condition. Confidence: high.
WHAT HAPPENED
Build 10 failed at 20:59:47Z because the fixture state was unhealthy. The deployed autoremediation mechanism accepted repair_demo_fixture, ran hermes-demo-repair-10, and patched only hermes-triage-demo-fixture to healthy. Test 11 then passed at 21:01:35Z. Final state is repaired/resolved: ConfigMap state=healthy, incident metric resolved=1, repair success counter=1, and the demo Flux Kustomization is Ready at aa36b06.
NEXT CHECKS
Human should verify the current deployed revision and readiness after the next Flux interval:
kubectl -n flux-system get kustomization hermes-triage-demo
kubectl -n flux-system describe gitrepository flux-system
Human should retain the build-10 Jenkins evidence if access is available:
curl -fsS https://ci.bstein.dev/job/hermes-triage-demo/10/api/json
curl -fsS https://ci.bstein.dev/job/hermes-triage-demo/10/consoleText
These currently require Jenkins authentication (HTTP 403).
Human should confirm that the resolved gauge remains the only asserted lifecycle state:
curl -G -fsS --data-urlencode 'query=last_over_time(ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo",build="10"}[24h])' "$VICTORIA_METRICS_URL/api/v1/query"
REPO-SIDE FIXES
None warranted for this incident. The observed behavior matches the committed demo design: an isolated ConfigMap fixture, one allowlisted repair action, a scoped repair ServiceAccount, and one repair per incident. A future hardening change, if desired, would be to add a build label to ariadne_hermes_triage_action_total so repair counters can be directly attributed to a specific incident rather than correlated by instance and time.
```

View File

@ -1,166 +0,0 @@
# Corrections for `mermaid/TestAutomationV2.mmd`
Written 2026-08-06 against the diagram as it stood that morning. The diagram
is not edited here on purpose: it is being maintained in parallel, so this is
the change list rather than a patch.
The structure holds. The trust boundary, the gate chain, the action registry
and the output taxonomy are all still accurate. What follows is what tonight's
work invalidated.
## Nodes that are now factually wrong
**`collector`** says *"Keeps the head and tail"* and *"Keeps the earliest
useful failure regions"*. Earliest-first selection was the defect: on a long
pipeline the byte budget was consumed entirely by successful tool output, and
the enforced failure never reached Hermes. Replace with:
```
Ariadne evidence collector
Reads the full console up to 2 MB
Ranks failure regions by evidence strength
A definite failure outranks a tool that merely ran
Ignores passing test lines that quote failures
Merges repeats and overlapping context
```
**`repair_fixture`** says *"Creates one scoped Job"*. It no longer does. The
repair is an in-process ConfigMap patch; no pod is created.
**`candidate_files`** says *"Ranks the earliest hints first"*. It now also
reads the failing test and follows its imports to the module under test, which
is what made a pull request on a real service repository possible at all:
```
Candidate file selection
Uses path and line hints from failure regions
Reads the failing test as context
Follows its imports to the module under test
Reading is wider than writing: a test is readable, never patchable
```
**`action_scope`** says *"No real service classification maps to a mutation"*.
No longer true. `retry_transient_infra` is allowlisted for every allowlisted
job, so a real service can now receive an action - a Jenkins rebuild.
**`timings`** - the fixture loop is not four minutes. Measure from the red
build, not from arming, because arming depends on the agent pool:
```
Red -> fixture patched healthy: 25s
Red -> rebuild green and incident resolved: 1m04s
```
**`examples`** - supersede with the stronger 2026-08-06 results, including
`ariadne/409` opening a real pull request with the correct one-line fix.
## Structurally misleading
**The `route` decision reads as three exclusive branches.** It is not. An
escalating incident attempts a code proposal *and* files an issue *and* stays
`human_required`. A pull request rides along on the escalation; it never
replaces the issue. As drawn, a reader concludes one substitutes for the
other.
**`branch_build` overstates coverage.** `hermes-code-demo-branches` only
builds branches in the demo repository. A pull request opened against a real
service repository gets no automatic branch build. Mark it `limited`.
**`alert_output` is stale.** Alerting no longer fires on every escalation.
Two narrow rules remain: a repair that ran and failed, and an escalation
untouched for six hours. The issue is the durable artifact; email is the
exception.
## Missing, and worth adding
- The demo classification and `repair_demo_fixture` are now forbidden on any
job other than `hermes-triage-demo`. Hermes misapplied that label to real
services twice at 0.96 and 0.99 confidence; the gates refused it both times,
and it can no longer be produced at all.
- **A build that never finishes** is escalated once it passes the time cap,
with no model call, because its console is still being written. Before this
existed such a build produced nothing anywhere while holding an agent slot.
- **A failure superseded by a newer build can be skipped entirely.** Detection
reads `lastBuild`, so a red build replaced quickly is never triaged.
## Second pass, after the remaining work landed
Everything previously marked conditional is now done, which changes three
things structurally rather than cosmetically.
**Not every incident reaches Hermes any more.** The diagram's main flow runs
intake -> hermes_plane -> controls -> response, which reads as though every
incident is diagnosed by the model. A build that overruns its time cap is now
escalated by Ariadne alone, with no model call at all, because its console is
still being written and any root cause would be invented. That needs its own
edge from the detector straight to the human path, bypassing `hermes_plane`
and `controls` entirely. It is also the honest picture of the trust boundary:
some conclusions are Ariadne's own observations, and the issue body now says
so rather than crediting a diagnosis that never happened.
**The Jenkins evidence limitation is gone, not narrowed.** Delete
`jenkins_limit` and the "Per-test evidence waits on Jenkins plugins" line from
`current_limits`. `junit` and `pipeline-stage-view` are installed, `testReport`
and `wfapi` return 200, and the bundle now carries `failed_tests` with the
failing test name, class and assertion, plus `first_failed_stage`. The
collector always fetched both and had simply been receiving 404s. The `bundle`
node should list them as populated fields, because they are what the diagnosis
now rests on rather than scraped console text.
**The detector watches branches, not just jobs.** A multibranch project is a
folder with no `lastBuild`; it was returned as skipped on every tick. The
detector now expands a folder into its branch jobs, capped at five, so
`detector` should say it watches configured jobs *and the branches inside
multibranch folders*.
Smaller corrections to the same nodes:
- `evidence_sources` may include a service's own namespace, not only the demo
namespace and `jenkins`, where the job is mapped to one.
- The human path files an issue even for an incident triaged before issue
filing existed, reusing the classification the original diagnosis recorded
so one open issue per job and classification still holds.
- Hermes's credential comes from Vault, not a manually created Secret. If the
diagram ever shows credential provenance, that is now the accurate source.
## Conditional on work that was blocked
`model_gate` should now read `anthropic/claude-opus-5` primary, with
`openai-codex/gpt-5.6-terra` first fallback and local `gpt-oss:20b` second.
`jenkins_limit` and *"Per-test evidence waits on Jenkins plugins"* come out
once the `junit` plugin is live. Note the plugin was never blocked on a core
upgrade as previously recorded: `junit 1369.v15da_00283f06` runs on core
2.528.3, only the latest release requires 2.533.
## Third pass: the escalation branch now has an output
The chart shows the human path as a dead end for the automation: no action
fits, an issue is filed, and Ariadne learns nothing. That is no longer the
whole story.
**A diagnosis may now propose a remediation Ariadne cannot perform.** When no
allowlisted action fits, Hermes may return `suggested_remediation` naming the
action it believes would work and the evidence that should be required before
running it. So the human path forks: an escalation always files an issue, and
some escalations also carry a proposal. The chart should show that second
output leaving the escalation node, labelled as a proposal for a maintainer
rather than as anything executable.
**The proposal must be drawn outside the trust boundary.** It is prose in an
issue and a field in an audit event. No gate reads it; an id that is not
already in the allowlist still fails `action_not_allowlisted`. If the boundary
is drawn as "Hermes advises, Ariadne acts", the proposal sits firmly on the
advisory side, and the arrow from it should terminate at a person, not at the
Action registry. The registry only grows when a human deploys a new action.
**The three allowlisted actions are now four.** `action_registry` should list
`repair_demo_fixture`, `retry_transient_infra`, `reclaim_workspace_storage`
and `clear_stuck_agent_pods`, each reachable only from its own classification.
Two of those are worth calling out on the chart as separate paths rather than
folding them into a generic retry, because the distinction is the point:
`workspace_storage_exhausted` reclaims stale workspace storage *before*
rebuilding, since a plain rebuild lands on the same full volume; and
`jenkins_agent_provisioning_failure` clears finished agent pods first, since a
retry otherwise queues behind the same stuck pool.

View File

@ -1,14 +1,12 @@
# Metis (node recovery) # Metis (node recovery)
## Node classes (current map) ## Node classes (current map)
- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent) - rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)
- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint) - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)
- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks) - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)
- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent) - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)
- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)
- amd64 agents: titan-22/24 (Debian 13, k3s agent) - amd64 agents: titan-22/24 (Debian 13, k3s agent)
- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels) - External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, future titan-20/21 (when added), plus any newcomers.
- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.
## Longhorn disk UUIDs (critical nodes) ## Longhorn disk UUIDs (critical nodes)
- titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4) - titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)
@ -19,9 +17,10 @@
## Metis repo (~/Development/metis) ## Metis repo (~/Development/metis)
- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`). - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).
- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints). - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).
- `AGENTS.md` in repo is untracked and holds raw notes.
## Next implementation steps ## Next implementation steps
- Add per-class golden image refs and checksums (Harbor or file://) when ready. - Add per-class golden image refs and checksums (Harbor or file://) when ready.
- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths. - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.
- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection. - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.
- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited. - Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.

View File

@ -3,7 +3,7 @@ title: "CI: Gitea → Jenkins pipeline"
tags: ["atlas", "ci", "gitea", "jenkins"] tags: ["atlas", "ci", "gitea", "jenkins"]
owners: ["brad"] owners: ["brad"]
entrypoints: ["scm.bstein.dev", "ci.bstein.dev"] entrypoints: ["scm.bstein.dev", "ci.bstein.dev"]
source_paths: ["services/gitea", "services/jenkins", "scripts/sync/jenkins_cred_sync.sh", "scripts/sync/gitea_cred_sync.sh"] source_paths: ["services/gitea", "services/jenkins", "scripts/jenkins_cred_sync.sh", "scripts/gitea_cred_sync.sh"]
--- ---
# CI: Gitea → Jenkins pipeline # CI: Gitea → Jenkins pipeline
@ -14,7 +14,7 @@ Atlas uses Gitea for source control and Jenkins for CI. Authentication is via Ke
## Where it is configured ## Where it is configured
- Gitea manifests: `services/gitea/` - Gitea manifests: `services/gitea/`
- Jenkins manifests: `services/jenkins/` - Jenkins manifests: `services/jenkins/`
- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh` - Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh`
## What users do (typical flow) ## What users do (typical flow)
- Create a repo in Gitea. - Create a repo in Gitea.

View File

@ -14,8 +14,8 @@ Bootstrapping risk to remember
- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back. - Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.
Script Script
- `scripts/ops/cluster_power_recovery.sh` - `scripts/cluster_power_recovery.sh`
- `scripts/ops/cluster_power_console.sh` - `scripts/cluster_power_console.sh`
- Modes: - Modes:
- `prepare` - `prepare`
- `shutdown` - `shutdown`
@ -26,21 +26,21 @@ Script
Dry-run examples Dry-run examples
- Shutdown preview: - Shutdown preview:
- `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain` - `scripts/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`
- Startup preview: - Startup preview:
- `scripts/ops/cluster_power_recovery.sh startup` - `scripts/cluster_power_recovery.sh startup`
- Harbor seed preview: - Harbor seed preview:
- `scripts/ops/cluster_power_recovery.sh harbor-seed` - `scripts/cluster_power_recovery.sh harbor-seed`
Execute examples Execute examples
- Prepare helper image on every node: - Prepare helper image on every node:
- `scripts/ops/cluster_power_recovery.sh prepare --execute` - `scripts/cluster_power_recovery.sh prepare --execute`
- Seed Harbor runtime images onto `titan-05` from the control-host bundle: - Seed Harbor runtime images onto `titan-05` from the control-host bundle:
- `scripts/ops/cluster_power_recovery.sh harbor-seed --execute` - `scripts/cluster_power_recovery.sh harbor-seed --execute`
- Planned shutdown: - Planned shutdown:
- `scripts/ops/cluster_power_recovery.sh shutdown --execute` - `scripts/cluster_power_recovery.sh shutdown --execute`
- Planned startup (canonical branch): - Planned startup (canonical branch):
- `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main` - `scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main`
Manual remote console examples Manual remote console examples
- Canonical operator hosts: - Canonical operator hosts:
@ -129,7 +129,7 @@ Operational notes
- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer. - Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.
- Harbor is reconciled after the first critical stateful services. - Harbor is reconciled after the first critical stateful services.
- Harbor bootstrap is now designed around a control-host bundle: - Harbor bootstrap is now designed around a control-host bundle:
- Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`. - Build the Harbor bundle locally with `scripts/build_harbor_bootstrap_bundle.sh`.
- Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`. - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.
- Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`. - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.
- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations. - The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.

View File

@ -1,145 +0,0 @@
# Hermes Automated Triage Demo — Runbook
How to arm, run, watch, and roll back the automated failure-to-repair demo.
Architecture background: `knowledge/hermes-automated-triage-24h-plan.md`.
## What the loop does
```text
You arm the failure (SEED_FAILURE=true)
-> Jenkins job hermes-triage-demo fails (fixture reads "unhealthy")
-> test-runner pod also writes the incident JSON to stdout
(Fluent Bit ships it to OpenSearch kube-*)
-> Ariadne polls the job every minute, opens incident <job>/<build>
-> Ariadne bundles Jenkins evidence + bounded OpenSearch excerpts
-> Ariadne calls the Hermes Agent API (/v1/runs) with
$triage-titan-test-failures
-> Hermes returns schema-valid diagnosis + requested_action
-> Ariadne authorizes (twelve gates) and, if remediation is enabled,
patches the fixture ConfigMap back to "healthy" in process
-> Ariadne triggers ONE rebuild with SEED_FAILURE=false
-> rebuild passes -> incident resolved
Anything else -> issue in the service repo + human_required metric
```
The model behind Hermes is `anthropic/claude-opus-5` as of 2026-08-06, with
`openai-codex/gpt-5.6-terra` as first fallback and a local `gpt-oss:20b`
behind that, so an expired Anthropic credential degrades rather than stops.
Two behaviours worth knowing before you demo, because both look like nothing
happening:
- **A build that never finishes** is escalated once it passes
`ARIADNE_HERMES_HUNG_BUILD_MINUTES` (default 45). No model is consulted -
the console is still being written - so the issue says only that the build
overran and is holding an agent slot.
- **Alerting no longer fires on every escalation.** The issue in the service
repository is the durable artifact. Email now means either a repair ran and
failed, or an escalation has sat untouched for six hours.
## Arming the demo
Jenkins UI: `https://ci.bstein.dev/job/hermes-triage-demo/` → *Build with
Parameters* → check `SEED_FAILURE` → Build.
CLI (any Jenkins user API token):
```bash
curl -u <user>:<token> -X POST \
"https://ci.bstein.dev/job/hermes-triage-demo/buildWithParameters?SEED_FAILURE=true"
```
That is the only manual step. Everything after the red build is automatic.
Run `scripts/ops/hermes_triage_demo.sh preflight` first. The check that most
often decides whether a rehearsal holds its timings is the agent pool: the
Kubernetes cloud caps concurrent agent pods at `containerCapStr: "5"`, and when
real CI has taken all five the demo build sits in the queue reporting *"All
nodes of label ... are offline"* with no other symptom. Observed 2026-08-06: a
run armed at 02:13:42 did not start seeding until 02:19:37, close to six
minutes of dead air, purely because five other builds held the pool. Wait for a
free slot before starting, or quiesce CI.
## Expected timings (measured live 2026-08-06, Ariadne 0.1.0-402)
Time the demo from the moment the build goes **red**, not from arming. Arming
only queues a Jenkins build, and that leg is at the mercy of the agent pool.
- Red → fixture patched back to healthy: **25s** (≤60 s detection tick, ~16 s
Hermes diagnosis, then a single in-process Kubernetes API call).
- Red → rebuild triggered: **38s**.
- Red → rebuild green and incident resolved: **1m04s**.
The whole automated leg is just over a minute, and there is no silent phase
longer than the detection tick. Budgeted Hermes timeout is 420 s; observed
diagnosis runs are 1521 s.
Arming → red was **6m21s** on this run, but 5m35s of that was queue wait
behind a saturated agent pool. On an idle pool expect roughly 1m45s.
The earlier 2026-08-05 figures (1m15s red→repaired, 4m00s total) were measured
when the repair spawned its own Kubernetes Job. Converting the repair to an
in-process call removed a whole pod launch from the critical path.
## Watching it live
- Jenkins: `https://ci.bstein.dev/job/hermes-triage-demo/` (red build N,
then green build N+1 with `SEED_FAILURE=false`).
- Ariadne incident state:
`GET http://ariadne.maintenance/api/internal/audit/events` (in-cluster) or
`/api/admin/audit/events` (Keycloak JWT) — event types
`hermes_autotriage_incident`, `hermes_autotriage_diagnosis`,
`hermes_autotriage_action`. Status flow:
`detected → diagnosed → repairing → awaiting_rebuild → resolved`.
- Metrics (VictoriaMetrics / Grafana Explore):
`ariadne_hermes_triage_incident{jenkins_job="hermes-triage-demo"}`,
`ariadne_hermes_triage_action_total`,
`ariadne_hermes_triage_duration_seconds`.
- Repair evidence: `kubectl -n hermes-triage-demo get jobs` shows
`hermes-demo-test-<N>` (failed) and `hermes-demo-repair-<N>` (succeeded);
both TTL-clean after 1 h.
- Hermes side: the run appears in the dashboard at
`https://agent.bstein.dev` (session/run history).
- Escalation path: alert `HermesTriageHumanRequired` in vmalert
(`vmalert-atlas-availability` deployment, 1 m interval, `for: 2m`)
fires to Alertmanager for any `human_required` incident. Gauges are
republished every tick from stored incident state (restart-safe), and
the alert self-clears once a newer build of the same job is green.
Note: Alertmanager's default receiver is currently null — the alert is
visible in vmalert/Alertmanager/Grafana but pushes no notification.
## Demonstrating safe escalation (second path)
Any failure that does not match the demo-fixture signature — or any
invalid/low-confidence/unknown-action Hermes response — ends as
`status="human_required"` with **no mutation**. The simplest live demo:
temporarily set `ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` (see below)
and arm the failure; Ariadne diagnoses fully but executes nothing, and the
alert fires instead.
## Kill switch and rollback
- Instant behavioral off-switch (Flux-managed, in
`services/maintenance/apps/ariadne-deployment.yaml`):
`ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false` → diagnose-only.
`ARIADNE_HERMES_AUTOTRIAGE_ENABLED=false` → fully off.
- The automatic loop can only ever: create Jobs named
`hermes-demo-repair-*` in namespace `hermes-triage-demo`, and trigger
rebuilds of allowlisted jobs (`ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST`,
currently `hermes-triage-demo` only). One action + one rebuild per
incident, ever (storage-backed idempotency).
- Full teardown: remove `hermes-triage-demo` from
`clusters/atlas/flux-system/applications/kustomization.yaml` (Flux prunes
the namespace) and delete the `pipelineJob('hermes-triage-demo')` block
from `services/jenkins/configmap-jcasc.yaml`.
## Credentials
- Ariadne → Hermes: `Authorization: Bearer` key shared via the
`hermes-api-server-key` Secret present in both `hermes` and `maintenance`
namespaces (Hermes's init container seeds it into the persistent `.env`).
NOTE: currently manually created (Vault migration pending — see plan
handoff); rotating = write new value to both Secrets, restart hermes
deployment and ariadne deployment.
- Ariadne → Jenkins: existing `JENKINS_API_USER/TOKEN` from Vault
(`atlas/maintenance/ariadne-db`).

View File

@ -3,7 +3,7 @@ title: "KB authoring: what to write (and what not to)"
tags: ["atlas", "kb", "runbooks"] tags: ["atlas", "kb", "runbooks"]
owners: ["brad"] owners: ["brad"]
entrypoints: [] entrypoints: []
source_paths: ["knowledge/runbooks", "scripts/render/knowledge_render_atlas.py"] source_paths: ["knowledge/runbooks", "scripts/knowledge_render_atlas.py"]
--- ---
# KB authoring: what to write (and what not to) # KB authoring: what to write (and what not to)

View File

@ -1,14 +1,12 @@
# Metis (node recovery) # Metis (node recovery)
## Node classes (current map) ## Node classes (current map)
- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent) - rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)
- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint) - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)
- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks) - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)
- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent) - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)
- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)
- amd64 agents: titan-22/24 (Debian 13, k3s agent) - amd64 agents: titan-22/24 (Debian 13, k3s agent)
- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels) - External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers.
- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.
### Jetson nodes (titan-20/21) ### Jetson nodes (titan-20/21)
- Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64. - Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.
@ -24,12 +22,13 @@
## Metis repo (~/Development/metis) ## Metis repo (~/Development/metis)
- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`). - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).
- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints). - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).
- `AGENTS.md` in repo is untracked and holds raw notes.
## Next implementation steps ## Next implementation steps
- Add per-class golden image refs and checksums (Harbor or file://) when ready. - Add per-class golden image refs and checksums (Harbor or file://) when ready.
- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths. - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.
- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection. - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.
- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited. - Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.
## Node OS/Kernel/CRI snapshot (Jan 2026) ## Node OS/Kernel/CRI snapshot (Jan 2026)
- titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64 - titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64
@ -56,10 +55,10 @@
- titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64 - titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64
### Dedicated and special-purpose hosts ### External hosts
- titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled. - titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.
- titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q). - titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).
- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement. - titan-23/oceanus: TODO audit (future).
### Control plane Pis (titan-0a/0b/0c) ### Control plane Pis (titan-0a/0b/0c)

View File

@ -1,503 +0,0 @@
%% Titan Lab physical hardware architecture.
%% Solid links are physical network, storage, power, or sensor paths.
%% Dashed links are administration, telemetry, or recovery control.
%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "NETWORK_SIMPLEX", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
flowchart TB
internet["Internet"]:::external
ac_cloud["AC Infinity account"]:::external
subgraph lab["Titan Lab"]
direction TB
subgraph link_legend["Link colors"]
direction LR
subgraph legend_network["Network"]
direction TB
legend_1g["1 Gbps"]:::legend1g
legend_25g["2.5 Gbps"]:::legend25g
legend_external["External network"]:::legendExternal
end
subgraph legend_resources["Resources"]
direction TB
legend_local["Local attachment"]:::legendLocal
legend_shared["Shared storage"]:::legendShared
legend_gpu["GPU"]:::legendGpu
end
subgraph legend_operations["Operations"]
direction TB
legend_power["Power and NUT"]:::legendPower
legend_telemetry["Telemetry and environment"]:::legendTelemetry
legend_control["Administration and recovery"]:::legendControl
legend_safety["Protective shutdown"]:::legendSafety
end
end
subgraph network["Network fabric"]
direction LR
router["ASUS RT-AX88U<br/>Router"]:::network
cisco["Cisco CBS110-24T<br/>1 Gbps switch"]:::network
trendnet["TRENDnet 9-port<br/>2.5 Gbps switch"]:::network
router ==> cisco
router ==> trendnet
end
subgraph atlas["Atlas Kubernetes cluster"]
direction TB
subgraph control_plane["High-availability control plane, ARM64"]
direction LR
subgraph titan_0a["titan-0a"]
direction TB
t0a_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
t0a_storage["Local<br/>500 GiB boot SSD"]:::localStorage
t0a_host --- t0a_storage
end
subgraph titan_0b["titan-0b"]
direction TB
t0b_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
t0b_storage["Local<br/>500 GiB boot SSD"]:::localStorage
t0b_host --- t0b_storage
end
subgraph titan_0c["titan-0c"]
direction TB
t0c_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
t0c_storage["Local<br/>500 GiB boot SSD"]:::localStorage
t0c_host --- t0c_storage
end
t0a_host --- t0b_host
t0b_host --- t0c_host
end
subgraph workers["Workers"]
direction TB
subgraph pi5_workers["Newer workers, ARM64 Raspberry Pi 5"]
direction TB
subgraph pi5_row_a[" "]
direction LR
subgraph titan_04["titan-04"]
direction TB
t04_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
t04_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
t04_host --- t04_storage
end
subgraph titan_05["titan-05"]
direction TB
t05_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
t05_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
t05_host --- t05_storage
end
subgraph titan_06["titan-06"]
direction TB
t06_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
t06_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
t06_host --- t06_storage
end
end
subgraph pi5_row_b[" "]
direction LR
subgraph titan_07["titan-07"]
direction TB
t07_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
t07_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
t07_host --- t07_storage
end
subgraph titan_08["titan-08"]
direction TB
t08_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
t08_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
t08_host --- t08_storage
end
subgraph titan_11["titan-11"]
direction TB
t11_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
t11_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
t11_host --- t11_storage
end
end
end
subgraph pi4_workers["Older workers, ARM64 Raspberry Pi 4"]
direction TB
subgraph pi4_row_a[" "]
direction LR
subgraph titan_12["titan-12"]
direction TB
t12_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t12_root["Local<br/>SD root"]:::localStorage
t12_host --- t12_root
end
subgraph titan_13["titan-13"]
direction TB
t13_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t13_root["Local<br/>SD root"]:::localStorage
t13_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
t13_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
t13_host --- t13_root
t13_host --- t13_astreae
t13_host --- t13_asteria
end
subgraph titan_14["titan-14"]
direction TB
t14_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t14_root["Local<br/>SD root"]:::localStorage
t14_host --- t14_root
end
subgraph titan_15["titan-15"]
direction TB
t15_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t15_root["Local<br/>SD root"]:::localStorage
t15_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
t15_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
t15_host --- t15_root
t15_host --- t15_astreae
t15_host --- t15_asteria
end
end
subgraph pi4_row_b[" "]
direction LR
subgraph titan_17["titan-17"]
direction TB
t17_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t17_root["Local<br/>SD root"]:::localStorage
t17_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
t17_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
t17_host --- t17_root
t17_host --- t17_astreae
t17_host --- t17_asteria
end
subgraph titan_18["titan-18"]
direction TB
t18_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t18_root["Local<br/>SD root"]:::localStorage
t18_host --- t18_root
end
subgraph titan_19["titan-19"]
direction TB
t19_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
t19_root["Local<br/>SD root"]:::localStorage
t19_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
t19_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
t19_host --- t19_root
t19_host --- t19_astreae
t19_host --- t19_asteria
end
end
end
subgraph accelerator_workers["Accelerators and high-capacity workers"]
direction TB
subgraph jetson_workers["ARM64 NVIDIA Jetson"]
direction LR
subgraph titan_20["titan-20"]
direction TB
t20_host["Jetson Xavier NX<br/>16 GiB RAM<br/>Ubuntu 20.04<br/>Accelerator"]:::accelerator
t20_gpu["GPU<br/>NVIDIA Xavier"]:::gpu
t20_storage["Local<br/>256 GiB disk"]:::localStorage
t20_host --- t20_gpu
t20_host --- t20_storage
end
subgraph titan_21["titan-21"]
direction TB
t21_host["Jetson Xavier NX<br/>16 GiB RAM<br/>Ubuntu 20.04<br/>Accelerator"]:::accelerator
t21_gpu["GPU<br/>NVIDIA Xavier"]:::gpu
t21_storage["Local<br/>256 GiB disk"]:::localStorage
t21_host --- t21_gpu
t21_host --- t21_storage
end
end
subgraph x86_workers["x86_64 systems"]
direction LR
subgraph titan_22["titan-22"]
direction TB
t22_host["X830 mini PC<br/>Core i9-12900H<br/>32 GiB RAM<br/>Debian 13"]:::accelerator
t22_gpu["GPU<br/>NVIDIA RTX 3050 Ti"]:::gpu
t22_igpu["Intel Iris Xe<br/>Not pooled"]:::localGpu
t22_storage["Local<br/>1 TiB disk"]:::localStorage
t22_host --- t22_gpu
t22_host --- t22_igpu
t22_host --- t22_storage
end
subgraph titan_23["titan-23, Oceanus"]
direction TB
t23_host["EPYC 74F3<br/>24 cores, 48 threads<br/>256 GiB RAM<br/>Debian 13"]:::server
t23_gpu["ASPEED graphics<br/>Not pooled"]:::localGpu
t23_root["Local<br/>1 TiB disk"]:::localStorage
t23_data["Local<br/>4 TiB Veles<br/>4 TiB SUI"]:::localStorage
t23_host --- t23_gpu
t23_host --- t23_root
t23_host --- t23_data
end
subgraph titan_24["titan-24, Tethys"]
direction TB
t24_host["Ryzen 9 3900X<br/>24 threads, 64 GiB RAM<br/>Debian 13"]:::accelerator
t24_gpu["GPU<br/>NVIDIA RTX 3080"]:::gpu
t24_root["Local<br/>500 GiB disk"]:::localStorage
t24_data["Local<br/>500 GiB home<br/>500 GiB temporary"]:::localStorage
t24_host --- t24_gpu
t24_host --- t24_root
t24_host --- t24_data
end
end
end
pi5_workers ~~~ pi4_workers
pi4_workers ~~~ accelerator_workers
end
subgraph virtual_resources["Virtual resource pools"]
direction LR
subgraph cluster_storage["Longhorn storage classes"]
direction LR
astreae["Astreae<br/>4 x 8 TiB<br/>32 TiB"]:::sharedPool
asteria["Asteria<br/>4 x 12 TiB<br/>48 TiB"]:::sharedPool
end
subgraph cluster_gpus["Kubernetes GPU pool"]
direction LR
arm_gpu_pool["NVIDIA ARM64<br/>titan-20 and titan-21"]:::gpuPool
x86_gpu_pool["NVIDIA x86_64<br/>titan-22 and titan-24"]:::gpuPool
end
end
subgraph cluster_telemetry["Cluster telemetry"]
direction LR
typhon["Typhon<br/>Kubernetes workload"]:::clusterService
victoria["VictoriaMetrics"]:::telemetry
grafana["Grafana"]:::telemetry
typhon ==>|environment metrics| victoria
victoria --> grafana
end
end
subgraph outside_cluster["Lab systems outside the Atlas cluster"]
direction TB
subgraph support_hosts["Support and host-level services"]
direction LR
subgraph titan_db["titan-db"]
direction TB
tdb_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::support
tdb_storage["Local<br/>500 GiB boot SSD"]:::localStorage
tdb_postgres["PostgreSQL<br/>HA control-plane database"]:::hostSoftware
ananke_db["NUT and Ananke<br/>Host services<br/>Automatic recovery"]:::hostService
tdb_host --- tdb_storage
tdb_host --- tdb_postgres
tdb_host --- ananke_db
end
subgraph titan_jh["titan-jh"]
direction TB
tjh_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Arch Linux ARM<br/>Jump host"]:::support
tjh_storage["Local<br/>256 GiB boot SSD"]:::localStorage
lesavka["Lesavka<br/>Remote control for Tethys"]:::hostService
tjh_host --- tjh_storage
tjh_host --- lesavka
end
ananke_t24["NUT and Ananke on titan-24<br/>Host services outside Kubernetes<br/>UPS telemetry and safe shutdown"]:::hostService
end
subgraph failed_hardware["Boards awaiting replacement"]
direction LR
titan_09["titan-09<br/>Raspberry Pi 5"]:::failed
titan_10["titan-10<br/>Raspberry Pi 5"]:::failed
titan_16["titan-16<br/>Raspberry Pi 4"]:::failed
end
end
subgraph facilities["Power and enclosure"]
direction LR
subgraph protected_power["UPS protection"]
direction LR
subgraph pyrphoros_power["Pyrphoros"]
direction TB
pyrphoros["CyberPower<br/>850 VA UPS"]:::power
pyrphoros_loads["Protected loads<br/>Router and Cisco switch<br/>Control plane and Pi workers<br/>Jetsons, titan-22, titan-db<br/>Cooling equipment"]:::powerLoad
pyrphoros --> pyrphoros_loads
end
subgraph statera_power["Statera"]
direction TB
statera["CyberPower<br/>1500PFCLCD"]:::power
statera_loads["Protected loads<br/>TRENDnet switch<br/>titan-23, titan-24, titan-jh"]:::powerLoad
statera --> statera_loads
end
end
subgraph cooling["Enclosure cooling"]
direction TB
enclosure["Filtered equipment tent<br/>AC-cooled intake<br/>Controlled exhaust"]:::environment
ac_controller["AC Infinity controller<br/>Temperature<br/>Humidity<br/>Pressure"]:::environment
fans["Controlled intake<br/>and exhaust fans"]:::environment
enclosure --- ac_controller
ac_controller --> fans
end
end
cisco ==> control_plane
cisco ==> pi5_workers
cisco ==> pi4_workers
cisco ==> jetson_workers
cisco ==> titan_db
trendnet ==> titan_jh
trendnet ==> x86_workers
t0b_host -.->|HA PostgreSQL| tdb_postgres
tjh_host -.->|administration| control_plane
tjh_host -.->|administration| workers
lesavka -.->|remote control| t24_host
t13_astreae ==> astreae
t15_astreae ==> astreae
t17_astreae ==> astreae
t19_astreae ==> astreae
t13_asteria ==> asteria
t15_asteria ==> asteria
t17_asteria ==> asteria
t19_asteria ==> asteria
t20_gpu ==> arm_gpu_pool
t21_gpu ==> arm_gpu_pool
t22_gpu ==> x86_gpu_pool
t24_gpu ==> x86_gpu_pool
t24_host -.->|same physical host| ananke_t24
pyrphoros -->|NUT| ananke_db
statera -->|NUT| ananke_t24
ananke_db -.->|power metrics| victoria
ananke_t24 -.->|power metrics| victoria
ananke_db -.->|database and cluster recovery| t0a_host
ananke_t24 -.->|low-battery shutdown| control_plane
ac_controller --> ac_cloud
ac_cloud -.->|environment data| typhon
end
internet --> router
classDef external fill:#292d33,stroke:#a0a8b3,color:#ffffff,stroke-width:2px
classDef network fill:#17324a,stroke:#65b5e8,color:#ffffff,stroke-width:2px
classDef control fill:#163e48,stroke:#65d2df,color:#ffffff,stroke-width:2px
classDef pi5 fill:#173d2a,stroke:#68d391,color:#ffffff,stroke-width:2px
classDef pi4 fill:#293b23,stroke:#9acb70,color:#ffffff,stroke-width:2px
classDef accelerator fill:#382b50,stroke:#b49aef,color:#ffffff,stroke-width:2px
classDef server fill:#3c3046,stroke:#c4a4dc,color:#ffffff,stroke-width:2px
classDef support fill:#293845,stroke:#9fc5dc,color:#ffffff,stroke-width:2px
classDef localStorage fill:#1d3040,stroke:#6f9fbd,color:#ffffff
classDef sharedStorage fill:#493b18,stroke:#efc65b,color:#ffffff,stroke-width:2px
classDef sharedPool fill:#594817,stroke:#ffd166,color:#ffffff,stroke-width:3px
classDef gpu fill:#482c5f,stroke:#d0a6ff,color:#ffffff,stroke-width:2px
classDef localGpu fill:#292d35,stroke:#9ba4b2,color:#ffffff
classDef gpuPool fill:#573476,stroke:#d8b4fe,color:#ffffff,stroke-width:3px
classDef clusterService fill:#153c38,stroke:#5fc9bc,color:#ffffff,stroke-width:2px
classDef telemetry fill:#203651,stroke:#67a9e9,color:#ffffff,stroke-width:2px
classDef hostSoftware fill:#243a45,stroke:#73b8d0,color:#ffffff,stroke-width:2px
classDef hostService fill:#48351c,stroke:#e3ae59,color:#ffffff,stroke-width:2px
classDef failed fill:#3d1f25,stroke:#f06b78,color:#ffffff,stroke-width:2px,stroke-dasharray:8 6
classDef power fill:#49301c,stroke:#ec9f55,color:#ffffff,stroke-width:2px
classDef powerLoad fill:#3b2d20,stroke:#c98e55,color:#ffffff
classDef environment fill:#153d3a,stroke:#58c9bd,color:#ffffff,stroke-width:2px
classDef legend1g fill:#0b1726,stroke:#4ea1ff,color:#ffffff,stroke-width:3px
classDef legend25g fill:#082027,stroke:#22d3ee,color:#ffffff,stroke-width:3px
classDef legendExternal fill:#17162c,stroke:#818cf8,color:#ffffff,stroke-width:3px
classDef legendLocal fill:#171b20,stroke:#a8b3c2,color:#ffffff,stroke-width:3px
classDef legendShared fill:#2f270d,stroke:#f6c453,color:#ffffff,stroke-width:3px
classDef legendGpu fill:#271536,stroke:#c084fc,color:#ffffff,stroke-width:3px
classDef legendPower fill:#301b0c,stroke:#fb923c,color:#ffffff,stroke-width:3px
classDef legendTelemetry fill:#0b2917,stroke:#4ade80,color:#ffffff,stroke-width:3px
classDef legendControl fill:#32132a,stroke:#f472b6,color:#ffffff,stroke-width:3px
classDef legendSafety fill:#321217,stroke:#fb7185,color:#ffffff,stroke-width:3px
style lab fill:#020202,stroke:#89919c,stroke-width:4px
style link_legend fill:#050505,stroke:#89919c,stroke-width:2px
style legend_network fill:#03070b,stroke:#4ea1ff,stroke-width:1px
style legend_resources fill:#080706,stroke:#f6c453,stroke-width:1px
style legend_operations fill:#080506,stroke:#f472b6,stroke-width:1px
style network fill:#03070b,stroke:#65b5e8,stroke-width:3px
style atlas fill:#020608,stroke:#65d2df,stroke-width:4px
style control_plane fill:#061014,stroke:#65d2df,stroke-width:3px
style workers fill:#030603,stroke:#72b879,stroke-width:3px
style pi5_workers fill:#050d08,stroke:#68d391,stroke-width:2px
style pi5_row_a fill:transparent,stroke:transparent
style pi5_row_b fill:transparent,stroke:transparent
style pi4_workers fill:#080d05,stroke:#9acb70,stroke-width:2px
style pi4_row_a fill:transparent,stroke:transparent
style pi4_row_b fill:transparent,stroke:transparent
style accelerator_workers fill:#09060d,stroke:#b49aef,stroke-width:2px
style jetson_workers fill:#0b0710,stroke:#b49aef,stroke-width:2px
style x86_workers fill:#0b0710,stroke:#c4a4dc,stroke-width:2px
style virtual_resources fill:#08080b,stroke:#a8a2b3,stroke-width:3px
style cluster_storage fill:#100d05,stroke:#ffd166,stroke-width:2px
style cluster_gpus fill:#0d0813,stroke:#d8b4fe,stroke-width:2px
style cluster_telemetry fill:#04100f,stroke:#5fc9bc,stroke-width:2px
style outside_cluster fill:#07090b,stroke:#9fc5dc,stroke-width:3px
style support_hosts fill:#07090b,stroke:#9fc5dc,stroke-width:2px
style failed_hardware fill:#0d0708,stroke:#f06b78,stroke-width:2px,stroke-dasharray:8 6
style facilities fill:#070604,stroke:#ec9f55,stroke-width:3px
style protected_power fill:#0e0905,stroke:#ec9f55,stroke-width:2px
style pyrphoros_power fill:#100b06,stroke:#d99a58,stroke-width:2px
style statera_power fill:#100b06,stroke:#d99a58,stroke-width:2px
style cooling fill:#04100f,stroke:#58c9bd,stroke-width:2px
linkStyle 0,54,55,56,57,58,59 stroke:#4ea1ff,color:#4ea1ff,stroke-width:3px
linkStyle 1,60 stroke:#22d3ee,color:#22d3ee,stroke-width:4px
linkStyle 86 stroke:#818cf8,color:#818cf8,stroke-width:3px
linkStyle 2,3,4,7,8,9,10,11,12,13,14,17,18,21,24,25,29,31,34,36,37,39,40,45,48 stroke:#a8b3c2,color:#a8b3c2,stroke-width:2px
linkStyle 15,16,19,20,22,23,26,27,65,66,67,68,69,70,71,72 stroke:#f6c453,color:#f6c453,stroke-width:3px
linkStyle 28,30,32,33,35,38,73,74,75,76 stroke:#c084fc,color:#c084fc,stroke-width:3px
linkStyle 50,51,78,79 stroke:#fb923c,color:#fb923c,stroke-width:3px
linkStyle 43,44,52,53,80,81,84,85 stroke:#4ade80,color:#4ade80,stroke-width:3px
linkStyle 5,6,46,47,49,61,62,63,64,77,82 stroke:#f472b6,color:#f472b6,stroke-width:2px
linkStyle 83 stroke:#fb7185,color:#fb7185,stroke-width:4px

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 724 KiB

View File

@ -1,705 +0,0 @@
%% Atlas service access architecture.
%% Domains, identity profiles, entry services, and selected internal dependencies.
%% Node placement and short labels carry the detail.
%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "LINEAR_SEGMENTS", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000", "lineColor": "#8b95a5", "primaryTextColor": "#ffffff", "clusterBkg": "#03040d", "clusterBorder": "#6d7485"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
flowchart TB
subgraph system["Atlas Service Access Architecture"]
direction TB
subgraph legend["Legend"]
direction LR
subgraph node_key["Nodes"]
direction LR
key_domain["DNS name"]:::domain
key_access["OIDC access"]:::access
key_oauth["Access proxy"]:::oauth
key_directory["LDAP or synced account"]:::directory
key_entry["Entry service"]:::user
key_internal["Internal service"]:::support
key_data["Data service"]:::data
key_job["Job or worker"]:::ephemeral
key_external["Outside Kubernetes"]:::external
end
subgraph link_key["Links"]
direction LR
key_ingress["Blue<br/>Domain to identity"]:::linkIngress
key_service["Green<br/>Identity to service"]:::linkService
key_direct["Gold<br/>Direct access"]:::linkDirect
key_auth["Purple<br/>Identity control"]:::linkAuth
key_data_link["Violet<br/>Data flow"]:::linkData
key_telemetry["Teal<br/>Telemetry"]:::linkTelemetry
key_control["Orange<br/>Delivery and control"]:::linkControl
key_internal_link["Gray<br/>Internal flow"]:::linkInternal
end
end
subgraph application_band["Applications and Communications"]
direction LR
subgraph application_domains["Domains"]
direction TB
portal_domain["bstein.dev"]:::domain
portal_chat_domain["chat.ai.bstein.dev"]:::domain
cassandra_domain["cassandra.bstein.dev"]:::domain
element_domain["live.bstein.dev"]:::domain
matrix_domain["matrix.live.bstein.dev"]:::domain
call_domain["call.live.bstein.dev"]:::domain
livekit_domain["kit.live.bstein.dev"]:::domain
turn_domain["turn.live.bstein.dev"]:::domain
end
subgraph application_access["Keycloak Access"]
direction LR
subgraph application_atlas_access["atlas realm"]
direction TB
portal_oidc_access["Portal API<br/>OIDC token"]:::access
portal_chat_oidc_access["Portal Chat<br/>OIDC"]:::access
element_oidc_access["Element Web<br/>OIDC through MAS"]:::access
matrix_oidc_access["Matrix<br/>OIDC through MAS"]:::access
end
subgraph application_cassandra_access["cassandra realm"]
direction TB
cassandra_oidc_access["Cassandra<br/>OIDC"]:::access
end
end
subgraph application_services["Cluster Services"]
direction TB
application_delivery["Flux delivery<br/>Application workloads"]:::control
application_state_clients["Shared PostgreSQL<br/>Portal and Matrix"]:::control
subgraph website_ns["bstein-dev-home"]
direction LR
website_frontend["Portal"]:::user
website_chat_gateway["Chat gateway"]:::user
website_backend["Backend"]:::support
website_frontend --> website_backend
website_chat_gateway --> website_backend
end
subgraph cassandra_ns["cassandra"]
direction LR
cassandra_frontend["Cassandra UI"]:::user
cassandra_backend["API"]:::support
cassandra_postgres["PostgreSQL"]:::data
cassandra_generator["Generator"]:::ephemeral
cassandra_simulation["Simulation jobs"]:::ephemeral
cassandra_retention["Retention job"]:::ephemeral
cassandra_artifacts["Artifacts"]:::data
cassandra_frontend --> cassandra_backend
cassandra_backend --> cassandra_postgres
cassandra_generator --> cassandra_backend
cassandra_simulation --> cassandra_backend
cassandra_simulation --> cassandra_artifacts
cassandra_retention --> cassandra_artifacts
end
subgraph comms_ns["comms"]
direction LR
comms_element_web["Element Web"]:::user
comms_element_call["Element Call"]:::user
comms_wellknown["Matrix discovery"]:::support
comms_guest["Guest registration"]:::support
comms_mas["Matrix auth service"]:::support
comms_synapse["Synapse"]:::support
comms_replication["Synapse replication"]:::support
comms_redis["Redis"]:::data
comms_livekit_token["LiveKit token service"]:::support
comms_livekit["LiveKit"]:::support
comms_coturn["Coturn"]:::support
comms_atlasbot["Atlasbot"]:::support
comms_element_web --> comms_mas
comms_element_web --> comms_synapse
comms_element_call --> comms_livekit_token
comms_livekit_token --> comms_livekit
comms_mas --> comms_synapse
comms_synapse --> comms_replication
comms_synapse --> comms_redis
comms_guest --> comms_synapse
comms_atlasbot --> comms_synapse
end
end
portal_domain portal_identity@-->|HTTPS| portal_oidc_access
portal_oidc_access portal_entry@--> website_frontend
portal_chat_domain portal_chat_identity@-->|HTTPS| portal_chat_oidc_access
portal_chat_oidc_access portal_chat_entry@--> website_chat_gateway
cassandra_domain cassandra_identity@-->|HTTPS| cassandra_oidc_access
cassandra_oidc_access cassandra_entry@--> cassandra_frontend
element_domain element_identity@-->|HTTPS| element_oidc_access
element_oidc_access element_entry@--> comms_element_web
matrix_domain matrix_identity@-->|HTTPS| matrix_oidc_access
matrix_oidc_access matrix_entry@--> comms_synapse
matrix_domain matrix_api@-->|Matrix API| comms_synapse
call_domain call_entry@-->|HTTPS| comms_element_call
livekit_domain livekit_entry@-->|HTTPS and WebRTC| comms_livekit
turn_domain turn_entry@-->|TURN| comms_coturn
end
subgraph personal_band["Personal Services"]
direction LR
subgraph personal_domains["Domains"]
direction LR
subgraph personal_domains_a["Accounts and Health"]
direction TB
budget_domain["budget.bstein.dev"]:::domain
firefly_domain["money.bstein.dev"]:::domain
wolf_domain["wolf.bstein.dev"]:::domain
moonlight_domain["moonlight.bstein.dev"]:::domain
wger_domain["health.bstein.dev"]:::domain
chat_domain["chat.bstein.dev"]:::domain
end
subgraph personal_domains_b["Media and Productivity"]
direction TB
jellyfin_domain["stream.bstein.dev"]:::domain
pegasus_domain["pegasus.bstein.dev"]:::domain
mailu_domain["mail.bstein.dev"]:::domain
cloud_domain["cloud.bstein.dev"]:::domain
office_domain["office.bstein.dev"]:::domain
outline_domain["notes.bstein.dev"]:::domain
planka_domain["tasks.bstein.dev"]:::domain
vaultwarden_domain["vault.bstein.dev"]:::domain
monero_domain["monero.bstein.dev"]:::domain
end
end
subgraph personal_access["atlas realm and directory access"]
direction LR
subgraph personal_access_a["Accounts"]
direction TB
budget_oidc_access["Actual Budget<br/>OpenID"]:::access
firefly_synced_access["Firefly<br/>Synced account"]:::directory
wolf_oauth_access["Wolf<br/>Access proxy"]:::oauth
wger_synced_access["Wger<br/>Synced account"]:::directory
chat_oidc_access["Hermes Chat<br/>OIDC"]:::access
end
subgraph personal_access_b["Media and Productivity"]
direction TB
jellyfin_ldap_access["Jellyfin<br/>LDAP"]:::directory
cloud_oidc_access["Nextcloud<br/>OIDC"]:::access
outline_oidc_access["Outline<br/>OIDC"]:::access
planka_oidc_access["Planka<br/>OIDC"]:::access
end
end
subgraph personal_services["Cluster Services"]
direction LR
personal_delivery["Flux delivery<br/>Personal workloads"]:::control
personal_state_clients["Shared PostgreSQL<br/>Personal services"]:::control
subgraph personal_services_a["Accounts, Health, and Media"]
direction TB
subgraph finance_ns["finance"]
direction LR
finance_budget["Actual Budget"]:::user
finance_firefly["Firefly"]:::user
finance_sync["Account sync"]:::ephemeral
finance_import["Import job"]:::ephemeral
finance_sync --> finance_firefly
finance_import --> finance_firefly
end
subgraph game_ns["game-stream"]
direction LR
game_proxy["Wolf access proxy"]:::oauth
game_manager["Wolf manager"]:::support
game_api["Wolf API"]:::support
game_wolf["Wolf"]:::support
game_gatekeeper["Gatekeeper"]:::support
game_moonlight["Moonlight"]:::user
game_proxy --> game_manager
game_manager --> game_api
game_api --> game_wolf
game_gatekeeper --> game_wolf
game_moonlight --> game_wolf
end
subgraph health_ns["health"]
direction LR
health_wger["Wger"]:::user
health_sync["Account sync"]:::ephemeral
health_admin["Admin sync"]:::ephemeral
health_sync --> health_wger
health_admin --> health_wger
end
subgraph hermes_chat_ns["hermes-chat"]
direction LR
hermes_chat["Hermes Chat"]:::user
end
subgraph jellyfin_ns["jellyfin"]
direction LR
streaming_pegasus["Pegasus"]:::user
streaming_jellyfin["Jellyfin"]:::user
streaming_pegasus --> streaming_jellyfin
end
end
subgraph personal_services_b["Mail and Productivity"]
direction TB
subgraph mail_ns["mailu-mailserver"]
direction LR
mail_front["Mail web"]:::user
mail_protocols["Mail protocols"]:::user
mail_admin["Admin"]:::support
mail_dovecot["Dovecot"]:::support
mail_postfix["Postfix"]:::support
mail_rspamd["Rspamd"]:::support
mail_clamav["ClamAV"]:::support
mail_extractors["Tika and oletools"]:::support
mail_redis["Redis"]:::data
mail_front --> mail_admin
mail_protocols --> mail_dovecot
mail_protocols --> mail_postfix
mail_postfix --> mail_rspamd
mail_rspamd --> mail_clamav
mail_rspamd --> mail_extractors
mail_admin --> mail_redis
end
subgraph cloud_ns["nextcloud"]
direction LR
cloud_nextcloud["Nextcloud"]:::user
cloud_collabora["Collabora"]:::support
cloud_mail_sync["Mail sync"]:::ephemeral
cloud_maintenance["Maintenance job"]:::ephemeral
cloud_nextcloud --> cloud_collabora
cloud_mail_sync --> cloud_nextcloud
cloud_maintenance --> cloud_nextcloud
end
subgraph notes_tasks_ns["Notes and Tasks"]
direction LR
outline_service["Outline"]:::user
outline_redis["Redis"]:::data
planka_service["Planka"]:::user
outline_service --> outline_redis
end
subgraph vaultwarden_ns["vaultwarden"]
direction LR
vaultwarden_service["Vaultwarden"]:::user
end
subgraph crypto_ns["crypto"]
direction LR
crypto_wallet["Wallet RPC"]:::support
crypto_p2pool["P2Pool"]:::support
crypto_monerod["Monerod"]:::user
crypto_wallet --> crypto_monerod
crypto_p2pool --> crypto_monerod
end
end
end
budget_domain budget_identity@-->|HTTPS| budget_oidc_access
budget_oidc_access budget_entry@--> finance_budget
firefly_domain firefly_identity@-->|HTTPS| firefly_synced_access
firefly_synced_access firefly_entry@--> finance_firefly
wolf_domain wolf_identity@-->|HTTPS| wolf_oauth_access
wolf_oauth_access wolf_entry@--> game_manager
moonlight_domain moonlight_entry@-->|Streaming| game_moonlight
wger_domain wger_identity@-->|HTTPS| wger_synced_access
wger_synced_access wger_entry@--> health_wger
chat_domain chat_identity@-->|HTTPS| chat_oidc_access
chat_oidc_access chat_entry@--> hermes_chat
jellyfin_domain jellyfin_identity@-->|HTTPS| jellyfin_ldap_access
jellyfin_ldap_access jellyfin_entry@--> streaming_jellyfin
pegasus_domain pegasus_entry@-->|HTTPS| streaming_pegasus
mailu_domain mail_web_entry@-->|HTTPS| mail_front
mailu_domain mail_protocol_entry@-->|Mail protocols| mail_protocols
cloud_domain cloud_identity@-->|HTTPS| cloud_oidc_access
cloud_oidc_access cloud_entry@--> cloud_nextcloud
office_domain office_entry@-->|WOPI| cloud_collabora
outline_domain outline_identity@-->|HTTPS| outline_oidc_access
outline_oidc_access outline_entry@--> outline_service
planka_domain planka_identity@-->|HTTPS| planka_oidc_access
planka_oidc_access planka_entry@--> planka_service
vaultwarden_domain vaultwarden_entry@-->|HTTPS and local auth| vaultwarden_service
monero_domain monero_entry@-->|RPC| crypto_monerod
end
subgraph platform_band["Platform Services"]
direction LR
subgraph platform_domains["Domains"]
direction LR
subgraph platform_domains_a["Delivery"]
direction TB
gitops_domain["cd.bstein.dev"]:::domain
gitea_domain["scm.bstein.dev"]:::domain
harbor_domain["registry.bstein.dev"]:::domain
hermes_domain["agent.bstein.dev"]:::domain
jenkins_domain["ci.bstein.dev"]:::domain
end
subgraph platform_domains_b["Operations"]
direction TB
logs_domain["logs.bstein.dev"]:::domain
longhorn_domain["longhorn.bstein.dev"]:::domain
grafana_domain["metrics.bstein.dev"]:::domain
alerts_domain["alerts.bstein.dev"]:::domain
quality_domain["quality.bstein.dev"]:::domain
keycloak_domain["sso.bstein.dev"]:::domain
vault_domain["secret.bstein.dev"]:::domain
end
end
subgraph platform_access["Keycloak Access"]
direction LR
subgraph platform_access_a["atlas realm delivery"]
direction TB
gitops_oidc_access["Weave GitOps<br/>OIDC"]:::access
gitea_atlas_oidc_access["Gitea<br/>atlas realm"]:::access
harbor_oidc_access["Harbor UI<br/>OIDC"]:::access
hermes_oidc_access["Hermes Agent<br/>OIDC"]:::access
jenkins_oidc_access["Jenkins<br/>OIDC"]:::access
end
subgraph platform_access_b["atlas realm operations"]
direction TB
logs_oauth_access["OpenSearch Dashboards<br/>Access proxy"]:::oauth
longhorn_oauth_access["Longhorn<br/>Admin proxy"]:::oauth
grafana_oidc_access["Grafana<br/>OIDC"]:::access
quality_oauth_access["SonarQube<br/>Admin and developer proxy"]:::oauth
keycloak_admin_access["Keycloak console<br/>Realm admin"]:::access
vault_oidc_access["Vault<br/>Admin OIDC"]:::access
end
subgraph platform_access_c["cassandra realm client"]
direction TB
gitea_cassandra_oidc_access["Gitea<br/>OIDC"]:::access
end
end
subgraph platform_services["Cluster Services"]
direction LR
platform_delivery["Flux delivery<br/>Platform workloads"]:::control
platform_state_clients["Shared PostgreSQL<br/>Platform services"]:::control
subgraph delivery_services["Delivery and Automation"]
direction TB
subgraph flux_ns["flux-system"]
direction LR
flux_source["Source controller"]:::support
flux_kustomize["Kustomize controller"]:::support
flux_helm["Helm controller"]:::support
flux_image_reflector["Image reflector"]:::support
flux_image_automation["Image automation"]:::support
flux_notification["Notification controller"]:::support
flux_webhook["Webhook receiver"]:::support
flux_weave["Weave GitOps"]:::user
flux_source --> flux_kustomize
flux_source --> flux_helm
flux_image_reflector --> flux_image_automation
flux_webhook --> flux_notification
flux_weave --> flux_source
end
subgraph gitea_ns["gitea"]
direction LR
gitea_http["Gitea web"]:::user
gitea_ssh["Gitea SSH"]:::support
end
subgraph harbor_ns["harbor"]
direction LR
harbor_portal["Harbor portal"]:::user
harbor_core["Harbor core"]:::support
harbor_jobservice["Job service"]:::support
harbor_registry["OCI registry"]:::data
harbor_redis["Redis"]:::data
harbor_portal --> harbor_core
harbor_jobservice --> harbor_core
harbor_core --> harbor_registry
harbor_core --> harbor_redis
end
subgraph hermes_ns["hermes"]
direction LR
hermes_agent["Hermes Agent"]:::user
hermes_model_gate["Model gate"]:::support
hermes_gpt_oss["gpt-oss:20b<br/>Fallback model"]:::support
hermes_agent --> hermes_model_gate
hermes_model_gate --> hermes_gpt_oss
end
subgraph jenkins_ns["jenkins"]
direction LR
jenkins_service["Jenkins"]:::user
jenkins_agents["Build agents"]:::ephemeral
jenkins_service --> jenkins_agents
end
end
subgraph operations_services["Operations"]
direction TB
subgraph logging_ns["logging"]
direction LR
logging_oauth_proxy["Dashboard proxy"]:::oauth
logging_dashboards["OpenSearch Dashboards"]:::user
logging_opensearch["OpenSearch"]:::data
logging_fluent_bit["Fluent Bit"]:::support
logging_otel["OpenTelemetry"]:::support
logging_data_prepper["Data Prepper"]:::support
logging_oauth_proxy --> logging_dashboards
logging_dashboards --> logging_opensearch
logging_fluent_bit --> logging_opensearch
logging_otel --> logging_data_prepper
logging_data_prepper --> logging_opensearch
end
subgraph longhorn_ns["longhorn-system"]
direction LR
longhorn_oauth_proxy["Longhorn proxy"]:::oauth
longhorn_frontend["Longhorn UI"]:::user
longhorn_backend["Longhorn manager"]:::data
longhorn_webhooks["Admission and recovery"]:::support
longhorn_oauth_proxy --> longhorn_frontend
longhorn_frontend --> longhorn_backend
longhorn_webhooks --> longhorn_backend
end
subgraph monitoring_ns["monitoring"]
direction LR
monitoring_grafana["Grafana"]:::user
monitoring_victoria["VictoriaMetrics"]:::data
monitoring_exporters["Cluster and node exporters"]:::support
monitoring_gpu_exporters["GPU exporters"]:::support
monitoring_quality_gateway["Quality gateway"]:::support
monitoring_postmark["Postmark exporter"]:::support
monitoring_vmalert["vmalert"]:::support
monitoring_alertmanager["Alertmanager"]:::user
monitoring_exporters --> monitoring_victoria
monitoring_gpu_exporters --> monitoring_victoria
monitoring_quality_gateway --> monitoring_victoria
monitoring_postmark --> monitoring_victoria
monitoring_victoria --> monitoring_grafana
monitoring_vmalert --> monitoring_alertmanager
end
end
subgraph security_services["Identity and Security"]
direction TB
subgraph quality_ns["quality"]
direction LR
quality_oauth_proxy["SonarQube proxy"]:::oauth
quality_sonarqube["SonarQube"]:::user
quality_exporter["Quality exporter"]:::support
quality_oauth_proxy --> quality_sonarqube
quality_exporter --> quality_sonarqube
end
subgraph sso_ns["sso"]
direction LR
sso_oauth_proxy["Shared access proxy"]:::oauth
sso_keycloak["Keycloak"]:::user
atlas_realm["atlas realm"]:::control
cassandra_realm["cassandra realm"]:::control
sso_openldap["OpenLDAP"]:::directory
sso_jobs["Realm bootstrap and checks"]:::ephemeral
sso_keycloak --> atlas_realm
sso_keycloak --> cassandra_realm
sso_jobs --> sso_keycloak
end
subgraph vault_ns["vault"]
direction LR
vault_service["Vault"]:::user
vault_internal["Vault cluster service"]:::support
vault_injector["Secret injector"]:::support
vault_jobs["Configuration jobs"]:::ephemeral
vault_service --> vault_internal
vault_injector --> vault_service
vault_jobs --> vault_service
end
end
end
gitops_domain gitops_identity@-->|HTTPS| gitops_oidc_access
gitops_oidc_access gitops_entry@--> flux_weave
gitea_domain gitea_atlas_identity@-->|HTTPS| gitea_atlas_oidc_access
gitea_atlas_oidc_access gitea_atlas_entry@--> gitea_http
gitea_domain gitea_cassandra_identity@-->|HTTPS| gitea_cassandra_oidc_access
gitea_cassandra_oidc_access gitea_cassandra_entry@--> gitea_http
gitea_domain gitea_ssh_entry@-->|SSH| gitea_ssh
harbor_domain harbor_identity@-->|HTTPS| harbor_oidc_access
harbor_oidc_access harbor_entry@--> harbor_portal
harbor_domain harbor_registry_entry@-->|OCI| harbor_registry
hermes_domain hermes_identity@-->|HTTPS| hermes_oidc_access
hermes_oidc_access hermes_entry@--> hermes_agent
jenkins_domain jenkins_identity@-->|HTTPS| jenkins_oidc_access
jenkins_oidc_access jenkins_entry@--> jenkins_service
logs_domain logs_identity@-->|HTTPS| logs_oauth_access
logs_oauth_access logs_entry@--> logging_oauth_proxy
longhorn_domain longhorn_identity@-->|HTTPS| longhorn_oauth_access
longhorn_oauth_access longhorn_entry@--> longhorn_oauth_proxy
grafana_domain grafana_identity@-->|HTTPS| grafana_oidc_access
grafana_oidc_access grafana_entry@--> monitoring_grafana
alerts_domain alerts_entry@-->|HTTPS| monitoring_alertmanager
quality_domain quality_identity@-->|HTTPS| quality_oauth_access
quality_oauth_access quality_entry@--> quality_oauth_proxy
keycloak_domain keycloak_identity@-->|HTTPS| keycloak_admin_access
keycloak_admin_access keycloak_entry@--> sso_keycloak
vault_domain vault_identity@-->|HTTPS| vault_oidc_access
vault_oidc_access vault_entry@--> vault_service
end
subgraph shared_plane["Shared Cluster Services"]
direction LR
subgraph ingress_plane["Ingress and Networking"]
direction TB
metallb_control["MetalLB<br/>Service addresses"]:::control
traefik_service["Traefik<br/>Load balancer"]:::support
traefik_controller["Traefik<br/>TLS and routing"]:::support
cert_manager["cert-manager"]:::support
cert_webhooks["Certificate webhooks"]:::support
metallb_control --> traefik_service
traefik_service --> traefik_controller
cert_webhooks --> cert_manager
cert_manager --> traefik_controller
end
subgraph shared_state["Shared State and Models"]
direction TB
postgres_service["Shared PostgreSQL<br/>Application and platform databases"]:::data
ai_ollama["Ollama<br/>Local model service"]:::support
default_zot_proxy["Zot access proxy"]:::oauth
end
subgraph maintenance_ns["maintenance"]
direction TB
maintenance_ariadne["Ariadne"]:::support
maintenance_metis_proxy["Metis proxy"]:::oauth
maintenance_metis["Metis"]:::support
maintenance_soteria_proxy["Soteria proxy"]:::oauth
maintenance_soteria["Soteria"]:::support
maintenance_node_ops["Node operations"]:::ephemeral
maintenance_jobs["Repair and migration jobs"]:::ephemeral
maintenance_metis_proxy --> maintenance_metis
maintenance_soteria_proxy --> maintenance_soteria
maintenance_ariadne --> maintenance_node_ops
maintenance_ariadne --> maintenance_jobs
end
subgraph telemetry_services["Additional Telemetry"]
direction TB
climate_typhon["Typhon<br/>Tent climate"]:::support
sui_metrics["SUI metrics"]:::support
end
end
subgraph external_systems["Host and Physical Systems"]
direction LR
ananke_hosts["Ananke<br/>titan-db and titan-24"]:::external
pyrphoros["Pyrphoros UPS"]:::external
statera["Statera UPS"]:::external
tent_system["AC Infinity<br/>Controller and cloud"]:::external
ananke_hosts --> pyrphoros
ananke_hosts --> statera
end
firefly_synced_access firefly_directory@-.-> sso_openldap
wger_synced_access wger_directory@-.-> sso_openldap
jellyfin_ldap_access jellyfin_directory@-.-> sso_openldap
website_backend website_model@-.-> ai_ollama
website_backend website_automation@-.-> maintenance_ariadne
hermes_chat hermes_chat_model@-.-> hermes_model_gate
hermes_agent hermes_jenkins@-.-> jenkins_service
hermes_agent hermes_metrics@-.-> monitoring_victoria
hermes_agent hermes_ariadne@-.-> maintenance_ariadne
gitea_http gitea_jenkins@-.->|SCM webhook| jenkins_service
gitea_http gitea_flux@-.->|Flux webhook| flux_webhook
jenkins_agents jenkins_gitea@-.->|Clone and fetch| gitea_http
jenkins_agents jenkins_quality@-.->|Quality scan| quality_sonarqube
jenkins_agents jenkins_harbor@-.->|Images and artifacts| harbor_registry
flux_source flux_gitea@-.->|Desired state| gitea_ssh
flux_image_reflector flux_harbor@-.->|Image metadata| harbor_registry
flux_image_automation flux_commit@-.->|Manifest update| gitea_ssh
flux_kustomize flux_apps@-.-> application_delivery
flux_kustomize flux_personal@-.-> personal_delivery
flux_kustomize flux_platform@-.-> platform_delivery
application_state_clients app_postgres@-.-> postgres_service
personal_state_clients personal_postgres@-.-> postgres_service
platform_state_clients platform_postgres@-.-> postgres_service
cloud_nextcloud nextcloud_mail@-.->|Mail delivery| mail_protocols
quality_exporter quality_metrics@-.-> monitoring_victoria
sui_metrics sui_telemetry@-.-> monitoring_victoria
monitoring_victoria ananke_telemetry@-.-> ananke_hosts
climate_typhon climate_source@--> tent_system
climate_typhon climate_telemetry@-.-> monitoring_victoria
class portal_identity,portal_chat_identity,cassandra_identity,element_identity,matrix_identity,budget_identity,firefly_identity,wolf_identity,wger_identity,chat_identity,jellyfin_identity,cloud_identity,outline_identity,planka_identity,gitops_identity,gitea_atlas_identity,gitea_cassandra_identity,harbor_identity,hermes_identity,jenkins_identity,logs_identity,longhorn_identity,grafana_identity,quality_identity,keycloak_identity,vault_identity routeIngress
class portal_entry,portal_chat_entry,cassandra_entry,element_entry,matrix_entry,budget_entry,firefly_entry,wolf_entry,wger_entry,chat_entry,jellyfin_entry,cloud_entry,outline_entry,planka_entry,gitops_entry,gitea_atlas_entry,gitea_cassandra_entry,harbor_entry,hermes_entry,jenkins_entry,logs_entry,longhorn_entry,grafana_entry,quality_entry,keycloak_entry,vault_entry routeService
class matrix_api,call_entry,livekit_entry,turn_entry,moonlight_entry,pegasus_entry,mail_web_entry,mail_protocol_entry,office_entry,vaultwarden_entry,monero_entry,gitea_ssh_entry,harbor_registry_entry,alerts_entry routeDirect
class firefly_directory,wger_directory,jellyfin_directory routeAuth
class app_postgres,personal_postgres,platform_postgres,nextcloud_mail,climate_source routeData
class quality_metrics,sui_telemetry,ananke_telemetry,climate_telemetry routeTelemetry
class website_automation,hermes_ariadne,gitea_jenkins,gitea_flux,jenkins_gitea,jenkins_quality,jenkins_harbor,flux_gitea,flux_harbor,flux_commit,flux_apps,flux_personal,flux_platform routeControl
class website_model,hermes_chat_model,hermes_jenkins,hermes_metrics routeInternal
end
classDef domain fill:#16324f,stroke:#6cb6ff,color:#ffffff
classDef user fill:#173f2b,stroke:#61d095,color:#ffffff
classDef support fill:#30343b,stroke:#aab2bf,color:#ffffff
classDef data fill:#3b2f52,stroke:#b79cff,color:#ffffff
classDef ephemeral fill:#4a301e,stroke:#f0a35b,color:#ffffff,stroke-dasharray:6 3
classDef control fill:#3b354e,stroke:#c1a7ff,color:#ffffff
classDef access fill:#213a56,stroke:#6cb6ff,color:#ffffff
classDef oauth fill:#3b2f52,stroke:#c1a7ff,color:#ffffff
classDef directory fill:#344a2d,stroke:#8ed081,color:#ffffff
classDef external fill:#44301f,stroke:#ffb86c,color:#ffffff
classDef linkIngress fill:#101820,stroke:#4ea1ff,color:#ffffff
classDef linkService fill:#101820,stroke:#61d095,color:#ffffff
classDef linkDirect fill:#101820,stroke:#e4bd55,color:#ffffff
classDef linkAuth fill:#101820,stroke:#c1a7ff,color:#ffffff
classDef linkData fill:#101820,stroke:#b79cff,color:#ffffff
classDef linkTelemetry fill:#101820,stroke:#55c8d3,color:#ffffff
classDef linkControl fill:#101820,stroke:#f0a35b,color:#ffffff
classDef linkInternal fill:#101820,stroke:#aab2bf,color:#ffffff
classDef routeIngress stroke:#4ea1ff,stroke-width:2.5px,color:#4ea1ff
classDef routeService stroke:#61d095,stroke-width:2.5px,color:#61d095
classDef routeDirect stroke:#e4bd55,stroke-width:2.5px,color:#e4bd55
classDef routeAuth stroke:#c1a7ff,stroke-width:2.2px,color:#c1a7ff,stroke-dasharray:6 4
classDef routeData stroke:#b79cff,stroke-width:2.2px,color:#b79cff,stroke-dasharray:6 4
classDef routeTelemetry stroke:#55c8d3,stroke-width:2.2px,color:#55c8d3,stroke-dasharray:6 4
classDef routeControl stroke:#f0a35b,stroke-width:2.2px,color:#f0a35b,stroke-dasharray:6 4
classDef routeInternal stroke:#aab2bf,stroke-width:2px,color:#aab2bf,stroke-dasharray:5 4
style system fill:#000000,stroke:#8b95a5,stroke-width:2px
style legend fill:#050505,stroke:#6d7485,stroke-width:1.5px
style application_band fill:#02030d,stroke:#55c8d3,stroke-width:2px
style personal_band fill:#06030d,stroke:#8ed081,stroke-width:2px
style platform_band fill:#07040c,stroke:#c1a7ff,stroke-width:2px
style shared_plane fill:#080604,stroke:#e4bd55,stroke-width:2px
style external_systems fill:#080402,stroke:#ffb86c,stroke-width:2px

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 1.0 MiB

View File

@ -1,213 +0,0 @@
%% Automatic test failure triage and response.
%% Hermes reads and recommends. Ariadne validates and performs every write.
%% Relationships carry the detail; node text stays concise.
%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 32, "rankSpacing": 48, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "LINEAR_SEGMENTS", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
flowchart TD
subgraph system["Hermes Test Failure Triage"]
direction TB
subgraph main_row["Main flow"]
direction TB
subgraph upper_flow["Evidence and analysis"]
direction LR
subgraph reference["Legend"]
direction TB
key_build["Blue<br/>Build and validation"]:::build
key_evidence["Green<br/>Evidence"]:::evidence
key_ariadne["Teal<br/>Ariadne orchestration"]:::ariadne
key_hermes["Purple<br/>Hermes"]:::hermes
key_policy["Gold<br/>Policy and action"]:::policy
key_human["Red<br/>Human required"]:::human
key_limit["Dashed gray<br/>Current boundary"]:::limited
end
subgraph intake["Detect and gather"]
direction TB
detector["Jenkins detector<br/>Allowlisted jobs<br/>Terminal failures only"]:::ariadne
evidence_sources["Evidence sources<br/>Jenkins and OpenSearch<br/>Metrics, Kubernetes, Flux<br/>Gitea and Grafana"]:::evidence
collector["Console reader<br/>Full console<br/>2 MB cap"]:::ariadne
failure_ranker["Failure ranker<br/>Strong evidence first<br/>Tool output alone is neutral"]:::ariadne
context_filter["Context filter<br/>Passing quotes removed<br/>Repeats and overlaps merged"]:::ariadne
bundle["Incident bundle<br/>Failures, logs, metrics<br/>Health, revisions, links"]:::evidence
detector --> evidence_sources
evidence_sources --> collector
collector --> failure_ranker
failure_ranker --> context_filter
context_filter --> bundle
end
subgraph hermes_plane["Hermes analysis"]
direction TB
hermes_access["Read-only access<br/>Jenkins, metrics, Kubernetes<br/>Flux, Gitea, Grafana<br/>Investigation by incident ID"]:::hermes
model_gate["Model gate<br/>openai-codex/gpt-5.6-terra<br/>local gpt-oss:20b fallback"]:::model
skills["Triage skills<br/>Jenkins evidence<br/>Quality and cluster health<br/>Flux and Git correlation"]:::skill
recommendation["Structured recommendation<br/>Classification and confidence<br/>Sourced facts<br/>Action request or patch data"]:::hermes
tool_stream["Agent event stream<br/>Evidence and tool calls"]:::audit
hermes_access --> skills
model_gate -.-> skills
skills --> recommendation
hermes_access -.-> tool_stream
end
end
subgraph lower_flow["Policy and response"]
direction LR
subgraph controls["Ariadne policy gates"]
direction TB
response_check["Response check<br/>Schema, incident, sources<br/>Confidence and action"]:::policy
repair_guard["Scoped repair guard<br/>Classification and action<br/>must match the owning job"]:::policy
marker_check["Transient marker check<br/>DNS, connection, TLS<br/>image pull, upstream 5xx<br/>Disk full is excluded"]:::policy
authorizer["Action authorizer<br/>Allowlist, evidence, confidence<br/>One action, kill switch"]:::policy
response_check -->|scoped repair| repair_guard
repair_guard --> authorizer
response_check -->|transient claim| marker_check
marker_check -->|marker confirmed| authorizer
response_check -->|other result| authorizer
end
subgraph response["Ariadne response"]
direction LR
subgraph response_paths["Response paths"]
direction TB
route{"Policy result"}:::decision
subgraph immediate_paths["Action or escalation"]
direction LR
subgraph operational_path["Allowlisted action"]
direction TB
registry["Action registry"]:::policy
repair_fixture["Scoped ConfigMap repair<br/>In-process patch<br/>No pod created"]:::action
retry_infra["Transient retry<br/>One Jenkins rebuild<br/>No cluster write"]:::action
operational_result["Action result<br/>One attempt<br/>One validation build"]:::action
registry --> repair_fixture
registry --> retry_infra
repair_fixture --> operational_result
retry_infra --> operational_result
end
subgraph escalation_path["Human-required response"]
direction TB
human_required["Diagnosis and next checks<br/>Incident stays human required"]:::human
gitea_issue["Ariadne opens an issue<br/>Evidence and suggested fix"]:::human
human_required --> gitea_issue
end
end
route -->|authorized action| registry
route -->|human required| human_required
operational_result -->|action failed| human_required
end
subgraph code_path["Optional source proposal"]
direction TB
subgraph proposal_stage["Prepare and check"]
direction TB
duplicate_guard["Open proposal check<br/>Existing repair branch"]:::policy
candidate_files["Candidate files<br/>Path and line hints<br/>Read failing test"]:::evidence
source_target["Patch target<br/>Follow test imports<br/>Tests remain read only"]:::policy
patch_proposal["Hermes patch data<br/>Path, anchor, replacement<br/>Reason"]:::hermes
patch_validator["Patch validator<br/>Allowed path and branch<br/>20 lines, 4 KB<br/>Exact anchor"]:::policy
duplicate_guard --> candidate_files
candidate_files --> source_target
source_target --> patch_proposal
patch_proposal --> patch_validator
end
subgraph pr_stage["Deliver for review"]
direction TB
gitea_pr["Ariadne opens a pull request<br/>hermes-repair branch<br/>Ariadne holds the token"]:::git
branch_build["Branch validation<br/>Configured repositories only"]:::limited
human_merge["Human review and merge<br/>No automatic merge"]:::human
flux_delivery["Flux delivery<br/>After human merge"]:::git
gitea_pr -.->|where configured| branch_build
branch_build --> human_merge
gitea_pr -->|otherwise| human_merge
human_merge --> flux_delivery
end
patch_validator -->|validated patch| gitea_pr
end
response_paths -.->|eligible source defect| code_path
end
subgraph outputs["Inspectable outputs"]
direction TB
output_entry["Ariadne records every outcome"]:::audit
audit_events["Audit events<br/>Incident, diagnosis, action<br/>Patch proposal and agent IDs"]:::audit
metrics_output["Triage metrics<br/>VictoriaMetrics and Grafana"]:::metrics
alert_output["Narrow alerts<br/>Failed repair<br/>Escalation untouched for six hours"]:::metrics
issue_output["Gitea issue<br/>Human-required incident<br/>Written by Ariadne"]:::git
pr_output["Gitea pull request<br/>Source proposal<br/>Written by Ariadne"]:::git
output_entry --> audit_events
output_entry --> metrics_output
metrics_output --> alert_output
end
end
reference ~~~ intake
intake ==>|incident bundle| hermes_plane
controls ==>|policy result| response
response ==>|records and artifacts| outputs
upper_flow ==>|recommendation| lower_flow
end
end
classDef build fill:#16324f,stroke:#6cb6ff,color:#ffffff
classDef evidence fill:#173f2b,stroke:#61d095,color:#ffffff
classDef ariadne fill:#24414a,stroke:#69c5d1,color:#ffffff
classDef hermes fill:#3c3155,stroke:#c1a7ff,color:#ffffff
classDef skill fill:#42345b,stroke:#c1a7ff,color:#ffffff
classDef model fill:#342b4f,stroke:#a78bfa,color:#ffffff
classDef policy fill:#3d3525,stroke:#e4bd55,color:#ffffff
classDef decision fill:#47391c,stroke:#e4bd55,color:#ffffff,stroke-width:3px
classDef action fill:#4a3b16,stroke:#e4bd55,color:#ffffff
classDef human fill:#4a2428,stroke:#ff6b78,color:#ffffff
classDef git fill:#253f35,stroke:#78c89a,color:#ffffff
classDef metrics fill:#243750,stroke:#65a9e8,color:#ffffff
classDef audit fill:#353125,stroke:#d9b55b,color:#ffffff
classDef limited fill:#1f2227,stroke:#9ba4b1,color:#c4c9d1,stroke-width:2px,stroke-dasharray:8 6
style system fill:#030303,stroke:#6b7280,stroke-width:3px
style main_row fill:#050607,stroke:#6b7280,stroke-width:2px
style upper_flow fill:#050607,stroke:#61d095,stroke-width:2px
style lower_flow fill:#070605,stroke:#e4bd55,stroke-width:2px
style reference fill:#07090c,stroke:#9ba4b1,stroke-width:2px
style intake fill:#070d09,stroke:#61d095,stroke-width:2px
style hermes_plane fill:#0c0912,stroke:#c1a7ff,stroke-width:3px
style controls fill:#0d0b07,stroke:#e4bd55,stroke-width:2px
style response fill:#090806,stroke:#e4bd55,stroke-width:3px
style response_paths fill:#070707,stroke:#6b7280,stroke-width:1px
style immediate_paths fill:#080706,stroke:#8b8170,stroke-width:1px
style operational_path fill:#0d0b07,stroke:#e4bd55,stroke-width:2px
style escalation_path fill:#0d090a,stroke:#ff6b78,stroke-width:2px
style code_path fill:#0b090d,stroke:#c1a7ff,stroke-width:2px
style proposal_stage fill:#09080b,stroke:#9f86da,stroke-width:1px
style pr_stage fill:#07100a,stroke:#78c89a,stroke-width:1px
style outputs fill:#07100d,stroke:#78c89a,stroke-width:2px

File diff suppressed because one or more lines are too long

Before

Width:  |  Height:  |  Size: 575 KiB

View File

@ -26,7 +26,7 @@ while [[ $# -gt 0 ]]; do
;; ;;
-h|--help) -h|--help)
cat <<USAGE cat <<USAGE
Usage: scripts/ops/build_ananke_node_helper.sh [--image <image>] [--docker-config <path>] [--platforms <csv>] [--builder <name>] Usage: scripts/build_ananke_node_helper.sh [--image <image>] [--docker-config <path>] [--platforms <csv>] [--builder <name>]
USAGE USAGE
exit 0 exit 0
;; ;;

View File

@ -31,7 +31,7 @@ while [[ $# -gt 0 ]]; do
;; ;;
-h|--help) -h|--help)
cat <<USAGE cat <<USAGE
Usage: scripts/ops/build_harbor_bootstrap_bundle.sh [--images-file <path>] [--bundle-file <path>] [--docker-config <path>] [--platform <linux/arm64>] [--zstd-level <level>] Usage: scripts/build_harbor_bootstrap_bundle.sh [--images-file <path>] [--bundle-file <path>] [--docker-config <path>] [--platform <linux/arm64>] [--zstd-level <level>]
USAGE USAGE
exit 0 exit 0
;; ;;

View File

@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
usage() { usage() {
cat <<'USAGE' cat <<'USAGE'
Usage: Usage:
scripts/ops/cluster_power_console.sh [--repo-dir <path>] [--delegate-host <host>] <shutdown|startup> [recovery-script-options...] scripts/cluster_power_console.sh [--repo-dir <path>] [--delegate-host <host>] <shutdown|startup> [recovery-script-options...]
Purpose: Purpose:
Friendly manual entrypoint for running Ananke from a remote console. Friendly manual entrypoint for running Ananke from a remote console.
@ -17,9 +17,9 @@ Defaults:
--delegate-host titan-db --delegate-host titan-db
Examples: Examples:
scripts/ops/cluster_power_console.sh shutdown --execute scripts/cluster_power_console.sh shutdown --execute
scripts/ops/cluster_power_console.sh startup --execute --force-flux-branch main scripts/cluster_power_console.sh startup --execute --force-flux-branch main
scripts/ops/cluster_power_console.sh --delegate-host titan-24 shutdown --execute scripts/cluster_power_console.sh --delegate-host titan-24 shutdown --execute
USAGE USAGE
} }
@ -57,7 +57,7 @@ if [[ $# -lt 1 ]]; then
fi fi
SIBLING_SCRIPT="${SCRIPT_DIR}/cluster_power_recovery.sh" SIBLING_SCRIPT="${SCRIPT_DIR}/cluster_power_recovery.sh"
REPO_SCRIPT="${REPO_DIR}/scripts/ops/cluster_power_recovery.sh" REPO_SCRIPT="${REPO_DIR}/scripts/cluster_power_recovery.sh"
LOCAL_SCRIPT="" LOCAL_SCRIPT=""
if [[ -x "${SIBLING_SCRIPT}" ]]; then if [[ -x "${SIBLING_SCRIPT}" ]]; then
@ -82,6 +82,6 @@ remote_cmd=""
if [[ -n "${REMOTE_REPO_DIR}" ]]; then if [[ -n "${REMOTE_REPO_DIR}" ]]; then
remote_cmd+="ANANKE_REPO_DIR=$(printf '%q' "${REMOTE_REPO_DIR}") " remote_cmd+="ANANKE_REPO_DIR=$(printf '%q' "${REMOTE_REPO_DIR}") "
fi fi
remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi" remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi"
exec ssh -o BatchMode=yes -o ConnectTimeout=8 "${DELEGATE_HOST}" "${remote_cmd}" exec ssh -o BatchMode=yes -o ConnectTimeout=8 "${DELEGATE_HOST}" "${remote_cmd}"

View File

@ -16,7 +16,7 @@ fi
usage() { usage() {
cat <<USAGE cat <<USAGE
Usage: Usage:
scripts/ops/cluster_power_recovery.sh <prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup> [options] scripts/cluster_power_recovery.sh <prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup> [options]
Options: Options:
--execute Actually run commands (default is dry-run) --execute Actually run commands (default is dry-run)
@ -77,14 +77,14 @@ Options:
-h, --help Show help -h, --help Show help
Examples: Examples:
scripts/ops/cluster_power_recovery.sh prepare --execute scripts/cluster_power_recovery.sh prepare --execute
scripts/ops/cluster_power_recovery.sh bootstrap-seed --execute scripts/cluster_power_recovery.sh bootstrap-seed --execute
scripts/ops/cluster_power_recovery.sh harbor-seed --execute scripts/cluster_power_recovery.sh harbor-seed --execute
scripts/ops/cluster_power_recovery.sh longhorn-unlock --execute scripts/cluster_power_recovery.sh longhorn-unlock --execute
scripts/ops/cluster_power_recovery.sh flux-hold --execute scripts/cluster_power_recovery.sh flux-hold --execute
scripts/ops/cluster_power_recovery.sh status scripts/cluster_power_recovery.sh status
scripts/ops/cluster_power_recovery.sh shutdown --execute scripts/cluster_power_recovery.sh shutdown --execute
scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main
USAGE USAGE
} }

5
scripts/comms_sync_kb.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
python scripts/knowledge_render_atlas.py --write
python scripts/knowledge_render_atlas.py --write --out services/comms/knowledge

View File

@ -1,5 +1,3 @@
#!/usr/bin/env fish
### ------- helpers --------------------------------------------------------- ### ------- helpers ---------------------------------------------------------
function _need --description "ensure a command exists" function _need --description "ensure a command exists"

View File

@ -1,5 +1,3 @@
#!/usr/bin/env fish
### --------- helpers ---------- ### --------- helpers ----------
function _need --description "ensure a command exists" function _need --description "ensure a command exists"
for c in $argv for c in $argv

View File

@ -16,7 +16,7 @@ from pathlib import Path
# Paths, folders, and shared metadata # Paths, folders, and shared metadata
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
ROOT = Path(__file__).resolve().parents[2] ROOT = Path(__file__).resolve().parents[1]
DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards" DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards"
CONFIG_TEMPLATE = textwrap.dedent( CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path} """# {relative_path}
@ -110,6 +110,8 @@ WORKER_NODES = [
"titan-06", "titan-06",
"titan-07", "titan-07",
"titan-08", "titan-08",
"titan-09",
"titan-10",
"titan-11", "titan-11",
"titan-20", "titan-20",
"titan-21", "titan-21",
@ -117,6 +119,7 @@ WORKER_NODES = [
"titan-13", "titan-13",
"titan-14", "titan-14",
"titan-15", "titan-15",
"titan-16",
"titan-17", "titan-17",
"titan-18", "titan-18",
"titan-19", "titan-19",
@ -181,8 +184,7 @@ def scoped_node_expr(base, scope=""):
def node_cpu_expr(scope=""): def node_cpu_expr(scope=""):
idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))' idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))'
# Scrape stalls can briefly report an impossible idle rate after recovery. base = f"(1 - {idle}) * 100"
base = f"clamp_max(clamp_min((1 - {idle}) * 100, 0), 100)"
return scoped_node_expr(base, scope) return scoped_node_expr(base, scope)
@ -282,14 +284,8 @@ def dcgm_gpu_util_by_node():
) )
def nvidia_gpu_util_by_node():
return "max by (node) (nvidia_gpu_device_utilization_percent)"
def gpu_util_by_node(): def gpu_util_by_node():
process_exporter = nvidia_gpu_util_by_node() return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"
dcgm_fallback = f"({dcgm_gpu_util_by_node()}) unless on(node) ({process_exporter})"
return f"{process_exporter} or {dcgm_fallback} or {jetson_gpu_util_by_node()}"
def gpu_util_by_hostname(): def gpu_util_by_hostname():
@ -299,11 +295,16 @@ def gpu_util_by_hostname():
GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*" GPU_RESOURCE_REGEX = "nvidia(_com_|[.]com/)gpu.*"
def gpu_node_labels():
return f'max by (node) (kube_node_status_allocatable{{resource=~"{GPU_RESOURCE_REGEX}"}} > bool 0)'
def gpu_requests_by_namespace_node(scope_var): def gpu_requests_by_namespace_node(scope_var):
return ( return (
"sum by (namespace,node) (" "sum by (namespace,node) ("
f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} ' f'kube_pod_container_resource_requests{{resource=~"{GPU_RESOURCE_REGEX}",{scope_var}}} '
"* on(namespace,pod) group_left(node) kube_pod_info " "* on(namespace,pod) group_left(node) kube_pod_info "
f"* on(node) group_left() ({gpu_node_labels()})"
")" ")"
) )
@ -325,8 +326,7 @@ def gpu_usage_by_namespace(scope_var):
def jetson_gpu_usage_by_namespace(scope_var): def jetson_gpu_usage_by_namespace(scope_var):
requests_by_ns = gpu_requests_by_namespace_node(scope_var) requests_by_ns = gpu_requests_by_namespace_node(scope_var)
all_requests = gpu_requests_by_namespace_node('namespace=~".*"') total_by_node = f"sum by (node) ({requests_by_ns})"
total_by_node = f"sum by (node) ({all_requests})"
return ( return (
"sum by (namespace) (" "sum by (namespace) ("
f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) " f"({requests_by_ns}) / on(node) group_left() clamp_min({total_by_node}, 1) "
@ -335,12 +335,6 @@ def jetson_gpu_usage_by_namespace(scope_var):
) )
def jetson_gpu_requested_nodes():
all_requests = gpu_requests_by_namespace_node('namespace=~".*"')
requested = f"(sum by (node) ({all_requests}) > 0)"
return f"({requested}) and on(node) ({jetson_gpu_util_by_node()})"
def namespace_share_expr(resource_expr): def namespace_share_expr(resource_expr):
total = f"clamp_min(sum( {resource_expr} ), 1)" total = f"clamp_min(sum( {resource_expr} ), 1)"
return f"100 * ( {resource_expr} ) / {total}" return f"100 * ( {resource_expr} ) / {total}"
@ -390,29 +384,14 @@ def gpu_total_devices_expr():
def unattributed_gpu_usage(): def unattributed_gpu_usage():
unresolved = ( legacy_total = f"(sum({legacy_gpu_util_without_process_exporter()}) or on() vector(0))"
f"({legacy_gpu_util_without_process_exporter()}) "
f"unless on(node) ({jetson_gpu_requested_nodes()})"
)
legacy_total = f"(sum({unresolved}) or on() vector(0))"
return ( return (
f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")' f'label_replace(({legacy_total} > 0), "namespace", "unattributed", "", "")'
) )
def gpu_utilization_raw(scope_var): def gpu_utilization_raw(scope_var):
nvidia = ( return f"({nvidia_process_gpu_usage_by_namespace(scope_var)}) or ({unattributed_gpu_usage()})"
'label_replace('
f'{nvidia_process_gpu_usage_by_namespace(scope_var)}, '
'"gpu_source", "nvidia", "", "")'
)
jetson = (
'label_replace('
f'(({jetson_gpu_usage_by_namespace(scope_var)}) > 0), '
'"gpu_source", "jetson", "", "")'
)
attributed = f"sum by (namespace) (({nvidia}) or ({jetson}))"
return f"({attributed}) or ({unattributed_gpu_usage()})"
def gpu_pool_used_expr(scope_var): def gpu_pool_used_expr(scope_var):
@ -432,20 +411,13 @@ def namespace_gpu_share_expr(scope_var):
PROBLEM_PODS_EXPR = ( PROBLEM_PODS_EXPR = (
'((sum(max by(namespace,pod) (' 'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) '
'(kube_pod_status_phase{phase="Pending",namespace!~"veles"} == 1) ' "or on() vector(0)"
'and on(namespace,pod) ((time() - kube_pod_created{namespace!~"veles"}) > 900)'
')) or on() vector(0)) + (sum(max by(namespace,pod) ('
'(kube_pod_status_phase{phase=~"Failed|Unknown",namespace!~"veles"} == 1) '
'unless on(namespace,pod) kube_pod_owner{owner_kind="Job"}'
')) or on() vector(0)))'
) )
CRASHLOOP_EXPR = ( CRASHLOOP_EXPR = (
'sum(max by(namespace,pod) (kube_pod_container_status_waiting_reason' 'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason'
'{namespace!~"veles",reason=~"CrashLoopBackOff|ImagePullBackOff"} ' '{reason=~"CrashLoopBackOff|ImagePullBackOff"})) '
'and on(namespace,pod) ' "or on() vector(0)"
'((time() - kube_pod_created{namespace!~"veles"}) > 900))) '
'or on() vector(0)'
) )
STUCK_TERMINATING_EXPR = ( STUCK_TERMINATING_EXPR = (
'sum(max by (namespace,pod) (' 'sum(max by (namespace,pod) ('
@ -456,24 +428,21 @@ STUCK_TERMINATING_EXPR = (
) )
UPTIME_WINDOW = "365d" UPTIME_WINDOW = "365d"
# vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series. # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
UPTIME_RECORDING_METRIC = ( UPTIME_RECORDING_METRIC = f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas"}}'
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="request-v4"}}' UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])"
TRAEFIK_READY_EXPR = (
"("
'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
" / clamp_min("
'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)'
")"
) )
AVAILABILITY_REQUESTS_1H_EXPR = ( CONTROL_READY_FRACTION_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{' f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
'entrypoint="websecure",protocol="http",code=~"[1-5].."}[1h]))' f" / {CONTROL_TOTAL})"
) )
AVAILABILITY_FAILURES_1H_EXPR = ( UPTIME_AVAIL_EXPR = (
'sum(increase(traefik_entrypoint_requests_total{' f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
'entrypoint="websecure",protocol="http",code=~"5.."}[1h]))'
)
UPTIME_LIVE_FALLBACK_EXPR = (
f"(1 - (({AVAILABILITY_FAILURES_1H_EXPR} or on() vector(0)) / "
f"clamp_min({AVAILABILITY_REQUESTS_1H_EXPR}, 1)))"
)
UPTIME_RECORDING_EXPR = (
f"(last_over_time({UPTIME_RECORDING_METRIC}[48h]) "
f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
) )
# Tie-breaker to deterministically pick one node per namespace when shares tie. # Tie-breaker to deterministically pick one node per namespace when shares tie.
@ -1901,9 +1870,9 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
"Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
"Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
"Atlas Availability (365d)": "Request-weighted Atlas ingress availability; every server-side 5xx response counts as a failed request.", "Atlas Availability (365d)": "Rolling one-year Atlas availability; higher is better, below target means users saw downtime.",
"Problem Pods": "Current-service pods Pending for more than 15 minutes or in an actionable failed phase. Completed Jobs and retained Veles migration workloads are kept on drill-down dashboards but excluded here.", "Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.",
"CrashLoop / ImagePull": "Current-service pods stuck in CrashLoopBackOff or ImagePullBackOff for more than 15 minutes. Retained Veles migration workloads remain visible on the Pods dashboard.", "CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.",
"Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.", "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
"Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.", "Hottest node: CPU": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.",
"Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.", "Hottest node: RAM": "Highest worker memory use right now; lower is safer, high values risk evictions.",
@ -1941,7 +1910,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.", "Postgres Connections Used": "Current Postgres connections; lower leaves room for apps during spikes.",
"Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.", "Postgres Hottest Connections": "Database with the most active connections; high values identify the pressure source.",
"Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.", "Namespace CPU Share": "CPU share by namespace in the selected scope; big slices show who is using compute.",
"Namespace GPU Utilization": "Current proportional share of observed GPU compute activity. Process-aware NVIDIA metrics attribute titan-22/24 work to namespaces and non-pod work to host. Jetson titan-20/21 compute is assigned by Kubernetes shared-GPU allocations; unallocated activity remains unattributed. The slices total 100% of compute in use now, independent of the selected dashboard time range; idle appears only when observed activity is zero.", "Namespace GPU Utilization": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.",
"Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.", "Namespace RAM Share": "Memory share by namespace in the selected scope; big slices show who may drive pressure.",
"Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.", "Worker Node CPU": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.",
"Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.", "Worker Node RAM": "Worker memory over time; lower is safer, sustained high use risks evictions.",
@ -2142,7 +2111,7 @@ def build_overview():
"decimals": 4, "decimals": 4,
"text_mode": "value", "text_mode": "value",
"instant": True, "instant": True,
"description": "Rolling request-weighted availability at the Atlas HTTPS ingress: responses below 500 divided by all HTTP responses. Every server-side 5xx is a real failed request; client 4xx responses count as served. Replica counts, Grafana health, and monitoring gaps are not inputs, so they cannot lower availability unless user traffic actually receives a 5xx. A daily rollup job publishes one annual sample from deduplicated daily totals; Grafana keeps it for up to 48 hours so one delayed retry cannot cause a fallback, and only uses the same one-hour request SLI before history exists.", "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.",
}, },
{ {
"id": 4, "id": 4,
@ -2366,10 +2335,10 @@ def build_overview():
} }
overview_avg_coverage = f"(avg(({QUALITY_GATE_COVERAGE_BY_SUITE})) or on() vector(0))" overview_avg_coverage = f"(avg(({QUALITY_GATE_COVERAGE_BY_SUITE})) or on() vector(0))"
overview_category_health = ( overview_category_health = (
f'(avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{' f'avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
f'suite=~"{PLATFORM_TEST_SUITE_CANONICAL_MATCHER}",branch!="",branch=~"main|master|origin/main|origin/master",' f'suite=~"{PLATFORM_TEST_SUITE_CANONICAL_MATCHER}",branch!="",branch=~"main|master|origin/main|origin/master",'
f'category=~"{PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' f'category=~"{PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"'
'})) or label_set(vector(0), "category", "none")' "})"
) )
for panel_id, title, draw_expr, runtime_expr, y_pos in [ for panel_id, title, draw_expr, runtime_expr, y_pos in [
(40, "Pyrphoros UPS Current", ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_RUNTIME_DB, 7), (40, "Pyrphoros UPS Current", ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_RUNTIME_DB, 7),
@ -5048,100 +5017,11 @@ def build_jobs_dashboard():
} }
HERMES_TRIAGE_OPEN_EXPR = (
'sum(ariadne_hermes_triage_incident{status="human_required"})'
)
HERMES_TRIAGE_RESOLVED_EXPR = (
'sum(increase(ariadne_hermes_triage_action_total{result="success"}[24h]))'
)
HERMES_TRIAGE_DIAGNOSIS_EXPR = (
'ariadne_hermes_triage_duration_seconds{phase="diagnosis"}'
)
HERMES_TRIAGE_ACTIONS_EXPR = (
"sum by (action, result) "
"(increase(ariadne_hermes_triage_action_total[1h]))"
)
HERMES_TRIAGE_INCIDENTS_EXPR = (
"sum by (jenkins_job, status) (ariadne_hermes_triage_incident) > 0"
)
def _hermes_triage_panels():
"""Return the Hermes automated-triage panels for the testing dashboard.
Inputs: none. Outputs: a list of Grafana panel definitions covering open
escalations, recent automated actions, diagnosis latency and per-job
incident state, sourced from Ariadne's bounded triage metrics.
"""
return [
stat_panel(
600,
"Triage Escalations Awaiting a Human",
HERMES_TRIAGE_OPEN_EXPR,
{"h": 4, "w": 6, "x": 0, "y": 100},
instant=True,
description=(
"Incidents Hermes diagnosed where Ariadne refused to act "
"automatically. Each one has a Gitea issue when its job is "
"mapped, and fires HermesTriageHumanRequired."
),
),
stat_panel(
601,
"Automated Actions Succeeded (24h)",
HERMES_TRIAGE_RESOLVED_EXPR,
{"h": 4, "w": 6, "x": 6, "y": 100},
instant=True,
description=(
"Allowlisted actions Ariadne executed and completed: fixture "
"repair, transient-infra retry, or a pushed patch proposal."
),
),
stat_panel(
602,
"Hermes Diagnosis Time (s)",
HERMES_TRIAGE_DIAGNOSIS_EXPR,
{"h": 4, "w": 6, "x": 12, "y": 100},
unit="s",
decimals=1,
instant=True,
description=(
"Wall-clock time of the most recent Hermes Agent run. This is "
"the pause between a red build and a diagnosis."
),
),
timeseries_panel(
603,
"Triage Actions by Result (1h rate)",
HERMES_TRIAGE_ACTIONS_EXPR,
{"h": 8, "w": 12, "x": 0, "y": 104},
legend="{{action}} · {{result}}",
description=(
"requested/accepted/rejected/success/failed per action id. A "
"rejected action means an authorization gate refused it."
),
),
timeseries_panel(
604,
"Incident State by Job",
HERMES_TRIAGE_INCIDENTS_EXPR,
{"h": 8, "w": 12, "x": 12, "y": 104},
legend="{{jenkins_job}} · {{status}}",
description=(
"Lifecycle of each incident: detected, diagnosed, repairing, "
"awaiting_rebuild, resolved, human_required or failed."
),
),
]
def build_testing_dashboard(): def build_testing_dashboard():
dashboard = build_jobs_dashboard() dashboard = build_jobs_dashboard()
dashboard["uid"] = "atlas-testing" dashboard["uid"] = "atlas-testing"
dashboard["folderUid"] = PUBLIC_DASHBOARD_FOLDER dashboard["folderUid"] = PUBLIC_DASHBOARD_FOLDER
dashboard["editable"] = False dashboard["editable"] = False
dashboard["panels"] = list(dashboard["panels"]) + _hermes_triage_panels()
return dashboard return dashboard
@ -5610,13 +5490,12 @@ def build_gpu_dashboard():
panels.append( panels.append(
table_panel( table_panel(
4, 4,
"GPU Processes by Pod", "GPU Pods Reporting Device Util",
'topk(10, sum by (namespace,pod,node,process) ' 'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
'(nvidia_process_gpu_sm_util_percent{pod!="host"}) > 0)',
{"h": 8, "w": 12, "x": 12, "y": 8}, {"h": 8, "w": 12, "x": 12, "y": 8},
unit="percent", unit="percent",
transformations=[{"id": "labelsToFields", "options": {}}], transformations=[{"id": "labelsToFields", "options": {}}],
description="NVML process-level SM samples mapped to Kubernetes pods through host cgroups; values are per-process activity rather than duplicated whole-device utilization.", description="DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.",
) )
) )
return { return {

View File

@ -2,8 +2,8 @@
"""Generate OpenSearch Dashboards saved objects and render them into ConfigMaps. """Generate OpenSearch Dashboards saved objects and render them into ConfigMaps.
Usage: Usage:
scripts/render/dashboards_render_logs.py --build # rebuild NDJSON + ConfigMap scripts/dashboards_render_logs.py --build # rebuild NDJSON + ConfigMap
scripts/render/dashboards_render_logs.py # re-render ConfigMap from NDJSON scripts/dashboards_render_logs.py # re-render ConfigMap from NDJSON
""" """
from __future__ import annotations from __future__ import annotations
@ -14,21 +14,14 @@ import textwrap
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
def repo_root() -> Path: ROOT = Path(__file__).resolve().parents[1]
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
ROOT = repo_root()
DASHBOARD_DIR = ROOT / "services" / "logging" / "dashboards" DASHBOARD_DIR = ROOT / "services" / "logging" / "dashboards"
NDJSON_PATH = DASHBOARD_DIR / "logs.ndjson" NDJSON_PATH = DASHBOARD_DIR / "logs.ndjson"
CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-dashboards-objects.yaml" CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-dashboards-objects.yaml"
CONFIG_TEMPLATE = textwrap.dedent( CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path} """# {relative_path}
# Generated by scripts/render/dashboards_render_logs.py --build # Generated by scripts/dashboards_render_logs.py --build
apiVersion: v1 apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:

View File

@ -4,10 +4,10 @@
# Fallback: kubectl port-forward to service (OK for small/medium files). # Fallback: kubectl port-forward to service (OK for small/medium files).
# #
# Usage: # Usage:
# scripts/ops/jellyfin_manual_load.fish <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY] # scripts/jellyfin_manual_load.fish <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]
# Examples: # Examples:
# scripts/ops/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN" # scripts/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN"
# scripts/ops/jellyfin_manual_load.fish "$HOME/Movies/." movies # copy contents-only into /media/movies # scripts/jellyfin_manual_load.fish "$HOME/Movies/." movies # copy contents-only into /media/movies
function usage function usage
echo "Usage: "(basename (status filename))" <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]" echo "Usage: "(basename (status filename))" <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]"

View File

@ -1,5 +1,3 @@
#!/usr/bin/env fish
# Pick the correct K3s asset for a remote host (arm64 vs x86_64) # Pick the correct K3s asset for a remote host (arm64 vs x86_64)
function __k3s_asset_for_host function __k3s_asset_for_host
set -l host $argv[1] set -l host $argv[1]

View File

@ -25,14 +25,7 @@ from typing import Any, Iterable
import yaml import yaml
def repo_root() -> Path: REPO_ROOT = Path(__file__).resolve().parents[1]
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
REPO_ROOT = repo_root()
DASHBOARD_DIR = REPO_ROOT / "services" / "monitoring" / "dashboards" DASHBOARD_DIR = REPO_ROOT / "services" / "monitoring" / "dashboards"
CLUSTER_SCOPED_KINDS = { CLUSTER_SCOPED_KINDS = {
@ -587,7 +580,7 @@ def main() -> int:
catalog_rel = catalog_path.relative_to(REPO_ROOT).as_posix() catalog_rel = catalog_path.relative_to(REPO_ROOT).as_posix()
catalog_path.write_text( catalog_path.write_text(
f"# {catalog_rel}\n" f"# {catalog_rel}\n"
"# Generated by scripts/render/knowledge_render_atlas.py (do not edit by hand)\n" "# Generated by scripts/knowledge_render_atlas.py (do not edit by hand)\n"
+ yaml.safe_dump(catalog, sort_keys=False), + yaml.safe_dump(catalog, sort_keys=False),
encoding="utf-8", encoding="utf-8",
) )

View File

@ -2,8 +2,8 @@
"""Generate OpenSearch Observability seed objects and render them into ConfigMaps. """Generate OpenSearch Observability seed objects and render them into ConfigMaps.
Usage: Usage:
scripts/render/logging_render_observability.py --build # rebuild JSON + ConfigMap scripts/logging_render_observability.py --build # rebuild JSON + ConfigMap
scripts/render/logging_render_observability.py # re-render ConfigMap from JSON scripts/logging_render_observability.py # re-render ConfigMap from JSON
""" """
from __future__ import annotations from __future__ import annotations
@ -14,14 +14,7 @@ import textwrap
from dataclasses import dataclass from dataclasses import dataclass
from pathlib import Path from pathlib import Path
def repo_root() -> Path: ROOT = Path(__file__).resolve().parents[1]
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
ROOT = repo_root()
OBS_DIR = ROOT / "services" / "logging" / "observability" OBS_DIR = ROOT / "services" / "logging" / "observability"
APPS_PATH = OBS_DIR / "applications.json" APPS_PATH = OBS_DIR / "applications.json"
QUERIES_PATH = OBS_DIR / "saved_queries.json" QUERIES_PATH = OBS_DIR / "saved_queries.json"
@ -30,7 +23,7 @@ CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-observability-objects.
CONFIG_TEMPLATE = textwrap.dedent( CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path} """# {relative_path}
# Generated by scripts/render/logging_render_observability.py --build # Generated by scripts/logging_render_observability.py --build
apiVersion: v1 apiVersion: v1
kind: ConfigMap kind: ConfigMap
metadata: metadata:

View File

@ -3,21 +3,14 @@
from pathlib import Path from pathlib import Path
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
def indent(text: str, spaces: int) -> str: def indent(text: str, spaces: int) -> str:
prefix = " " * spaces prefix = " " * spaces
return "".join(prefix + line if line.strip("\n") else line for line in text.splitlines(keepends=True)) return "".join(prefix + line if line.strip("\n") else line for line in text.splitlines(keepends=True))
def main() -> None: def main() -> None:
root = repo_root() root = Path(__file__).resolve().parents[1]
source = root / "scripts" / "sync" / "monitoring_postmark_exporter.py" source = root / "scripts" / "monitoring_postmark_exporter.py"
target = root / "services" / "monitoring" / "postmark-exporter-script.yaml" target = root / "services" / "monitoring" / "postmark-exporter-script.yaml"
payload = source.read_text(encoding="utf-8") payload = source.read_text(encoding="utf-8")

View File

@ -3,7 +3,7 @@ set -euo pipefail
usage() { usage() {
cat <<USAGE cat <<USAGE
Usage: scripts/ops/node_recover.sh <node-name> [options] Usage: scripts/node_recover.sh <node-name> [options]
Options: Options:
--yes Skip confirmation prompt --yes Skip confirmation prompt

View File

@ -1,202 +0,0 @@
#!/usr/bin/env bash
# Drive and narrate the Hermes code-proposal demo.
#
# This is the proposal loop: a build fails on a real defect, Ariadne asks
# Hermes for a minimal patch, validates it as data, pushes a branch and opens a
# pull request. Nothing merges. The point of this half is the stop, not the fix.
#
# hermes_code_demo.sh monitor # follow the Test Automation Diagram live
# hermes_code_demo.sh reset # restore the demo repository to pre-run state
# hermes_code_demo.sh preflight # confirm the lab is ready to demo
# hermes_code_demo.sh run # seed the defect and narrate the loop
# hermes_code_demo.sh status # current incident/alert state, no changes
#
# The autonomous-repair demo is a separate script: hermes_triage_demo.sh.
#
# FIRST RUN: copy hermes_demo.env.example to hermes_demo.env in this directory
# and fill it in. That file is git-ignored precisely so it can hold real
# tokens; this script sources it automatically. You also need bstein/
# hermes-code-demo cloned locally (default ~/Development/hermes-code-demo,
# override with CODE_REPO_DIR).
#
# The only thing this mutates is the demo repository: it pushes a seeded defect
# to master and reverts it on reset.
set -euo pipefail
# shellcheck source=scripts/ops/hermes_demo_lib.sh
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_demo_lib.sh"
# DEMO_REPOS is the whole blast radius and is deliberately explicit: a real
# service's issues are genuine triage records, and clearing them to tidy a demo
# would destroy the evidence the system exists to produce.
DEMO_REPOS="${DEMO_REPOS:-hermes-code-demo}"
cmd_reset() {
say "Reset — restoring the code demo to its pre-run state"
if [ -z "${GITEA_TOKEN:-}" ]; then
note "GITEA_TOKEN unset; skipping repository cleanup"
else
for repo in $DEMO_REPOS; do
note "clearing bstein/$repo (demo repository)"
local items
items="$(gitea_get "/api/v1/repos/bstein/$repo/issues?state=all&limit=100" |
python3 -c 'import json,sys
for i in json.load(sys.stdin):
print(i["number"], "pr" if i.get("pull_request") else "issue")' 2>/dev/null || true)"
if [ -z "$items" ]; then
note " no issues or pull requests"
else
while read -r num kind; do
[ -z "$num" ] && continue
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
"$GITEA_URL/api/v1/repos/bstein/$repo/issues/$num"
note " deleted $kind #$num"
done <<< "$items"
fi
local branches
branches="$(gitea_get "/api/v1/repos/bstein/$repo/branches" |
python3 -c 'import json,sys,urllib.parse
for b in json.load(sys.stdin):
if b["name"].startswith("hermes-repair/"):
print(urllib.parse.quote(b["name"], safe=""))' 2>/dev/null || true)"
if [ -z "$branches" ]; then
note " no repair branches"
else
for ref in $branches; do
curl -s --max-time 25 -o /dev/null -X DELETE -H "Authorization: token $GITEA_TOKEN" \
"$GITEA_URL/api/v1/repos/bstein/$repo/branches/$ref"
note " deleted branch $(printf '%b' "${ref//%/\\x}")"
done
fi
done
fi
# The demo seeds its defect by pushing to master, and the fix only lands if
# someone merges the pull request - which, by design, nobody does during a
# demo. So master stays broken, and the next run aborts on "defect already
# present". Reset has to undo the seed rather than just report it, or the
# second demo of the day fails before it starts.
if [ -d "$CODE_REPO_DIR/.git" ]; then
note "restoring the demo repository working state"
( cd "$CODE_REPO_DIR" && git checkout -q master && git fetch -q origin &&
git reset -q --hard origin/master ) || note " could not sync master"
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
note " src/discount.py is correct; demo is armable"
else
note " src/discount.py carries the seeded defect; reverting it on master"
( cd "$CODE_REPO_DIR" &&
python3 - <<'PY'
import pathlib, re, sys
path = pathlib.Path("src/discount.py")
source = path.read_text()
# Matches the seeded `percent / 10` without also matching a correct
# `percent / 100`, so re-running reset on a healthy file changes nothing.
fixed = re.sub(r"percent / 10(?!\d)", "percent / 100", source)
if fixed == source:
sys.exit("unrecognised defect; fix src/discount.py by hand")
path.write_text(fixed)
PY
git commit -qam "revert: restore the discount divisor" && git push -q origin master &&
note " reverted and pushed; demo is armable" ) || note " revert failed — fix src/discount.py by hand"
fi
else
note "demo repository not cloned at $CODE_REPO_DIR; skipping"
fi
say "Ready"
note "real service repositories were not touched"
note "run 'preflight' next, then 'run'"
}
cmd_preflight() {
require_jenkins
say "Preflight — code demo"
if [ -d "$CODE_REPO_DIR/.git" ]; then
if grep -q 'percent / 100' "$CODE_REPO_DIR/src/discount.py" 2>/dev/null; then
note "demo repository: src/discount.py is correct; armable"
else
note "demo repository: src/discount.py carries a defect — run 'reset' first"
fi
else
note "demo repository: NOT CLONED at $CODE_REPO_DIR"
fi
local open_prs
open_prs="$(gitea_get "/api/v1/repos/bstein/hermes-code-demo/pulls?state=open" 2>/dev/null |
python3 -c 'import json,sys; print(len(json.load(sys.stdin)))' 2>/dev/null || echo '?')"
note "open hermes-code-demo PRs: $open_prs (must be 0 — the duplicate guard refuses while one is open)"
note "code proposals enabled: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_CODE_ENABLED 2>/dev/null)"
note "fix categories: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_FIX_CATEGORIES 2>/dev/null)"
shared_preflight
}
cmd_status() {
shared_status
say "Open proposals"
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
}
cmd_run() {
require_jenkins
[ -d "$CODE_REPO_DIR/.git" ] || { echo "clone bstein/hermes-code-demo to $CODE_REPO_DIR first" >&2; exit 1; }
local start_num next_num
start_num="$(last_build_number "$CODE_JOB")"
next_num=$((start_num + 1))
say "Seeding a one-line defect in src/discount.py"
( cd "$CODE_REPO_DIR" && git checkout -q master && git pull -q &&
python3 - <<'PY'
import pathlib
p = pathlib.Path("src/discount.py")
s = p.read_text()
old, new = "percent / 100", "percent / 10"
if old not in s:
raise SystemExit("defect already present or file changed; run reset first")
p.write_text(s.replace(old, new))
PY
)
# Shown before the push, not after: the audience should watch the defect go
# in rather than take on trust that the later diagnosis matched it. It is
# also the only moment in the whole demo where a human changes any code.
say "The change about to be pushed"
note "$ git diff -- src/discount.py"
( cd "$CODE_REPO_DIR" && git --no-pager diff --unified=2 -- src/discount.py ) |
sed 's/^/ /'
( cd "$CODE_REPO_DIR" &&
git commit -qam "refactor: simplify discount percentage math" && git push -q origin master )
note "pushed $(cd "$CODE_REPO_DIR" && git rev-parse --short HEAD) to master"
note "a plausible-looking change that breaks three regression tests"
say "Running the test gate -> build #$next_num"
note "HTTP $(jenkins_post "/job/$CODE_JOB/build")"
note "result: $(wait_for_build "$CODE_JOB" "$next_num")"
say "Asking Ariadne to look now rather than on its next minute"
note "\$ POST /api/internal/hermes/autotriage/run"
note "$(poke_ariadne)"
say "Ariadne collects evidence and asks Hermes for a minimal patch"
note "Hermes returns an anchored patch as data; Ariadne validates path, size,"
note "changed lines, and that the anchor is unique, then pushes hermes-repair/$next_num"
for _ in $(seq 1 40); do
sleep 15
ariadne_ticks 300 1 | grep -q "code_fix_proposed" && break
done
ariadne_ticks 400 3
say "Pull request awaiting human review (nothing merges automatically)"
note "https://scm.bstein.dev/bstein/hermes-code-demo/pulls"
}
case "${1:-}" in
run|code) cmd_run ;;
status) cmd_status ;;
preflight) cmd_preflight ;;
reset) cmd_reset ;;
monitor) run_monitor "$CODE_JOB" ;;
*) sed -n '2,23p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
esac

View File

@ -1,17 +0,0 @@
# Copy to hermes_demo.env (same directory) and fill in. That filename is
# git-ignored so it can hold real tokens. Both demo drivers read it:
# hermes_triage_demo.sh and hermes_code_demo.sh.
# Jenkins API token: https://ci.bstein.dev -> your user -> Configure -> API Token
export JENKINS_USER="your-jenkins-user"
export JENKINS_TOKEN="your-jenkins-api-token"
# Gitea token. The code demo needs it to clear its own demo repository on
# reset and to report open pull requests during preflight; the triage demo
# touches no repository and works without it.
export GITEA_TOKEN="your-gitea-token"
# Override only if you are not pointing at the usual lab.
# export JENKINS_URL="https://ci.bstein.dev"
# export GITEA_URL="https://scm.bstein.dev"
# export CODE_REPO_DIR="$HOME/Development/hermes-code-demo"

View File

@ -1,172 +0,0 @@
#!/usr/bin/env bash
# Shared plumbing for the two Hermes demo drivers.
#
# The triage demo and the code demo are separate scripts on purpose: they prove
# different halves of the Test Automation Diagram, they reset different things,
# and mixing them behind one command invited exactly the confusion of running
# the wrong subcommand in front of an audience. What they genuinely share -
# credentials, Jenkins access, the Ariadne tick reader - lives here, so a fix
# to any of it applies to both instead of being made twice and drifting.
#
# Not executable on its own; both drivers source it.
# Local, git-ignored credentials. `hermes_demo.env` is the current name;
# `hermes_triage_demo.env` is still read so an existing filled-in file keeps
# working after the split.
_DEMO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DEMO_ENV=""
for _candidate in "$_DEMO_DIR/hermes_demo.env" "$_DEMO_DIR/hermes_triage_demo.env"; do
if [ -r "$_candidate" ]; then
DEMO_ENV="$_candidate"
# shellcheck disable=SC1090
. "$_candidate"
break
fi
done
[ -n "$DEMO_ENV" ] || DEMO_ENV="$_DEMO_DIR/hermes_demo.env"
JENKINS_URL="${JENKINS_URL:-https://ci.bstein.dev}"
GITEA_URL="${GITEA_URL:-https://scm.bstein.dev}"
FIXTURE_JOB="hermes-triage-demo"
CODE_JOB="hermes-code-demo"
DEMO_NS="hermes-triage-demo"
CODE_REPO_DIR="${CODE_REPO_DIR:-$HOME/Development/hermes-code-demo}"
say() { printf '\n\033[1m[%s] %s\033[0m\n' "$(date -u +%H:%M:%S)" "$*"; }
note() { printf ' %s\n' "$*"; }
require_jenkins() {
if [ -z "${JENKINS_USER:-}" ] || [ -z "${JENKINS_TOKEN:-}" ]; then
echo "Missing Jenkins credentials." >&2
echo "Create $DEMO_ENV from hermes_demo.env.example and fill it in." >&2
exit 1
fi
}
# Every remote call is time-bounded. A hung curl during a demo is worse than a
# failed one: a failure says what to do next, a hang says nothing at all.
jenkins_get() { curl -sk --max-time 25 -u "$JENKINS_USER:$JENKINS_TOKEN" "$JENKINS_URL$1"; }
jenkins_post() {
curl -sk --max-time 25 -o /dev/null -w '%{http_code}' \
-u "$JENKINS_USER:$JENKINS_TOKEN" -X POST "$JENKINS_URL$1"
}
gitea_get() { curl -s --max-time 25 -H "Authorization: token ${GITEA_TOKEN:-}" "$GITEA_URL$1"; }
# Jenkins tree selectors use square brackets, which some curl builds treat as
# glob metacharacters and refuse to send - the request never leaves, the body
# is empty, and the JSON parse dies with a traceback that says nothing about
# the real cause. Encoded, so the demo does not depend on how curl was built.
last_build_number() {
local body
body="$(jenkins_get "/job/$1/api/json?tree=lastBuild%5Bnumber%5D")"
if [ -z "$body" ]; then
echo "Jenkins returned nothing for job $1 (check credentials and $JENKINS_URL)" >&2
return 1
fi
printf '%s' "$body" |
python3 -c 'import json,sys; print(json.load(sys.stdin)["lastBuild"]["number"])'
}
# Polls before sleeping, not after. Sleeping first meant a build that had
# already finished still cost a full interval of silence, which on stage reads
# as the script having missed it. The interval is short for the same reason:
# the wait is dead air in front of an audience, and a Jenkins status read is
# cheap.
BUILD_POLL_SECONDS="${BUILD_POLL_SECONDS:-3}"
wait_for_build() { # job number [max_seconds] -> prints result
local job="$1" num="$2" budget="${3:-1500}"
local waited=0 body building result
while [ "$waited" -le "$budget" ]; do
body="$(jenkins_get "/job/$job/$num/api/json?tree=result,building" || true)"
building="$(printf '%s' "$body" |
python3 -c 'import json,sys; print(json.load(sys.stdin).get("building"))' 2>/dev/null || echo unknown)"
if [ "$building" = "False" ]; then
result="$(printf '%s' "$body" | python3 -c 'import json,sys; print(json.load(sys.stdin).get("result"))')"
printf '%s' "$result"
return 0
fi
sleep "$BUILD_POLL_SECONDS"
waited=$((waited + BUILD_POLL_SECONDS))
done
printf 'TIMEOUT'
}
# Ariadne's triage tick is on cron, which cannot fire more often than once a
# minute. That minute is the largest gap between a build going red and the
# system visibly reacting, and it is pure dead air on stage. This runs the same
# tick immediately over the pod's own loopback, so nothing is exposed outside
# the cluster. The tick is idempotent - incidents dedupe on job and build
# number - so provoking it can only ever be a no-op, never a second incident.
poke_ariadne() {
kubectl -n maintenance exec deploy/ariadne -c ariadne -- python3 -c "
import json, urllib.request
req = urllib.request.Request(
'http://127.0.0.1:8080/api/internal/hermes/autotriage/run', method='POST')
body = json.load(urllib.request.urlopen(req, timeout=120))
jobs = body.get('jobs') or {}
print(body.get('status', 'ok'), '|', ', '.join(
f\"{name}={info.get('status')}\" for name, info in jobs.items()) or 'no jobs')
" 2>/dev/null || echo "tick request failed; the scheduler will pick it up within a minute"
}
ariadne_ticks() { # tail the autotriage decisions in human-readable form
kubectl -n maintenance logs deploy/ariadne -c ariadne --tail="${1:-400}" 2>/dev/null |
grep 'hermes autotriage tick' |
python3 -c '
import sys, json
for line in sys.stdin:
try:
d = json.loads(line)
except ValueError:
continue
print(" ", d["timestamp"][11:19], d.get("jobs"))' | tail -"${2:-5}"
}
# Both drivers narrate the same Test Automation Diagram; the monitor selects
# which branch of it to follow from MONITOR_JOB.
run_monitor() { # job
export MONITOR_JOB="$1"
exec python3 "$_DEMO_DIR/hermes_triage_monitor.py"
}
# The checks that are true of the lab regardless of which demo is running.
shared_preflight() {
note "ariadne image: $(kubectl -n maintenance get deploy ariadne -o jsonpath='{.spec.template.spec.containers[0].image}')"
note "autoremediation: $(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOREMEDIATION_ENABLED 2>/dev/null)"
# Printed as a list rather than the raw comma-separated setting: this is the
# outermost safety boundary, so it is worth being able to read at a glance.
local allowlist count
allowlist="$(kubectl -n maintenance exec deploy/ariadne -c ariadne -- printenv ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST 2>/dev/null | tr ',' ' ')"
count=0
for _job in $allowlist; do count=$((count + 1)); done
note "jobs Ariadne may triage ($count):"
for _job in $allowlist; do note " - $_job"; done
note "hermes: $(kubectl -n hermes get pods -l app=hermes --no-headers | awk '{print $2, $3}')"
local queued
queued="$(jenkins_get '/queue/api/json' | python3 -c 'import json,sys; print(len(json.load(sys.stdin)["items"]))')"
note "jenkins queue depth: $queued (demo is fastest when this is 0)"
# The Kubernetes cloud caps concurrent agent pods at containerCapStr. When
# real CI saturates that cap the demo build sits in the queue reporting
# "all nodes are offline" and the timings in the runbook do not apply.
local agents cap
cap="$(kubectl -n jenkins get cm jenkins-jcasc -o jsonpath='{.data.jenkins\.yaml}' 2>/dev/null |
grep -o 'containerCapStr: "[0-9]*"' | head -1 | grep -o '[0-9]*' || echo 5)"
agents="$(kubectl -n jenkins get pods --no-headers 2>/dev/null | grep -cE '\-[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{5}' || true)"
note "jenkins agent pods: ${agents:-0}/${cap:-5} (a full pool stalls the demo — wait for a free slot)"
}
# The alert and incident state both demos are judged by.
shared_status() {
say "Incident state (last ticks)"
ariadne_ticks 600 8
say "Firing alerts"
kubectl -n monitoring exec deploy/vmalert-atlas-availability -- wget -qO- localhost:8880/api/v1/alerts 2>/dev/null |
python3 -c '
import json,sys
alerts = json.load(sys.stdin).get("data", {}).get("alerts", [])
print(" none" if not alerts else "")
for a in alerts:
print(" ", a["name"], a["state"], "build", a.get("labels", {}).get("build"))' 2>/dev/null ||
note "(query vmalert directly if this fails)"
}

View File

@ -1,102 +0,0 @@
#!/usr/bin/env bash
# Drive and narrate the Hermes automated-triage demo.
#
# This is the autonomous loop: a build fails, Ariadne diagnoses it through
# Hermes, authorizes a predefined repair, performs it, and rebuilds green
# without a human touching anything.
#
# hermes_triage_demo.sh monitor # follow the Test Automation Diagram live
# hermes_triage_demo.sh reset # restore the demo to its pre-run state
# hermes_triage_demo.sh preflight # confirm the lab is ready to demo
# hermes_triage_demo.sh run # arm the failure and narrate the loop
# hermes_triage_demo.sh status # current incident/alert state, no changes
#
# The code-proposal demo is a separate script: hermes_code_demo.sh. They prove
# different halves of the diagram and reset different things, so they are kept
# apart rather than behind one command.
#
# FIRST RUN: copy hermes_demo.env.example to hermes_demo.env in this directory
# and fill it in. That file is git-ignored precisely so it can hold real
# tokens; this script sources it automatically.
#
# Needs kubectl access to the cluster as well. Nothing here mutates the cluster
# directly: the demo only asks Jenkins to run a parameterized build.
set -euo pipefail
# shellcheck source=scripts/ops/hermes_demo_lib.sh
. "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/hermes_demo_lib.sh"
# The fixture is a ConfigMap the demo job reads. Resetting it is the whole
# blast radius of this script: no repository is touched, because the triage
# loop repairs infrastructure rather than source.
cmd_reset() {
say "Reset — restoring the triage demo to its pre-run state"
note "fixture -> healthy"
if kubectl -n "$DEMO_NS" patch cm hermes-triage-demo-fixture \
--type merge -p '{"data":{"state":"healthy"}}' >/dev/null 2>&1; then
note " fixture: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
else
note " fixture patch failed (is the demo namespace present?)"
fi
say "Ready"
note "no repository was touched; this demo repairs infrastructure, not source"
note "run 'preflight' next, then 'run'"
}
cmd_preflight() {
require_jenkins
say "Preflight — triage demo"
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null || echo MISSING)"
shared_preflight
}
cmd_status() {
shared_status
say "Demo namespace"
kubectl -n "$DEMO_NS" get jobs --no-headers 2>/dev/null | sed 's/^/ /'
}
cmd_run() {
require_jenkins
local start_num next_num
start_num="$(last_build_number "$FIXTURE_JOB")"
next_num=$((start_num + 1))
say "Arming the demo failure (SEED_FAILURE=true) -> build #$next_num"
note "HTTP $(jenkins_post "/job/$FIXTURE_JOB/buildWithParameters?SEED_FAILURE=true")"
note "Only manual step. Everything after this is automatic."
say "Waiting for the seeded build to fail"
note "result: $(wait_for_build "$FIXTURE_JOB" "$next_num")"
say "Asking Ariadne to look now rather than on its next minute"
note "\$ POST /api/internal/hermes/autotriage/run"
note "$(poke_ariadne)"
say "Ariadne detects, gathers evidence, asks Hermes, authorizes, repairs"
note "the repair is a single in-process ConfigMap patch, so watch the fixture"
for _ in $(seq 1 40); do
sleep 10
if [ "$(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}' 2>/dev/null)" = "healthy" ]; then
note "fixture patched back to healthy"
break
fi
done
ariadne_ticks 400 4
say "Ariadne triggers one rebuild with seeding disabled"
note "result: $(wait_for_build "$FIXTURE_JOB" $((next_num + 1)))"
say "Resolution"
sleep 45
ariadne_ticks 200 3
note "fixture state: $(kubectl -n "$DEMO_NS" get cm hermes-triage-demo-fixture -o jsonpath='{.data.state}')"
}
case "${1:-}" in
run|fixture) cmd_run ;;
status) cmd_status ;;
preflight) cmd_preflight ;;
reset) cmd_reset ;;
monitor) run_monitor "$FIXTURE_JOB" ;;
*) sed -n '2,17p' "$0" | sed 's/^# \{0,1\}//' ; exit 1 ;;
esac

View File

@ -1,552 +0,0 @@
#!/usr/bin/env python3
"""Narrate the triage flow live, showing every command it runs.
Run in a second terminal beside the demo. When a stage of
the Test Automation Diagram (`mermaid/TestAutomation.mmd`) is reached this
prints, in order:
the stage banner, naming the diagram subgraph
the service UI to look at, if one changes at that stage
each command, echoed before it runs, then its output
Echoing the commands is the point. An audience watching a dashboard has to
take the result on trust; watching `kubectl` run against the cluster and
reading the raw answer is the difference between a demonstration and an
assertion.
Two honest limits. This reports at subgraph granularity, not per node: the
diagram draws the collector, response check, marker check and authorizer
separately, and a refusal here names the gate rather than walking all twelve.
And it follows one incident on one job, whereas the diagram describes the whole
system.
Read-only. Every command below is a read; nothing here changes the cluster.
"""
from __future__ import annotations
import json
import os
import re
import subprocess
import sys
import time
from datetime import datetime, timezone
JOB = os.environ.get("MONITOR_JOB", "hermes-triage-demo")
NS_DEMO = "hermes-triage-demo"
NS_ARIADNE = "maintenance"
JENKINS = os.environ.get("JENKINS_URL", "https://ci.bstein.dev")
GITEA = os.environ.get("GITEA_URL", "https://scm.bstein.dev")
GRAFANA = os.environ.get("GRAFANA_URL", "https://metrics.bstein.dev")
HERMES_UI = os.environ.get("HERMES_URL", "https://agent.bstein.dev")
POLL_SECONDS = 6
# `--filter <text>` restricts the monitor to incidents whose id contains that
# text; `--incident <id>` pins it to exactly one. Without either, it follows
# the newest incident it can see for this job, which is what someone who just
# triggered a build actually wants.
def _argv_option(name: str) -> str:
"""Read `--name value` or `--name=value` from argv, or "" when absent."""
prefix = f"--{name}="
for index, arg in enumerate(sys.argv[1:]):
if arg.startswith(prefix):
return arg[len(prefix) :].strip()
if arg == f"--{name}" and index + 2 <= len(sys.argv[1:]):
return sys.argv[index + 2].strip()
return ""
FILTER = _argv_option("filter")
PIN = _argv_option("incident")
# The fixture job repairs a ConfigMap; the code job proposes a patch. Evidence
# that suits one is false for the other, so the stages branch on it.
IS_CODE_JOB = JOB == "hermes-code-demo"
BOLD, DIM, GREEN, CYAN, YELLOW, RED, RESET = (
"\033[1m", "\033[2m", "\033[32m", "\033[36m", "\033[33m", "\033[31m", "\033[0m"
)
# stage key -> (diagram subgraph, what it means, diagram path, UI worth showing)
STAGES: dict[str, tuple[str, str, str, str]] = {
"detect": (
"Detect and gather",
"A failed build becomes one incident",
"Jenkins detector -> Evidence sources",
f"{JENKINS}/job/{JOB}/ — the red build",
),
"evidence": (
"Detect and gather",
"Ariadne assembles the bounded evidence bundle",
"Console reader -> Failure ranker -> Context filter -> Incident bundle",
"",
),
"hermes": (
"Hermes analysis",
"Hermes returns a recommendation it cannot act on",
"Triage skills -> Structured recommendation",
f"{HERMES_UI} — the agent run appears here",
),
"gates": (
"Ariadne policy gates",
"Ariadne decides on its own reading of the evidence",
"Response check -> Scoped repair guard -> Action authorizer",
"",
),
"route": (
"Ariadne response",
"The policy result, and which branch it opens",
"Policy result -> Action registry",
"",
),
"response": (
"Ariadne response",
"Ariadne executes, proposes, or escalates",
"Action registry -> Scoped ConfigMap repair -> Action result",
"",
),
"verify": (
"Ariadne response",
"The branch build checks the proposal" if JOB == "hermes-code-demo"
else "One rebuild decides whether the repair held",
"Action result -> validation build",
f"{JENKINS}/job/{JOB}/ — a new build starts on its own",
),
"outputs": (
"Inspectable outputs",
"Incident closed; the artifacts remain",
"Ariadne records every outcome -> Audit events, Triage metrics",
f"{GRAFANA}/d/atlas-testing — triage panels",
),
}
ORDER = ["detect", "evidence", "hermes", "gates", "route", "response", "verify", "outputs"]
def stamp() -> str:
return datetime.now(timezone.utc).strftime("%H:%M:%S")
def run(cmd: list[str], *, show: bool = True, limit: int = 12) -> str:
"""Echo a command, run it, print its output, and return stdout."""
if show:
print(f" {DIM}${RESET} {CYAN}{' '.join(cmd)}{RESET}")
try:
# Ariadne emits ~200 lines per tick, so a ten-minute window is a
# few thousand lines; the fetch needs room to finish.
done = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
out = done.stdout.strip()
except Exception as exc: # a failed read must never stop the narration
print(f" {RED}(command failed: {exc}){RESET}")
return ""
if show:
for line in (out.split("\n")[:limit] or ["(no output)"]):
print(f" {line[:150]}")
print()
return out
def quiet(cmd: list[str]) -> str:
return run(cmd, show=False)
def banner(key: str) -> None:
subgraph, meaning, path, ui = STAGES[key]
print(f"\n{GREEN}{'' * 70}{RESET}")
print(f"{GREEN}{BOLD} {stamp()} {subgraph}{RESET}{GREEN}{meaning}{RESET}")
print(f"{DIM} diagram: {path}{RESET}")
if ui:
print(f"{YELLOW} look at: {ui}{RESET}")
print(f"{GREEN}{'' * 70}{RESET}")
def ariadne_records(since: str = "10m") -> list[dict]:
"""Return recent Ariadne log records that parse as JSON.
Windowed by time, not line count. Ariadne emits roughly two hundred lines
per tick once it is polling every job, so a --tail window wide enough to
be useful is impossible to guess and a narrow one silently drops ticks.
"""
raw = quiet(
["kubectl", "-n", NS_ARIADNE, "logs", "deploy/ariadne", "-c", "ariadne", f"--since={since}"]
)
records = []
for line in raw.split("\n"):
line = line.strip()
if line.startswith("{"):
try:
records.append(json.loads(line))
except ValueError:
continue
return records
def job_states(records: list[dict]) -> list[dict]:
"""Return every tick result for this job in the window, oldest first.
The stages are reconstructed from all of them rather than from the newest
alone. A tick reporting the repair is replaced by the next tick within
seconds, so reading only the latest state means a poll landing at the
wrong moment loses that step permanently.
"""
states = []
for record in records:
if record.get("event") == "hermes_autotriage" and record.get("jobs"):
try:
state = json.loads(record["jobs"]).get(JOB)
except ValueError:
continue
if state:
states.append(state)
return _one_incident(states)
def _one_incident(states: list[dict]) -> list[dict]:
"""Narrow a window's ticks to the single incident worth narrating.
A ten-minute window routinely holds two incidents - the build just pushed
and the one before it - and interleaving them produces a transcript that
reads as though the system is doing everything twice. Following one at a
time is both clearer and closer to the truth: the diagram describes the
life of one incident.
`--incident` pins an exact id, `--filter` matches a substring, and
otherwise the newest incident in the window wins.
"""
if PIN:
return [s for s in states if str(s.get("incident_id") or "") == PIN]
if FILTER:
states = [s for s in states if FILTER in str(s.get("incident_id") or "")]
ids = [str(s.get("incident_id") or "") for s in states if s.get("incident_id")]
if not ids:
return states
newest = ids[-1]
return [s for s in states if str(s.get("incident_id") or "") == newest]
_DIAG_QUERY = (
". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \""
"import json,os,psycopg;"
"conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);"
"cur=conn.cursor();"
"cur.execute(\\\"select event_type,detail from ariadne_events where event_type in"
" ('hermes_autotriage_diagnosis','hermes_autotriage_code_proposal')"
" order by id desc limit 200\\\");"
"rows=[(t, d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];"
"rows=[(t,d) for t,d in rows if d.get('incident_id')=='__INCIDENT__'];"
"print(json.dumps({'note':'no Hermes run recorded for this incident yet'},indent=1))"
" if not rows else None;"
"t,d=(rows[0] if rows else ('',{}));"
"print(json.dumps({'event':t,'incident':d.get('incident_id'),"
"'authorized':d.get('authorized'),'authorize_reason':d.get('authorize_reason'),"
"'validated':d.get('validated'),'reject_reason':d.get('reject_reason'),"
"'chosen_path':d.get('chosen_path'),'url':d.get('url'),"
"'run_id':d.get('run_id') or (d.get('run') or {}).get('run_id'),"
"'outcome':d.get('outcome')},indent=1)) if rows else None\""
)
_HISTORY_QUERY = (
". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; python3 -c \""
"import json,os,psycopg;"
"conn=psycopg.connect(os.environ['ARIADNE_DATABASE_URL']);"
"cur=conn.cursor();"
"cur.execute(\\\"select created_at,detail from ariadne_events where"
" event_type='hermes_autotriage_incident' order by id desc limit 40\\\");"
"rows=[(t,d if isinstance(d,dict) else json.loads(d)) for t,d in cur.fetchall()];"
"rows=[r for r in rows if r[1].get('incident_id')=='__INCIDENT__'];"
"[print(str(t)[11:19], v.get('status'), json.dumps(v.get('phase') or {})[:80])"
" for t,v in reversed(rows)]\""
)
_BUNDLE_QUERY = (
"python3 -c \""
"from ariadne.services import hermes_autotriage_evidence as ev;"
"lb={'number':__BUILD__,'result':'FAILURE','building':False,'timestamp':0,'duration':0,'url':''};"
"b=ev.collect_evidence('__INCIDENT__','__JOB__',lb);"
"j=b['jenkins']; regions=j.get('console_failures') or []; tests=j.get('failed_tests') or [];"
"recs=(b.get('log_evidence') or {}).get('records') or [];"
"print('jenkins.console_failures : %d region(s), truncated=%s' % (len(regions), j.get('console_truncated')));"
"[print(' | ' + l[:110]) for l in ((regions[0].get('text') or '').strip().split(chr(10))[-3:] if regions else [])];"
"print('jenkins.failed_tests : %d' % len(tests));"
"[print(' | %s :: %s' % (t.get('className'), t.get('name'))) for t in tests[:2]];"
"print('log_evidence.records : %d from OpenSearch kube-*' % len(recs));"
"[print(' | [%s] %s' % (r.get('namespace'), (r.get('message') or '')[:90])) for r in recs[:2]]\""
)
def bundle_sample(incident: str) -> None:
"""Show a trimmed sample of the bundle that was sent to Hermes.
Rebuilt from the same collector Ariadne used. A finished build's console
does not change, so this is durable rather than a live reading, and it is
trimmed hard on purpose: the point is to show what kind of evidence each
source contributes, not to reprint the bundle.
"""
if "/" not in incident:
return
job, _, build = incident.rpartition("/")
query = (
_BUNDLE_QUERY.replace("__BUILD__", build)
.replace("__INCIDENT__", incident)
.replace("__JOB__", job)
)
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"sh", "-c", ". /vault/secrets/ariadne-env.sh >/dev/null 2>&1; " + query], limit=14)
def incident_history(incident: str) -> None:
"""Show this incident's recorded state changes, oldest first.
Durable evidence on purpose. A stage describes a moment that has passed,
so reading live cluster state at that point misrepresents it: by the time
the detection stage is narrated the repair has already run, and the
fixture would read healthy as though it had never failed.
"""
if not incident:
return
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"sh", "-c", _HISTORY_QUERY.replace("__INCIDENT__", incident)], limit=10)
def diagnosis_event(incident: str) -> str:
"""Show the diagnosis Ariadne stored for one incident, and return its run id.
Scoped to the incident on purpose. Reading the newest diagnosis in the
table meant an unrelated service's run could appear in the middle of this
incident's narration - which is worse than showing nothing, because it
looks like the answer to the question on screen.
"""
out = run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"sh", "-c", _DIAG_QUERY.replace("__INCIDENT__", incident)], limit=26)
match = re.search(r'"run_id":\s*"([^"]+)"', out or "")
return match.group(1) if match else ""
def hermes_run_link(run_id: str) -> None:
"""Print the deep link that reopens this run in the Hermes console.
The whole point of the stage is showing that a model made the call, and a
run id nobody can open is not that. This is the same route the pull request
links to, so the audience lands on the page the artifact points at.
"""
if not run_id:
print(f" {DIM}no run id recorded yet for this incident{RESET}\n")
return
print(f" {BOLD}open the decision itself:{RESET} {HERMES_UI}/chat?resume={run_id}")
print(f" {DIM}that page shows the prompt Hermes was given, the evidence bundle it read,"
f" the skill it invoked, and the JSON it returned{RESET}\n")
def evidence_for(key: str, incident: str = "") -> None:
"""Run the reads that show this stage actually happened."""
if key == "detect":
print(f" {DIM}the incident's recorded state changes; durable, so it still reads"
f" true after the repair has run:{RESET}")
incident_history(incident)
elif key == "evidence":
print(f" {DIM}a sample of what each source contributed to the bundle Hermes"
f" received:{RESET}")
bundle_sample(incident)
elif key == "hermes":
print(f" {DIM}what Hermes actually returned, as Ariadne stored it:{RESET}")
run_id = diagnosis_event(incident)
hermes_run_link(run_id)
print(f" {DIM}Hermes holds no Git or Kubernetes write access; this JSON is its"
f" entire output. When outcome.suggested_remediation is populated, Hermes found"
f" no action that fits and is proposing one for a maintainer to build; it is"
f" recorded and printed in the issue, and no gate reads it{RESET}\n")
elif key == "gates":
print(f" {DIM}the authorization is a match between three separate things: an action"
f" Ariadne already has code to perform, an action id Hermes is permitted to"
f" request, and the action Hermes actually recommended. Ariadne holds the"
f" registry below; Hermes cannot add to it, and the recommendation only"
f" proceeds because it names something already in it.{RESET}")
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", "ARIADNE_HERMES_ALLOWED_ACTIONS"], limit=2)
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", "ARIADNE_HERMES_MIN_CONFIDENCE"], limit=2)
print(f" {DIM}a separate setting governs what Hermes may be asked to propose a fix"
f" for. It is separate because these become pull requests rather than"
f" changes Ariadne makes itself:{RESET}")
run(["kubectl", "-n", NS_ARIADNE, "exec", "deploy/ariadne", "-c", "ariadne", "--",
"printenv", "ARIADNE_HERMES_FIX_CATEGORIES"], limit=2)
print()
elif key == "route":
if IS_CODE_JOB:
print(f" {DIM}a source fix is not one of the registered actions, so this takes"
f" the Optional source proposal branch rather than the Action registry.{RESET}\n")
else:
print(f" {DIM}the fixture repair is an operational action, so the Optional source"
f" proposal branch is not taken for this incident{RESET}\n")
elif key in {"response", "outputs"}:
if IS_CODE_JOB:
print(f" {DIM}the pull request Ariadne opened is recorded on the incident above"
f" (branch, pr_number, url). Hermes produced the patch as data; Ariadne"
f" validated it and pushed the branch.{RESET}")
print(f" {YELLOW}pull requests: {GITEA}/bstein/hermes-code-demo/pulls{RESET}\n")
else:
run(["kubectl", "-n", NS_DEMO, "get", "cm", "hermes-triage-demo-fixture",
"-o", "jsonpath={.data.state}"])
if key == "response":
print(f" {DIM}that value read 'unhealthy' when the build failed - the seeded"
f" fault - and reads 'healthy' above because Ariadne has just patched"
f" it. That single field changing is the repair.{RESET}\n")
if key == "outputs":
print(f" {DIM}on the diagram this is the 'records and artifacts' edge out of the"
f" whole Ariadne response box, not out of one branch. Every path ends"
f" here: an executed action, an escalation, or a pull request all record"
f" the same audit events and metrics.{RESET}")
print(f" {YELLOW}issues filed by triage: {GITEA}/bstein/ariadne/issues{RESET}\n")
elif key == "verify":
if IS_CODE_JOB:
print(f" {DIM}the branch build validates the proposal. Nothing merges"
f" automatically: the incident stays human-required whatever the branch"
f" build says, because a person decides whether the fix is right.{RESET}\n")
else:
print(f" {DIM}exactly one rebuild is triggered; it never retries in a loop. This"
f" completes the operational branch: because the action was authorized and"
f" performed, neither the human-required response nor the optional source"
f" proposal is entered for this incident.{RESET}\n")
class Monitor:
"""Track which Test Automation Diagram stage the current incident has reached."""
def __init__(self) -> None:
# Keyed by incident. A single shared set was cleared whenever the
# incident changed, so two live incidents in the same window wiped each
# other's progress and reprinted every stage on every poll.
self.done: dict[str, set[str]] = {}
self.incident = ""
self.summarised: set[str] = set()
self.last_tick = ""
def mark(self, key: str, evidence: str) -> None:
seen = self.done.setdefault(self.incident, set())
if key in seen:
return
seen.add(key)
banner(key)
print(f" {BOLD}what happened:{RESET} {evidence}\n")
evidence_for(key, self.incident)
def new_incident(self, incident: str) -> None:
if not incident or incident == self.incident:
return
self.incident = incident
if incident not in self.done:
self.done[incident] = set()
print(f"\n{BOLD}{YELLOW}══ incident {incident} ══{RESET}")
def checklist(self) -> None:
print(f"\n{BOLD} Test Automation Diagram progress{RESET}")
seen = self.done.get(self.incident, set())
for key in ORDER:
tick = f"{GREEN}{RESET}" if key in seen else f"{DIM}·{RESET}"
print(f" {tick} {key:<9} {STAGES[key][0]:<22} {DIM}{STAGES[key][2]}{RESET}")
print()
def main() -> None:
print(f"{BOLD}Hermes triage monitor — following the Test Automation Diagram"
f" (mermaid/TestAutomation.mmd){RESET}")
print(f"{DIM}Every command is echoed before it runs. Read-only. Ctrl-C to stop.{RESET}")
print(f"{YELLOW}Jenkins: {JENKINS}/job/{JOB}/{RESET}")
print(f"{YELLOW}Gitea: {GITEA}/bstein{RESET}")
print(f"{YELLOW}Grafana: {GRAFANA}/d/atlas-testing{RESET}")
if PIN:
print(f"{DIM}following incident {PIN} only{RESET}\n")
elif FILTER:
print(f"{DIM}following incidents matching {FILTER!r}{RESET}\n")
else:
print(f"{DIM}following the newest incident for {JOB};"
f" pass --filter TEXT or --incident ID to pin one{RESET}\n")
monitor = Monitor()
while True:
states = job_states(ariadne_records())
# Replay every tick in the window, so a step that lasted one tick is
# never lost to poll timing. mark() is idempotent.
for state in states:
status = str(state.get("status") or "")
monitor.new_incident(str(state.get("incident_id") or ""))
if status in {"human_required", "awaiting_rebuild", "deduped", "failed"}:
monitor.mark("detect", f"incident {monitor.incident} opened from a terminal failure")
if status == "awaiting_rebuild":
monitor.mark("evidence", "bundle collected: console regions, tests, logs")
monitor.mark("hermes", "diagnosis returned and parsed against the frozen schema")
monitor.mark("gates", "every gate passed; the predefined action was authorized")
monitor.mark(
"route",
"policy result: authorized -> Action registry (Optional source proposal not taken)",
)
monitor.mark(
"response",
f"Scoped ConfigMap repair: {state.get('repair', 'action')} on {state.get('target', '')}",
)
monitor.mark("verify", "one rebuild triggered with seeding disabled")
if status == "human_required":
reason = str(state.get("reason") or "")
proposed = reason == "code_fix_proposed"
monitor.mark("evidence", "bundle collected")
monitor.mark(
"hermes",
"patch proposed as data: a path, an exact anchor and a replacement"
if proposed
else "diagnosis returned",
)
monitor.mark(
"gates",
"no action was authorized; a source fix is not an action, so this takes"
" the proposal branch rather than the registry"
if proposed
else f"Ariadne refused: {reason} - nothing ran",
)
monitor.mark(
"route",
"policy result: no action -> Optional source proposal -> Patch validator"
if proposed
else "policy result: refused -> Diagnosis and next checks -> opens an issue",
)
monitor.mark(
"response",
"Ariadne opens a pull request; nothing merges without a human"
if proposed
else "escalated; issue filed in the service repository",
)
if proposed:
monitor.mark(
"verify",
"the branch build validates the proposal; the incident stays"
" human-required either way",
)
if status == "healthy" and state.get("resolved") and monitor.incident not in monitor.summarised:
monitor.mark("verify", "rebuild finished green")
monitor.mark("outputs", f"resolved: {', '.join(state['resolved'])}")
monitor.checklist()
monitor.summarised.add(monitor.incident)
status = str(states[-1].get("status") or "") if states else ""
if status and status != monitor.last_tick:
monitor.last_tick = status
print(f"{DIM}{stamp()} tick: {status}{RESET}")
sys.stdout.flush()
time.sleep(POLL_SECONDS)
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print("\nstopped")

View File

@ -1,85 +0,0 @@
#!/usr/bin/env python3
"""Print a compact inventory of Flux Kustomization resources."""
from __future__ import annotations
import argparse
from pathlib import Path
from typing import Any
import yaml
def _iter_docs(path: Path) -> list[dict[str, Any]]:
docs: list[dict[str, Any]] = []
for raw in yaml.safe_load_all(path.read_text(encoding="utf-8")):
if isinstance(raw, dict):
docs.append(raw)
return docs
def _is_flux_kustomization(doc: dict[str, Any]) -> bool:
return (
doc.get("kind") == "Kustomization"
and str(doc.get("apiVersion") or "").startswith("kustomize.toolkit.fluxcd.io/")
)
def _depends_on(spec: dict[str, Any]) -> str:
names = []
for dep in spec.get("dependsOn") or []:
if isinstance(dep, dict) and dep.get("name"):
names.append(str(dep["name"]))
return ",".join(names)
def _row(doc: dict[str, Any], path: Path) -> dict[str, str]:
metadata = doc.get("metadata") or {}
spec = doc.get("spec") or {}
annotations = metadata.get("annotations") or {}
return {
"name": str(metadata.get("name") or ""),
"path": str(spec.get("path") or ""),
"namespace": str(spec.get("targetNamespace") or ""),
"depends": _depends_on(spec),
"prune": str(spec.get("prune", "")),
"wait": str(spec.get("wait", "")),
"suspend": str(spec.get("suspend", False)),
"reason": str(annotations.get("atlas.bstein.dev/suspend-reason") or ""),
"file": str(path),
}
def build_inventory(root: Path) -> list[dict[str, str]]:
rows: list[dict[str, str]] = []
for path in sorted(root.rglob("*.yaml")):
for doc in _iter_docs(path):
if _is_flux_kustomization(doc):
rows.append(_row(doc, path))
return sorted(rows, key=lambda item: item["name"])
def _print_table(rows: list[dict[str, str]]) -> None:
columns = ["name", "path", "namespace", "depends", "prune", "wait", "suspend", "reason"]
widths = {
column: max([len(column), *[len(row[column]) for row in rows]]) for column in columns
}
header = " ".join(column.ljust(widths[column]) for column in columns)
print(header)
print(" ".join("-" * widths[column] for column in columns))
for row in rows:
print(" ".join(row[column].ljust(widths[column]) for column in columns))
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", nargs="?", default="clusters/atlas/flux-system")
args = parser.parse_args()
rows = build_inventory(Path(args.root))
_print_table(rows)
return 0
if __name__ == "__main__":
raise SystemExit(main())

View File

Some files were not shown because too many files have changed in this diff Show More