bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: restructure services layout, retire oceanus, add aether scaffolding

Browse Source 
- Move flat service manifests into structured subdirs (apps/, bootstrap-jobs/,
  repair-jobs/, migration-jobs/, validation-jobs/, node-ops/, networking/)
- Retire oneoffs/ directories across services
- Remove oceanus cluster and its host roles; add aether cluster + terraform scaffolding
- Reorganize scripts/ into ops/, render/, sync/, manual-tests/
- Add Makefile with render/validate/test/flux targets and repo-structure tests
- Update flux-system application CRs to the new paths
- Add hermes-automated-triage-24h-plan knowledge doc (+ comms mirror)
- Refresh knowledge catalogs, dashboards, vmalert rules, quality contract

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
jenkins 2026-08-05 16:21:36 -03:00
parent 7f0b7176e2
commit 0da9e4c82d
257 changed files with 13256 additions and 1821 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -5,6 +5,7 @@
__pycache__/
*.py[cod]
.pytest_cache
.ruff_cache/
.coverage
build/
test-results/
@ -12,6 +13,7 @@ artifacts/
.venv
.venv-ci
tmp/
.mainfix/
.terraform/
**/.terraform/
*.tfvars

55
Jenkinsfile vendored
View File

@ -56,6 +56,11 @@ spec:
command:
- cat
tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command:
- cat
tty: true
"""
}
}
@ -70,6 +75,8 @@ spec:
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
@ -110,6 +117,26 @@ spec:
'''
}
}
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') {
steps {
container('quality-tools') {
@ -126,6 +153,7 @@ spec:
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
)
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]}
@ -326,6 +354,33 @@ PY
esac
fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1

105
Makefile Normal file
View File

@ -0,0 +1,105 @@
# Makefile
SHELL := /usr/bin/env bash
KUSTOMIZE ?= kustomize
PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3)
TERRAFORM ?= terraform
TRIVY ?= trivy
BUILD_DIR ?= build
ATLAS_FLUX_ROOT := clusters/atlas/flux-system
AETHER_FLUX_ROOT := clusters/aether/flux-system
AETHER_TF_DIR := terraform/aether
FLUX_KUSTOMIZATION ?= flux-system
FLUX_NAMESPACE ?= flux-system
FLUX_PATH ?= $(ATLAS_FLUX_ROOT)
TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache
TRIVY_DB_FLAGS ?= --skip-db-update
.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan
help:
@printf '%s\n' \
'Targets:' \
' render Render Atlas, Aether, services, and infrastructure.' \
' render-atlas Render the Atlas Flux root.' \
' render-aether Render the Aether Flux root.' \
' render-services Render every service kustomization.' \
' render-infrastructure Render infrastructure kustomizations.' \
' validate Run render checks plus the local quality gate.' \
' dashboards Regenerate Grafana dashboard artifacts.' \
' knowledge Regenerate Atlas knowledge artifacts and comms mirror.' \
' test Run the local quality gate.' \
' flux-diff Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \
' flux-inventory Print Flux Kustomization inventory.' \
' trivy-scan Run the local Trivy filesystem scan.' \
' security-report Build IronBank report from build/trivy-fs.json.' \
' security Run Trivy scan and build the IronBank report.' \
' aether-fmt Check Terraform formatting for Aether.' \
' aether-init Initialize Aether Terraform providers.' \
' aether-validate Validate Aether Terraform after init.' \
' aether-plan Plan Aether Terraform from local variables/env.'
render: render-atlas render-aether render-services render-infrastructure
render-atlas:
$(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml
render-aether:
$(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml
render-services:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find services -name kustomization.yaml | sort)
render-infrastructure:
@set -euo pipefail; \
while IFS= read -r k; do \
d="$${k%/kustomization.yaml}"; \
printf 'render %s\n' "$$d"; \
$(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \
done < <(find infrastructure -name kustomization.yaml | sort)
validate: render test
dashboards:
$(PYTHON) scripts/render/dashboards_render_atlas.py --build
$(PYTHON) scripts/render/dashboards_render_logs.py --build
$(PYTHON) scripts/render/logging_render_observability.py --build
knowledge:
$(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms
test:
$(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR)
flux-diff:
flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH)
flux-inventory:
$(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT)
trivy-scan:
mkdir -p $(BUILD_DIR)
$(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL .
security-report:
@test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; }
$(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json
security: trivy-scan security-report
aether-fmt:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check
aether-init:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) init
aether-validate:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate
aether-plan:
$(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan

55
ci/Jenkinsfile.titan-iac
View File

@ -55,6 +55,11 @@ spec:
command:
- cat
tty: true
- name: semgrep
image: semgrep/semgrep:1.171.0
command:
- cat
tty: true
"""
}
}
@ -69,6 +74,8 @@ spec:
VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428'
QUALITY_GATE_SONARQUBE_ENFORCE = '0'
QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json'
QUALITY_GATE_SEMGREP_ENFORCE = '0'
QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json'
QUALITY_GATE_IRONBANK_ENFORCE = '1'
QUALITY_GATE_IRONBANK_REQUIRED = '0'
QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json'
@ -109,6 +116,26 @@ spec:
'''
}
}
stage('Collect Semgrep evidence') {
steps {
container('semgrep') {
sh '''#!/bin/sh
set -eu
mkdir -p build
set +e
semgrep scan --config auto --metrics=off --json --output build/semgrep.json .
semgrep_rc=$?
set -e
printf '%s\n' "${semgrep_rc}" > build/semgrep.rc
'''
}
sh '''
set -eu
semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)"
python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json
'''
}
}
stage('Collect SonarQube evidence') {
steps {
container('quality-tools') {
@ -125,6 +152,7 @@ spec:
"-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx"
)
[ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml")
[ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json")
set +e
sonar-scanner "${args[@]}" | tee build/sonar-scanner.log
rc=${PIPESTATUS[0]}
@ -325,6 +353,33 @@ PY
esac
fi
if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then
semgrep_status="$(python3 - <<'PY'
import json
from pathlib import Path
path = Path("build/semgrep-report.json")
if not path.exists():
print("missing")
raise SystemExit(0)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception: # noqa: BLE001
print("error")
raise SystemExit(0)
status = str(payload.get("status") or "").strip().lower()
print(status or "missing")
PY
)"
case "${semgrep_status}" in
ok|pass|passed|success) ;;
*)
echo "semgrep gate failed: ${semgrep_status}" >&2
fail=1
;;
esac
fi
ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}"
if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then
ironbank_required=1

3
ci/scripts/publish_test_metrics.py
View File

@ -20,6 +20,7 @@ CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS
_build_check_statuses = _quality_helpers._build_check_statuses
_combine_statuses = _quality_helpers._combine_statuses
_infer_sonarqube_status = _quality_helpers._infer_sonarqube_status
_infer_semgrep_status = _quality_helpers._infer_semgrep_status
_infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500
_infer_supply_chain_status = _quality_helpers._infer_supply_chain_status
_infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent
@ -286,6 +287,7 @@ def main() -> int:
if source_lines_over_500 <= 0:
source_lines_over_500 = _infer_source_lines_over_500(summary)
sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json"))
semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json"))
supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json"))
truthy = {"1", "true", "yes", "on"}
supply_chain_required = (
@ -298,6 +300,7 @@ def main() -> int:
workspace_line_coverage_percent=workspace_line_coverage_percent,
source_lines_over_500=source_lines_over_500,
sonarqube_report=sonarqube_report,
semgrep_report=semgrep_report,
supply_chain_report=supply_chain_report,
supply_chain_required=supply_chain_required,
)

17
ci/scripts/publish_test_metrics_quality.py
View File

@ -18,6 +18,7 @@ CANONICAL_CHECKS = [
"docs_naming",
"gate_glue",
"sonarqube",
"semgrep",
"supply_chain",
]
@ -135,12 +136,26 @@ def _infer_supply_chain_status(report: dict, required: bool) -> str:
return normalized
def _infer_semgrep_status(report: dict) -> str:
"""Infer canonical Semgrep check status from its JSON report payload."""
if not report:
return "not_applicable"
status = report.get("status")
if status is None:
blocking_findings = report.get("blocking_findings")
errors_total = report.get("errors_total")
if isinstance(blocking_findings, int) and isinstance(errors_total, int):
return "failed" if blocking_findings > 0 or errors_total > 0 else "ok"
return _normalize_result_status(str(status) if status is not None else None, default="failed")
def _build_check_statuses(
summary: dict | None,
tests: dict[str, int],
workspace_line_coverage_percent: float,
source_lines_over_500: int,
sonarqube_report: dict,
semgrep_report: dict,
supply_chain_report: dict,
supply_chain_required: bool,
) -> dict[str, str]:
@ -188,6 +203,7 @@ def _build_check_statuses(
gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable"
sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report)
semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report)
supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status(
supply_chain_report,
required=supply_chain_required,
@ -200,5 +216,6 @@ def _build_check_statuses(
"docs_naming": docs_naming_status,
"gate_glue": gate_glue_status,
"sonarqube": sonarqube_status,
"semgrep": semgrep_status,
"supply_chain": supply_chain_status,
}

159
ci/scripts/semgrep_report.py Normal file
View File

@ -0,0 +1,159 @@
#!/usr/bin/env python3
"""Build a compact Semgrep gate report from Semgrep JSON output."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
BLOCKING_SEVERITIES = {"ERROR"}
SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN")
SONAR_SEVERITY_BY_SEMGREP = {
"ERROR": "CRITICAL",
"WARNING": "MAJOR",
"INFO": "MINOR",
"UNKNOWN": "INFO",
}
def _read_json(path: Path) -> dict[str, Any]:
"""Read a JSON object from disk, returning an error-shaped payload on failure."""
if not path.exists():
return {"errors": [{"message": f"report missing: {path}"}], "results": []}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []}
if not isinstance(payload, dict):
return {"errors": [{"message": "report payload is not an object"}], "results": []}
return payload
def _finding_severity(finding: dict[str, Any]) -> str:
"""Return a normalized Semgrep finding severity."""
extra = finding.get("extra")
severity = extra.get("severity") if isinstance(extra, dict) else None
normalized = str(severity or "UNKNOWN").strip().upper()
return normalized if normalized in SEVERITIES else "UNKNOWN"
def build_report(
semgrep_payload: dict[str, Any],
*,
semgrep_exit_code: int,
blocking_severities: set[str] | None = None,
) -> dict[str, Any]:
"""Summarize Semgrep evidence into the quality-gate report contract."""
blocking = blocking_severities or BLOCKING_SEVERITIES
raw_results = semgrep_payload.get("results", [])
raw_errors = semgrep_payload.get("errors", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else []
severity_counts = {severity: 0 for severity in SEVERITIES}
blocking_findings = 0
for finding in results:
severity = _finding_severity(finding)
severity_counts[severity] += 1
if severity in blocking:
blocking_findings += 1
engine_error = semgrep_exit_code not in {0, 1} or bool(errors)
status = "failed" if engine_error or blocking_findings else "ok"
return {
"status": status,
"scanner": "semgrep",
"semgrep_rc": semgrep_exit_code,
"findings_total": len(results),
"blocking_findings": blocking_findings,
"errors_total": len(errors),
"severity_counts": severity_counts,
"blocking_severities": sorted(blocking),
}
def _line_number(value: Any, default: int = 1) -> int:
"""Return a Sonar-compatible one-indexed line number."""
try:
line = int(value)
except (TypeError, ValueError):
return default
return max(line, default)
def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None:
"""Convert one Semgrep finding into SonarQube generic issue format."""
path = str(finding.get("path") or "").strip()
if not path:
return None
extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {}
severity = _finding_severity(finding)
start = finding.get("start") if isinstance(finding.get("start"), dict) else {}
end = finding.get("end") if isinstance(finding.get("end"), dict) else {}
start_line = _line_number(start.get("line") if isinstance(start, dict) else None)
end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line)
if end_line < start_line:
end_line = start_line
return {
"engineId": "semgrep",
"ruleId": str(finding.get("check_id") or "semgrep.unknown"),
"type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL",
"severity": SONAR_SEVERITY_BY_SEMGREP[severity],
"primaryLocation": {
"message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"),
"filePath": path,
"textRange": {
"startLine": start_line,
"endLine": end_line,
},
},
}
def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
"""Build SonarQube generic external issues from Semgrep JSON output."""
raw_results = semgrep_payload.get("results", [])
results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else []
issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))]
return {"issues": issues}
def main(argv: list[str] | None = None) -> int:
"""CLI entrypoint used by Jenkins after Semgrep finishes."""
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--semgrep-json", required=True)
parser.add_argument("--exit-code", type=int, default=0)
parser.add_argument("--output", required=True)
parser.add_argument("--sonar-issues-output")
parser.add_argument(
"--blocking-severity",
action="append",
default=[],
help="Severity that should mark the report failed. Defaults to ERROR.",
)
args = parser.parse_args(argv)
blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()}
payload = _read_json(Path(args.semgrep_json))
report = build_report(
payload,
semgrep_exit_code=args.exit_code,
blocking_severities=blocking or BLOCKING_SEVERITIES,
)
output = Path(args.output)
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8")
if args.sonar_issues_output:
sonar_output = Path(args.sonar_issues_output)
sonar_output.parent.mkdir(parents=True, exist_ok=True)
sonar_output.write_text(
json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n",
encoding="utf-8",
)
return 0
if __name__ == "__main__": # pragma: no cover
raise SystemExit(main())

6
ci/scripts/supply_chain_report.py
View File

@ -131,7 +131,11 @@ def build_report(
critical = _count_vulnerabilities(trivy_payload, "CRITICAL")
high = _count_vulnerabilities(trivy_payload, "HIGH")
secrets = _count_secrets(trivy_payload)
status = "ok" if critical == 0 and secrets == 0 and not open_misconfigs else "failed"
status = (
"ok"
if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0
else "failed"
)
return {
"status": status,

274
ci/titan-iac-trivy-waivers.json
View File

@ -1,7 +1,7 @@
{
"version": 1,
"generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan",
"default_expires_at": "2026-05-22",
"default_expires_at": "2026-09-30",
"ticket": "atlas-quality-wave-k8s-hardening",
"default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.",
"misconfigurations": [
@ -15,15 +15,15 @@
"id": "KSV-0009",
"targets": [
"services/mailu/vip-controller.yaml",
"services/maintenance/k3s-agent-restart-daemonset.yaml"
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml"
]
},
{
"id": "KSV-0010",
"targets": [
"services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml"
]
},
@ -47,8 +47,8 @@
"services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
"services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml",
@ -59,16 +59,16 @@
"services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml",
"services/comms/oneoffs/bstein-force-leave-job.yaml",
"services/comms/oneoffs/comms-secrets-ensure-job.yaml",
"services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/oneoffs/mas-db-ensure-job.yaml",
"services/comms/oneoffs/mas-local-users-ensure-job.yaml",
"services/comms/oneoffs/othrys-kick-numeric-job.yaml",
"services/comms/oneoffs/synapse-admin-ensure-job.yaml",
"services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
"services/comms/oneoffs/synapse-user-seed-job.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml",
@ -83,7 +83,7 @@
"services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/oneoffs/finance-secrets-ensure-job.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml",
@ -94,63 +94,62 @@
"services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/deployment.yaml",
"services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/ldap-federation-job.yaml",
"services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
"services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/oneoffs/portal-e2e-client-job.yaml",
"services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
"services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/realm-settings-job.yaml",
"services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/user-overrides-job.yaml",
"services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml",
"services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
"services/logging/oneoffs/opensearch-ism-job.yaml",
"services/logging/oneoffs/opensearch-observability-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml",
"services/mailu/oneoffs/mailu-sync-job.yaml",
"services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml",
"services/maintenance/ariadne-deployment.yaml",
"services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/metis-deployment.yaml",
"services/maintenance/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/oauth2-proxy-metis.yaml",
"services/maintenance/oauth2-proxy-soteria.yaml",
"services/maintenance/oneoffs/ariadne-migrate-job.yaml",
"services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/pod-cleaner-cronjob.yaml",
"services/maintenance/soteria-deployment.yaml",
"services/maintenance/vault-sync-deployment.yaml",
"services/maintenance/apps/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
"services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml",
@ -188,15 +187,15 @@
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/metis-deployment.yaml",
"services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml"
]
@ -211,7 +210,7 @@
"services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml",
"services/comms/mas-secrets-ensure-rbac.yaml",
"services/maintenance/soteria-rbac.yaml"
"services/maintenance/apps/soteria-rbac.yaml"
]
},
{
@ -226,7 +225,7 @@
"services/comms/comms-secrets-ensure-rbac.yaml",
"services/comms/mas-db-ensure-rbac.yaml",
"services/jenkins/serviceaccount.yaml",
"services/maintenance/ariadne-rbac.yaml"
"services/maintenance/apps/ariadne-rbac.yaml"
]
},
{
@ -235,8 +234,8 @@
"infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml",
"infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml",
"services/jenkins/serviceaccount.yaml",
"services/maintenance/disable-k3s-traefik-rbac.yaml",
"services/maintenance/k3s-traefik-cleanup-rbac.yaml"
"services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml",
"services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml"
]
},
{
@ -266,8 +265,8 @@
"services/bstein-dev-home/backend-deployment.yaml",
"services/bstein-dev-home/chat-ai-gateway-deployment.yaml",
"services/bstein-dev-home/frontend-deployment.yaml",
"services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml",
"services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml",
"services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml",
"services/bstein-dev-home/vault-sync-deployment.yaml",
"services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml",
"services/comms/atlasbot-deployment.yaml",
@ -277,16 +276,16 @@
"services/comms/livekit-token-deployment.yaml",
"services/comms/livekit.yaml",
"services/comms/mas-deployment.yaml",
"services/comms/oneoffs/bstein-force-leave-job.yaml",
"services/comms/oneoffs/comms-secrets-ensure-job.yaml",
"services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/oneoffs/mas-db-ensure-job.yaml",
"services/comms/oneoffs/mas-local-users-ensure-job.yaml",
"services/comms/oneoffs/othrys-kick-numeric-job.yaml",
"services/comms/oneoffs/synapse-admin-ensure-job.yaml",
"services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/oneoffs/synapse-signingkey-ensure-job.yaml",
"services/comms/oneoffs/synapse-user-seed-job.yaml",
"services/comms/repair-jobs/bstein-force-leave-job.yaml",
"services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-db-ensure-job.yaml",
"services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml",
"services/comms/repair-jobs/othrys-kick-numeric-job.yaml",
"services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml",
"services/comms/bootstrap-jobs/synapse-user-seed-job.yaml",
"services/comms/pin-othrys-job.yaml",
"services/comms/reset-othrys-room-job.yaml",
"services/comms/seed-othrys-room.yaml",
@ -300,7 +299,7 @@
"services/finance/firefly-cronjob.yaml",
"services/finance/firefly-deployment.yaml",
"services/finance/firefly-user-sync-cronjob.yaml",
"services/finance/oneoffs/finance-secrets-ensure-job.yaml",
"services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml",
"services/gitea/deployment.yaml",
"services/harbor/vault-sync-deployment.yaml",
"services/health/wger-admin-ensure-cronjob.yaml",
@ -309,63 +308,62 @@
"services/jellyfin/loader.yaml",
"services/jenkins/deployment.yaml",
"services/jenkins/vault-sync-deployment.yaml",
"services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/ldap-federation-job.yaml",
"services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/mas-secrets-ensure-job.yaml",
"services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/oneoffs/portal-e2e-client-job.yaml",
"services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/oneoffs/portal-e2e-target-client-job.yaml",
"services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/realm-settings-job.yaml",
"services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/oneoffs/user-overrides-job.yaml",
"services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/ldap-federation-job.yaml",
"services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml",
"services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml",
"services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/realm-settings-job.yaml",
"services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml",
"services/keycloak/bootstrap-jobs/user-overrides-job.yaml",
"services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml",
"services/keycloak/vault-sync-deployment.yaml",
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/logging/oauth2-proxy.yaml",
"services/logging/oneoffs/opensearch-dashboards-setup-job.yaml",
"services/logging/oneoffs/opensearch-ism-job.yaml",
"services/logging/oneoffs/opensearch-observability-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml",
"services/logging/bootstrap-jobs/opensearch-ism-job.yaml",
"services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml",
"services/logging/opensearch-prune-cronjob.yaml",
"services/logging/vault-sync-deployment.yaml",
"services/mailu/mailu-sync-cronjob.yaml",
"services/mailu/mailu-sync-listener.yaml",
"services/mailu/oneoffs/mailu-sync-job.yaml",
"services/mailu/vault-sync-deployment.yaml",
"services/mailu/vip-controller.yaml",
"services/maintenance/ariadne-deployment.yaml",
"services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/k3s-agent-restart-daemonset.yaml",
"services/maintenance/metis-deployment.yaml",
"services/maintenance/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/oauth2-proxy-metis.yaml",
"services/maintenance/oauth2-proxy-soteria.yaml",
"services/maintenance/oneoffs/ariadne-migrate-job.yaml",
"services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/pod-cleaner-cronjob.yaml",
"services/maintenance/soteria-deployment.yaml",
"services/maintenance/vault-sync-deployment.yaml",
"services/maintenance/apps/ariadne-deployment.yaml",
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml",
"services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml",
"services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/networking/oauth2-proxy-metis.yaml",
"services/maintenance/networking/oauth2-proxy-soteria.yaml",
"services/maintenance/migration-jobs/ariadne-migrate-job.yaml",
"services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml",
"services/maintenance/node-ops/pod-cleaner-cronjob.yaml",
"services/maintenance/apps/soteria-deployment.yaml",
"services/maintenance/bootstrap/vault-sync-deployment.yaml",
"services/monitoring/dcgm-exporter.yaml",
"services/monitoring/jetson-tegrastats-exporter.yaml",
"services/monitoring/oneoffs/grafana-org-bootstrap.yaml",
"services/monitoring/oneoffs/grafana-user-dedupe-job.yaml",
"services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml",
"services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml",
"services/monitoring/platform-quality-gateway-deployment.yaml",
"services/monitoring/platform-quality-suite-probe-cronjob.yaml",
"services/monitoring/postmark-exporter-deployment.yaml",
@ -395,12 +393,12 @@
"services/logging/node-image-gc-rpi4-daemonset.yaml",
"services/logging/node-image-prune-rpi5-daemonset.yaml",
"services/logging/node-log-rotation-daemonset.yaml",
"services/maintenance/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/image-sweeper-cronjob.yaml",
"services/maintenance/metis-deployment.yaml",
"services/maintenance/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-nofile-daemonset.yaml",
"services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml"
"services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml",
"services/maintenance/node-ops/image-sweeper-cronjob.yaml",
"services/maintenance/apps/metis-deployment.yaml",
"services/maintenance/node-ops/node-image-sweeper-daemonset.yaml",
"services/maintenance/node-ops/node-nofile-daemonset.yaml",
"services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml"
]
}
]

2
infrastructure/modules/profiles/oceanus-validator/kustomization.yaml → clusters/aether/flux-system/applications/kustomization.yaml
View File

@ -1,4 +1,4 @@
# infrastructure/modules/profiles/oceanus-validator/kustomization.yaml
# clusters/aether/flux-system/applications/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources: []

6
clusters/aether/flux-system/kustomization.yaml Normal file
View File

@ -0,0 +1,6 @@
# clusters/aether/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- platform
- applications

2
clusters/oceanus/applications/kustomization.yaml → clusters/aether/flux-system/platform/kustomization.yaml
View File

@ -1,4 +1,4 @@
# clusters/oceanus/applications/kustomization.yaml
# clusters/aether/flux-system/platform/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources: []

3
clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml
View File

@ -6,9 +6,10 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes."
spec:
interval: 10m
path: ./services/bstein-dev-home/oneoffs/migrations
path: ./services/bstein-dev-home/migration-jobs
prune: true
force: true
sourceRef:

1
clusters/atlas/flux-system/applications/finance/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/game-stream/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/health/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/jenkins/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/mailu/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/outline/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/planka/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/quality/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/typhon/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/platform/descheduler/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization."
spec:
interval: 30m
suspend: true

1
clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work."
spec:
interval: 10m
suspend: true

1
clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml
View File

@ -6,6 +6,7 @@ metadata:
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized."
spec:
interval: 10m
suspend: true

9
clusters/oceanus/flux-system/kustomization.yaml
View File

@ -1,9 +0,0 @@
# clusters/oceanus/flux-system/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Populate when oceanus cluster is bootstrapped with Flux.
# - gotk-components.yaml
# - gotk-sync.yaml
- ../platform
- ../applications

6
clusters/oceanus/platform/kustomization.yaml
View File

@ -1,6 +0,0 @@
# clusters/oceanus/platform/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../../infrastructure/modules/base
- ../../infrastructure/modules/profiles/oceanus-validator

6
hosts/inventory/lab.yaml
View File

@ -12,7 +12,7 @@ all:
ansible_host: REPLACE_ME
ansible_user: debian
roleset: minipc_gpu
baremetal:
dedicated_hosts:
hosts:
titan-db:
ansible_host: REPLACE_ME
@ -22,7 +22,3 @@ all:
ansible_host: REPLACE_ME
ansible_user: jump
roleset: jumphost
oceanus:
ansible_host: REPLACE_ME
ansible_user: validator
roleset: validator

7
hosts/playbooks/site.yaml
View File

@ -14,13 +14,6 @@
- common
- titan_jh
- name: Configure oceanus validator host
hosts: oceanus
gather_facts: true
roles:
- common
- oceanus_base
- name: Prepare hybrid tethys node
hosts: titan-24
gather_facts: true

6
hosts/roles/oceanus_base/tasks/main.yaml
View File

@ -1,6 +0,0 @@
# hosts/roles/oceanus_base/tasks/main.yaml
---
- name: Placeholder for oceanus base configuration
ansible.builtin.debug:
msg: "Install validator prerequisites and monitoring exporters here."
tags: ['oceanus']

2
infrastructure/sources/cert-manager/letsencrypt-prod.yaml → infrastructure/core/cert-manager/letsencrypt-prod.yaml
View File

@ -1,4 +1,4 @@
# infrastructure/sources/cert-manager/letsencrypt-prod.yaml
# infrastructure/core/cert-manager/letsencrypt-prod.yaml
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:

2
infrastructure/sources/cert-manager/letsencrypt.yaml → infrastructure/core/cert-manager/letsencrypt.yaml
View File

@ -1,4 +1,4 @@
# infrastructure/sources/cert-manager/letsencrypt.yaml
# infrastructure/core/cert-manager/letsencrypt.yaml
apiVersion: cert-manager.io/v1
kind: ClusterIssuer
metadata:

4
infrastructure/core/kustomization.yaml
View File

@ -11,5 +11,5 @@ resources:
- coredns-deployment.yaml
- ntp-sync-daemonset.yaml
- workload-profiles.yaml
- ../sources/cert-manager/letsencrypt.yaml
- ../sources/cert-manager/letsencrypt-prod.yaml
- cert-manager/letsencrypt.yaml
- cert-manager/letsencrypt-prod.yaml

2
infrastructure/flux-system/kustomization.yaml
View File

@ -2,4 +2,4 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ../clusters/atlas/flux-system
- ../../clusters/atlas/flux-system

2
knowledge/INDEX.md
View File

@ -12,7 +12,7 @@ Layout
Regeneration
- Update manifests/docs, then regenerate generated artifacts:
- `python scripts/knowledge_render_atlas.py --write`
- `python scripts/render/knowledge_render_atlas.py --write`
Authoring rules
- Never include secret values. Prefer `secretRef` names or Vault paths like `kv/atlas/...`.

8
knowledge/catalog/atlas-summary.json
View File

@ -1,8 +1,8 @@
{
"counts": {
"helmrelease_host_hints": 19,
"http_endpoints": 45,
"services": 47,
"workloads": 74
"helmrelease_host_hints": 22,
"http_endpoints": 54,
"services": 70,
"workloads": 100
}
}

1103
knowledge/catalog/atlas.json
View File

File diff suppressed because it is too large Load Diff

714
knowledge/catalog/atlas.yaml
View File

File diff suppressed because it is too large Load Diff

2144
knowledge/catalog/metrics.json
View File

File diff suppressed because it is too large Load Diff

18
knowledge/catalog/runbooks.json
View File

File diff suppressed because one or more lines are too long

58
knowledge/diagrams/atlas-http.mmd
View File

@ -1,9 +1,19 @@
flowchart LR
host_agent_bstein_dev["agent.bstein.dev"]
svc_hermes_hermes["hermes/hermes (Service)"]
host_agent_bstein_dev --> svc_hermes_hermes
wl_hermes_hermes["hermes/hermes (Deployment)"]
svc_hermes_hermes --> wl_hermes_hermes
host_auth_bstein_dev["auth.bstein.dev"]
svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"]
host_auth_bstein_dev --> svc_sso_oauth2_proxy
wl_sso_oauth2_proxy["sso/oauth2-proxy (Deployment)"]
svc_sso_oauth2_proxy --> wl_sso_oauth2_proxy
host_backup_bstein_dev["backup.bstein.dev"]
svc_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Service)"]
host_backup_bstein_dev --> svc_maintenance_oauth2_proxy_soteria
wl_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Deployment)"]
svc_maintenance_oauth2_proxy_soteria --> wl_maintenance_oauth2_proxy_soteria
host_bstein_dev["bstein.dev"]
svc_bstein_dev_home_bstein_dev_home_frontend["bstein-dev-home/bstein-dev-home-frontend (Service)"]
host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_frontend
@ -111,6 +121,16 @@ flowchart LR
host_pegasus_bstein_dev --> svc_jellyfin_pegasus
wl_jellyfin_pegasus["jellyfin/pegasus (Deployment)"]
svc_jellyfin_pegasus --> wl_jellyfin_pegasus
host_quality_bstein_dev["quality.bstein.dev"]
svc_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Service)"]
host_quality_bstein_dev --> svc_quality_oauth2_proxy_sonarqube
wl_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Deployment)"]
svc_quality_oauth2_proxy_sonarqube --> wl_quality_oauth2_proxy_sonarqube
host_recovery_bstein_dev["recovery.bstein.dev"]
svc_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Service)"]
host_recovery_bstein_dev --> svc_maintenance_oauth2_proxy_metis
wl_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Deployment)"]
svc_maintenance_oauth2_proxy_metis --> wl_maintenance_oauth2_proxy_metis
host_scm_bstein_dev["scm.bstein.dev"]
svc_gitea_gitea["gitea/gitea (Service)"]
host_scm_bstein_dev --> svc_gitea_gitea
@ -141,6 +161,20 @@ flowchart LR
host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service
wl_vaultwarden_vaultwarden["vaultwarden/vaultwarden (Deployment)"]
svc_vaultwarden_vaultwarden_service --> wl_vaultwarden_vaultwarden
host_veles_bstein_dev["veles.bstein.dev"]
svc_veles_veles_frontend["veles/veles-frontend (Service)"]
host_veles_bstein_dev --> svc_veles_veles_frontend
wl_veles_veles_frontend["veles/veles-frontend (Deployment)"]
svc_veles_veles_frontend --> wl_veles_veles_frontend
svc_veles_veles_backend["veles/veles-backend (Service)"]
host_veles_bstein_dev --> svc_veles_veles_backend
wl_veles_veles_backend["veles/veles-backend (Deployment)"]
svc_veles_veles_backend --> wl_veles_veles_backend
host_wolf_bstein_dev["wolf.bstein.dev"]
svc_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Service)"]
host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf
wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"]
svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf
subgraph bstein_dev_home[bstein-dev-home]
svc_bstein_dev_home_bstein_dev_home_frontend
@ -175,6 +209,10 @@ flowchart LR
svc_finance_firefly
wl_finance_firefly
end
subgraph game_stream[game-stream]
svc_game_stream_oauth2_proxy_wolf
wl_game_stream_oauth2_proxy_wolf
end
subgraph gitea[gitea]
svc_gitea_gitea
wl_gitea_gitea
@ -183,6 +221,10 @@ flowchart LR
svc_health_wger
wl_health_wger
end
subgraph hermes[hermes]
svc_hermes_hermes
wl_hermes_hermes
end
subgraph jellyfin[jellyfin]
svc_jellyfin_pegasus
wl_jellyfin_pegasus
@ -204,6 +246,12 @@ flowchart LR
subgraph mailu_mailserver[mailu-mailserver]
svc_mailu_mailserver_mailu_front
end
subgraph maintenance[maintenance]
svc_maintenance_oauth2_proxy_soteria
wl_maintenance_oauth2_proxy_soteria
svc_maintenance_oauth2_proxy_metis
wl_maintenance_oauth2_proxy_metis
end
subgraph nextcloud[nextcloud]
svc_nextcloud_nextcloud
wl_nextcloud_nextcloud
@ -218,6 +266,10 @@ flowchart LR
svc_planka_planka
wl_planka_planka
end
subgraph quality[quality]
svc_quality_oauth2_proxy_sonarqube
wl_quality_oauth2_proxy_sonarqube
end
subgraph sso[sso]
svc_sso_oauth2_proxy
wl_sso_oauth2_proxy
@ -232,3 +284,9 @@ flowchart LR
svc_vaultwarden_vaultwarden_service
wl_vaultwarden_vaultwarden
end
subgraph veles[veles]
svc_veles_veles_frontend
wl_veles_veles_frontend
svc_veles_veles_backend
wl_veles_veles_backend
end

985
knowledge/hermes-automated-triage-24h-plan.md Normal file
View File

@ -0,0 +1,985 @@
# Hermes Automated Triage: 24-Hour Delivery Plan
- Status: planning only
- Created: 2026-08-05
- Hard delivery limit: 24 elapsed hours from execution authorization
- Priority: deadline first, then the most complete safe automation that fits
## Executive decision
The first delivery will automate one narrow, real, repeatable failure-to-repair path. It will not attempt general autonomous coding before that path works end to end.
The target demonstration is:
```text
Controlled test failure
-> Jenkins build fails
-> Ariadne detects the terminal failure automatically
-> Ariadne gathers Jenkins evidence and bounded OpenSearch logs
-> Ariadne automatically invokes the Hermes Agent
-> Hermes selects its Titan test-triage skills
-> Hermes returns a structured diagnosis and requested action
-> Ariadne validates the request against an exact allowlist
-> Ariadne creates one predefined repair Job
-> Ariadne triggers one Jenkins rebuild
-> the rebuild passes and the incident resolves
```
A second path will demonstrate safe escalation:
```text
Unsupported, ambiguous or unsafe failure
-> Hermes returns human_required
-> Ariadne publishes a triage metric
-> VictoriaMetrics and vmalert evaluate it
-> Grafana and Alertmanager provide the human-facing signal
```
At H+24 work stops. Lower-priority functionality is removed or deferred rather than extending the deadline.
## Definition of done
The delivery is fully successful when all of the following are demonstrated:
- A deliberately seeded test failure causes a Jenkins build to fail.
- Ariadne detects that exact terminal build once, preferably within two minutes.
- Ariadne assigns a stable incident ID based on the Jenkins job and build number.
- The diagnosis bundle includes the first failed stage, retained Jenkins evidence and provenance-bearing OpenSearch excerpts.
- Ariadne invokes the Hermes Agent without a human prompt.
- The invocation explicitly selects the Titan test-triage skill.
- Hermes returns schema-valid facts, inferences, classification, confidence and action request.
- Ariadne accepts only the predefined demo action and executes it once.
- A predefined repair Job corrects only the isolated demo fixture.
- Ariadne triggers one rebuild with failure seeding disabled.
- The rebuilt job passes and Ariadne records the incident as resolved.
- An unsupported classification produces a human-required metric and no mutation.
- No secret values are stored in evidence, model prompts, responses or logs.
- No path can automatically write to `main`, reconcile Flux, or execute model-supplied commands.
- A single Flux-managed setting can disable all automatic actions immediately.
Target demo timings are:
- Failure detection: less than two minutes.
- Evidence collection and Hermes response: less than three minutes.
- Repair launch and rebuild request: less than two minutes.
- Complete failure-to-green loop: less than ten minutes, excluding image pulls during a cold cluster start.
## Current system context
This section records the facts needed to execute the plan without relying on prior conversation.
### Jenkins and the homegrown suites
Jenkins runs the homegrown repositories on ephemeral Kubernetes agent pods. The active quality scope includes Ariadne, Metis, Ananke, Atlasbot, Pegasus, Soteria, titan-iac, bstein-dev-home, data-prepper and Lesavka, with additional build/test jobs for projects such as Arcanagon, Typhon and Veles.
The normalized quality evidence includes, as applicable:
```text
build
-> style/docs
-> LOC/naming
-> coverage
-> tests
-> gate glue
-> SonarQube
-> Semgrep
-> Trivy/supply-chain evidence
-> gate enforcement
-> image build and Harbor push
```
Jenkins retains build metadata, console output, stage results, JUnit, coverage and quality reports. These retained artifacts are the authoritative source for the exact failed test or gate.
### Logging and OpenSearch
OpenSearch does not collect logs by itself. Fluent Bit tails Kubernetes container logs and writes them directly into `kube-*` indices. It also writes node journal records into `journald-*` indices.
The relevant path is:
```text
Kubernetes pod stdout/stderr
-> Fluent Bit DaemonSet
-> OpenSearch kube-* indices
```
Jenkins-retained console logs and OpenSearch pod logs are separate evidence paths. Jenkins agent command output must not be assumed to exist in CRI container logs. The demo must use Jenkins artifacts for the exact assertion and must ensure that a dedicated demo test-runner or related workload writes a correlated incident message to pod stdout/stderr for OpenSearch.
Data Prepper is not part of the ordinary log path. It receives OpenTelemetry traces, transforms them and writes trace/service-map indices into OpenSearch. It is irrelevant to the Jenkins log integration except as another separately tested workload.
Relevant IaC sources include:
- `services/logging/fluent-bit-helmrelease.yaml`
- `services/logging/opensearch-observability-objects.yaml`
- `services/logging/Jenkinsfile.data-prepper`
### Metrics and human-facing status
Jenkins publishers and runtime probes send normalized quality metrics to `platform-quality-gateway`. VictoriaMetrics scrapes Prometheus-compatible endpoints, vmalert evaluates rules, Grafana displays the Atlas Testing dashboards, and Alertmanager owns notification delivery.
The intended human-escalation path is therefore:
```text
Ariadne triage metric
-> VictoriaMetrics
-> vmalert
-> Alertmanager notification and Grafana status
```
Hermes does not send an alert "to Grafana." Grafana visualizes the metric and alert state.
Relevant IaC sources include:
- `services/monitoring/platform-quality-gateway-deployment.yaml`
- `services/monitoring/platform-quality-suite-probe-cronjob.yaml`
- `services/monitoring/vmalert-atlas-availability.yaml`
- `services/monitoring/dashboards/atlas-testing.json`
- `services/quality/sonarqube-exporter-deployment.yaml`
### Ariadne today
Ariadne is the deterministic automation and evidence-collection service in the `maintenance` namespace. Its current deployment has access to:
- Jenkins API credentials and `JENKINS_BASE_URL`.
- VictoriaMetrics through `ARIADNE_VM_URL`.
- OpenSearch through `OPENSEARCH_URL`.
- Kubernetes through its service account.
- The local model through `ARIADNE_TESTING_TRIAGE_MODEL_URL`.
The deployment schedules Jenkins build-weather collection every ten minutes and testing triage every fifteen minutes. Those intervals are too slow for a smooth demo and should be reduced only for the allowlisted demo path.
Ariadne's current OpenSearch integration manages index retention. It does not currently retrieve historical log content for testing diagnosis.
Ariadne currently points model-assisted testing triage at the Hermes Ollama service. Calling Ollama uses the same Qwen model but bypasses the Hermes Agent, its tool loop, skills, memory and approval model. This is not equivalent to invoking Hermes.
Ariadne already has patterns for launching a platform quality probe Job and for performing Jenkins workspace maintenance. These patterns should be reused for the predefined demo repair rather than introducing a general job-execution system.
Relevant IaC source:
- `services/maintenance/apps/ariadne-deployment.yaml`
Relevant application repository:
- Gitea: `bstein/Ariadne`
- Local checkout currently at `/home/brad/Development/Ariadne`
### Hermes today
Hermes runs as a single read-only triage agent in the `hermes` namespace. Its current configuration provides:
- Hermes Agent API on port `8642`.
- Hermes dashboard on port `9119`.
- Local Qwen 2.5 `7b-instruct-q4_0` inference through Hermes Ollama.
- Ariadne, Jenkins, VictoriaMetrics, Gitea and Grafana base URLs.
- `kubectl` installed as a read-only cluster inspection tool.
- A persistent home PVC containing memories and runtime-installed skills.
Hermes does not currently receive `OPENSEARCH_URL`. In the desired design it does not need direct OpenSearch access: Ariadne will provide bounded, sanitized excerpts in the diagnosis bundle.
Hermes's Flux-managed instructions say to start with Ariadne as the source of truth, then use Jenkins, quality metrics, Flux, Grafana and Kubernetes read-only evidence. Its RBAC allows reads of workloads, events, pod logs, services, storage objects and Flux resources. It cannot read Kubernetes Secret values.
Hermes explicitly denies or lacks permission for actions such as:
- `kubectl apply`, delete, patch, scale, cordon, drain or rollout restart.
- Flux suspend, resume or reconcile.
- Vault secret reads.
- Kubernetes Secret reads.
- Jenkins rebuilds.
- Git writes.
This boundary remains in place. Hermes will request actions; Ariadne will validate and execute exact predefined actions.
Relevant IaC sources include:
- `services/hermes/deployment.yaml`
- `services/hermes/configmap.yaml`
- `services/hermes/rbac.yaml`
### Hermes's current Atlas skills
A read-only inventory of the live Hermes PVC on 2026-08-05 showed the Atlas triage entry skills:
- `triage-titan-test-failures`
- `triage-atlas-service-health`
- `tune-atlas-alerts`
- `master-hermes-on-atlas`
The test-triage skill pack includes:
- `titan-triage-orchestrator`
- `jenkins-retained-evidence`
- `platform-quality-metrics`
- `kubernetes-readonly-failure-classifier`
- `flux-git-change-correlation`
- `grafana-metric-provenance`
- `triage-evidence-reporting`
- `approved-triage-actions`
- `soteria-backup-health`
The orchestrator already defines the needed evidence order: Ariadne first, then the smallest relevant specialist, followed by a fact-versus-inference report. The demo invocation should explicitly request `$triage-titan-test-failures`; a new general triage framework is unnecessary.
These skills currently live on the runtime PVC and are not all declared in `titan-iac`. The demo should not depend on mutating the live PVC manually. Any essential prompt or output contract must be supplied by Ariadne or added through Flux-managed Hermes configuration.
### Current gaps
The desired closed loop does not exist today because:
- Ariadne does not query OpenSearch for triage log evidence.
- Ariadne calls Ollama rather than the Hermes Agent API.
- There is no shared Ariadne-to-Hermes API authentication contract.
- There is no schema for a Hermes triage response.
- There is no allowlisted Hermes-to-Ariadne action request.
- Ariadne does not automatically execute a Hermes-selected repair.
- There is no incident-level idempotency preventing duplicate repair attempts.
- There is no dedicated human-required triage metric and alert.
- The current architecture chart compresses Hermes's internal trigger, routing and decision steps into a single box.
## Worktree and source-control safety
The existing working directories must not be used as clean implementation bases.
As of 2026-08-05:
- `/home/brad/Development/titan-iac` has extensive unrelated modified, deleted and untracked user work, including an ongoing repository-layout migration and untracked Mermaid files.
- `/home/brad/Development/Ariadne` is dirty on branch `codex/ariadne-metrics-hotfix`.
Execution must:
1. Inspect the actual revisions deployed by Flux and Jenkins.
2. Fetch without altering either existing worktree.
3. Create dedicated clean Git worktrees for the demo branches.
4. Never reset, clean, stash, rebase or overwrite either existing dirty worktree.
5. Keep Ariadne application changes and titan-iac/Flux changes in separately reviewable commits.
6. Bring over only the intentional planning/diagram change when appropriate.
Suggested isolated paths are:
```text
/home/brad/Development/_worktrees/ariadne-hermes-triage-demo
/home/brad/Development/_worktrees/titan-iac-hermes-triage-demo
```
The implementation branch point must be the current authoritative primary/deployed revision, not an assumed local branch.
## Deadline-driven scope
### Required scope
- One dedicated demo Jenkins job.
- One deterministic failure signature.
- One isolated demo fixture.
- One bounded OpenSearch search contract.
- One automatic Hermes invocation.
- One structured decision contract.
- One exact repair action.
- One rebuild attempt.
- One human-required metric and alert path.
- One complete source-level Mermaid update.
### Explicit non-goals
- General conversational OpenSearch access.
- Arbitrary index selection or unbounded log retrieval.
- General autonomous repository editing.
- Direct commits or merges to `main`.
- Model-generated Kubernetes manifests or commands.
- Automatic Flux reconciliation.
- Automatic SonarQube, Semgrep or Trivy waivers.
- Secret inspection.
- Multiple repair retries.
- Multiple repositories or failure signatures before the primary loop is green.
- A permanent generalized incident-management platform.
## Demo design
### Isolated test fixture
Use a dedicated `hermes-triage-demo` namespace, Jenkins job and fixture. The preferred deterministic design is:
1. A small PVC stores a single state value such as `healthy` or `unhealthy`.
2. The demo is explicitly armed through a Jenkins parameter such as `SEED_FAILURE=true`.
3. A dedicated Kubernetes test-runner Job reads the fixture.
4. When the fixture is unhealthy, it writes a structured incident line to stdout/stderr and exits nonzero.
5. Jenkins waits for the Job, records the test failure and archives a small JUnit result.
6. Fluent Bit forwards the test-runner Job output to OpenSearch.
7. The test-runner Job name and message include the Jenkins job/build incident ID.
Example structured log:
```json
{
"event": "hermes_demo_test_failure",
"incident_id": "hermes-triage-demo/42",
"classification_hint": "demo_fixture_unhealthy",
"message": "expected fixture state healthy; found unhealthy"
}
```
The Jenkins rebuild must set `SEED_FAILURE=false`, otherwise every rebuild would deliberately recreate the failure.
The namespace isolates the test-runner permissions and prevents a demo repair from touching production workloads.
### Predefined repair Job
Ariadne will contain one exact action mapping:
```text
repair_demo_fixture
-> create hermes-demo-repair-<incident> Job
-> mount only the demo fixture PVC
-> write healthy state
-> emit a structured repair result
-> exit
```
Hermes cannot supply the image, command, namespace, PVC, labels or arbitrary parameters. It can request only the action ID.
After the Job succeeds, Ariadne requests one Jenkins rebuild with failure seeding disabled.
### Why a repair Job is the critical path
A source-code test failure generally requires a Git change, but safe autonomous code editing requires repository credentials, checkout isolation, patch generation, tests, branch creation and PR lifecycle handling. Making that the primary demo would threaten the 24-hour limit.
The fixture repair demonstrates a real closed loop:
- Test failure is real.
- Jenkins and OpenSearch evidence are real.
- Hermes classification is real.
- Hermes action selection is real.
- Ariadne authorization and execution are real.
- The repair and green rebuild are real.
Autonomous Gitea PR creation remains a stretch goal that begins only after the required loop is stable.
## Evidence contract
### Incident identity
Use a stable ID:
```text
<jenkins-job>/<build-number>
```
Every log line, bundle, Hermes request, action, repair Job and metric must carry that ID.
### Jenkins evidence
The bundle should include:
- Job and build number.
- Branch and commit when available.
- Build URL.
- Start and end timestamps.
- Terminal result.
- First failed stage.
- Console tail.
- Failed JUnit cases.
- Selected quality reports when relevant.
- Evidence source and retrieval timestamp.
### OpenSearch evidence
The demo query should be intentionally fixed and bounded:
- Indices: `kube-*` only.
- Time range: build start minus five minutes through build end plus five minutes.
- Namespace: `hermes-triage-demo`, plus `jenkins` only if it adds useful context.
- Correlation: exact incident ID first; namespace and time fallback second.
- Size: at most 50 records initially, never more than 100.
- Fields: timestamp, namespace, pod, container, log/message and selected safe labels.
- Timeout: approximately five seconds.
- Response byte cap: small enough to prevent prompt flooding.
- Pagination: none in the demo.
Each excerpt must retain:
- Index or source identifier.
- Timestamp.
- Namespace.
- Pod.
- Container.
- Sanitized message.
Basic sanitization must remove or mask:
- `Authorization` headers.
- Bearer tokens.
- Cookies and session IDs.
- Common password, token and secret assignments.
- Private key blocks.
This is demo-grade sanitization, not a claim of comprehensive data-loss prevention. Queries remain restricted to the isolated namespace wherever possible.
### Ariadne diagnosis bundle
Add a `log_evidence` section without replacing the existing bundle fields:
```json
{
"incident_id": "hermes-triage-demo/42",
"generated_at": "...",
"jenkins": {},
"quality_metrics": {},
"runtime_context": {},
"log_evidence": {
"query_window": {},
"records": [],
"truncated": false,
"error": null
}
}
```
Empty or unavailable OpenSearch results must be represented explicitly. They must not block Jenkins-based diagnosis.
## Automatic Hermes invocation
### Required behavior
Ariadne must call the Hermes Agent API, not only the Ollama completion endpoint. The invocation prompt should be stable and small:
```text
Use $triage-titan-test-failures.
Analyze incident <incident-id>.
Treat the attached Ariadne bundle as the source of truth.
Identify the first enforced failure.
Distinguish facts from inference.
Return only the required triage response schema.
Do not perform mutations.
```
The bundle is attached as structured content. Ariadne records the Hermes request ID or session ID for audit and idempotency.
### Authentication
No API key may be committed to Git.
The preferred path is:
1. Generate one internal Hermes API key.
2. Store it in an approved Vault path.
3. Inject it into Hermes as `API_SERVER_KEY`.
4. Inject it into Ariadne as `HERMES_API_KEY`.
5. Restrict network access to the required namespaces where practical.
The exact Hermes Agent API path and authentication behavior must be verified in the first hour. The current Hermes init container generates a key in its persistent `.env` if none exists; the integration must not scrape that file or copy it into Git.
### Response schema
Hermes must return a bounded machine-readable response:
```json
{
"incident_id": "hermes-triage-demo/42",
"classification": "known_demo_fixture_failure",
"confidence": 0.95,
"facts": [
{
"statement": "...",
"source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea",
"reference": "..."
}
],
"inferences": [],
"first_failed_gate": "tests",
"requested_action": {
"type": "run_ariadne_job",
"id": "repair_demo_fixture"
},
"human_required": false,
"reason": ""
}
```
Invalid JSON, unknown fields, mismatched incident IDs, low confidence and unknown actions all become human-required results.
## Detection and deduplication
For the demo, Ariadne should detect only an allowlisted job such as `hermes-triage-demo`. Reducing the global scheduler interval must not cause automatic action against unrelated failures.
Suggested configuration:
```text
ARIADNE_HERMES_AUTOTRIAGE_ENABLED=true
ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST=hermes-triage-demo
ARIADNE_HERMES_AUTOTRIAGE_INTERVAL_SECONDS=60
ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false
ARIADNE_HERMES_ALLOWED_ACTIONS=repair_demo_fixture
ARIADNE_HERMES_MIN_CONFIDENCE=0.85
ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT=1
```
The remediation flag remains false until the evidence-and-Hermes path passes in observe mode.
Ariadne must persist or otherwise reliably track:
- Incident observed.
- Hermes invocation started/completed.
- Action requested/accepted/rejected.
- Repair Job created/completed.
- Rebuild requested.
- Final build result.
The same Jenkins build must never invoke a second repair after its action record exists.
## Action authorization
Ariadne may accept the demo action only when:
- The Jenkins build is terminal and failed.
- The job is allowlisted.
- The incident ID matches the current record.
- The classification is exactly `known_demo_fixture_failure`.
- The action is exactly `repair_demo_fixture`.
- Confidence is at least the configured threshold.
- The Jenkins/JUnit or log evidence contains the expected failure signature.
- No action has already run for the incident.
- Automatic remediation is enabled.
Hermes must not provide:
- Shell commands.
- Kubernetes YAML.
- Container images.
- Namespaces.
- PVC names.
- Git commands.
- Retry counts.
These remain hardcoded, tested Ariadne behavior.
## Failure-class policy
| Failure class | Automatic behavior in the 24-hour delivery |
|---|---|
| Known demo fixture failure | Run `hermes-demo-repair`, then rebuild once |
| Stale runtime quality telemetry | Optionally run the existing platform quality suite probe |
| Clearly transient demo Jenkins failure | At most one allowlisted rebuild if time permits |
| Ordinary source/test defect | Diagnose and recommend; human required |
| SonarQube failure | Human required; never waive automatically |
| Semgrep failure | Human required; never waive automatically |
| Trivy/supply-chain failure | Human required; never waive automatically |
| Kubernetes or Flux defect | Read-only diagnosis; human-required repo/Flux change |
| Missing or stale decisive evidence | Human required |
| Authentication, Secret or Vault issue | Human required |
| Storage-destructive or hardware issue | Human required |
| Unknown class or low confidence | Human required |
## Triage metrics and escalation
Ariadne should expose a small stable metric set, for example:
```text
ariadne_hermes_triage_incident{job="...",build="...",status="detected|diagnosed|repairing|resolved|human_required"} 1
ariadne_hermes_triage_action_total{action="...",result="accepted|rejected|success|failed"} 1
ariadne_hermes_triage_last_success_timestamp_seconds ...
ariadne_hermes_triage_duration_seconds ...
```
Avoid unbounded labels such as full error messages, commit messages or model text.
The minimum alert is:
```text
human_required incident remains active for a short demo-safe interval
-> vmalert fires
-> Alertmanager handles notification
-> Grafana links to Jenkins and the Ariadne incident report
```
If dashboard changes threaten the deadline, the metric and alert rule take precedence over a polished panel.
## 24-hour execution schedule
### H+0 to H+1: establish authoritative state
- Confirm the deployed Flux revision and Ariadne image/source revision.
- Confirm the Jenkins job definitions and branch behavior.
- Create clean dedicated worktrees without touching existing dirty worktrees.
- Verify the Hermes Agent API route, request format and authentication.
- Verify Ariadne-to-Hermes network reachability.
- Verify Ariadne-to-OpenSearch reachability.
- Verify the maintenance service account can create/watch the intended isolated Job or identify the minimal RBAC change.
- Freeze the exact demo job, namespace, failure signature and repair action.
Gate at H+1:
- No additional repositories or failure classes enter scope.
- If the Hermes Agent API path is still unknown, allocate no more than one additional hour before activating the fallback.
### H+1 to H+4: bounded OpenSearch evidence
- Add the fixed OpenSearch query client to Ariadne.
- Filter by the demo namespace, incident ID and build time window.
- Add response size, count and timeout limits.
- Add basic sanitization.
- Add `log_evidence` to the diagnosis bundle.
- Add unit tests for success, empty result, timeout, malformed payload, truncation and sanitization.
- Perform one live read-only query.
Compromise at H+4:
- Use a single fixed `kube-*` query rather than a general query builder.
- Retain raw chronological excerpts rather than semantic grouping.
- OpenSearch failure becomes explicit missing evidence and does not block Jenkins diagnosis.
### H+4 to H+7: automatic Hermes invocation
- Configure the shared internal API credential through Vault.
- Implement Ariadne's Hermes Agent client.
- Invoke the explicit Titan triage skill with the incident bundle.
- Require and validate structured JSON.
- Add a bounded timeout and one retry.
- Record request/session ID and duration.
- Deduplicate by incident ID.
Compromise at H+7:
- If the Hermes Agent API remains blocked, call the existing Hermes Ollama endpoint with the triage workflow embedded in the prompt.
- Clearly label this fallback as model-assisted Ariadne triage, not full Hermes Agent skill execution.
- Preserve the same response schema so the rest of the loop remains usable.
### H+7 to H+10: decision and authorization contract
- Freeze the response schema.
- Implement incident, class, confidence and action validation.
- Add idempotency and maximum-action checks.
- Add the global action kill switch.
- Reject arbitrary action content.
- Emit requested, accepted and rejected action metrics.
- Test invalid JSON, mismatched incident, low confidence, unknown action and duplicate incident.
Gate at H+10:
- Response schema and action list freeze.
- No additional automatic action type is added.
### H+10 to H+14: isolated repair loop
- Add the isolated namespace, fixture and test-runner resources.
- Add the dedicated Jenkins demo job or pipeline definition.
- Ensure the test-runner emits the incident ID to pod stdout/stderr.
- Add the predefined repair Job builder/executor to Ariadne.
- Wait for repair completion.
- Trigger one Jenkins rebuild with `SEED_FAILURE=false`.
- Record the rebuilt job and final status.
- Stop after one failed repair or rebuild.
Compromise at H+14:
- If repair execution is not repeatable, disable remediation.
- Ship automatic detection, evidence, Hermes diagnosis and human escalation.
- Do not consume remaining time attempting general repair logic.
### H+14 to H+16: human-required signal
- Add incident/action metrics.
- Add one vmalert rule for unresolved human-required incidents.
- Add Jenkins and Ariadne report links where the current dashboard model supports them.
- Confirm the signal clears on a resolved incident.
Compromise at H+16:
- Use the existing Atlas Testing dashboard and direct VictoriaMetrics query if a new panel is not ready.
- Preserve the metric and alert rule over dashboard polish.
### H+16 to H+18: Flux integration and documentation
- Add only required environment, Vault injection, network and RBAC changes.
- Keep the repair Job namespace and permissions isolated.
- Add the automatic-action kill switch to Flux-managed configuration.
- Rewrite `mermaid/TestAutomation.mmd` to show the actual trigger, evidence collection, Agent invocation, skill routing, decision schema, action gate, Ariadne execution and human escalation.
- Update this plan with final deviations and deferred hardening.
- Do not render SVG output unless separately requested.
### H+18 to H+21: validation and first rehearsal
- Run focused and complete relevant Ariadne tests.
- Run repository quality checks required by its Jenkins pipeline.
- Run `kustomize build` for each affected titan-iac kustomization.
- Run client-side dry-run validation without mutating the cluster.
- Deliver through Git and Flux only.
- Seed the controlled failure and observe the full loop.
- Fix only critical-path failures.
Gate at H+21:
- Feature freeze.
- No autonomous-code stretch work starts after this point.
### H+21 to H+23: repeatability rehearsal
Run these scenarios:
1. Known fixture failure -> repair Job -> green rebuild.
2. Unsupported classification -> human-required signal and no repair.
3. Duplicate observation -> no duplicate repair.
4. OpenSearch timeout -> Jenkins-based diagnosis continues.
5. Hermes timeout or invalid response -> human-required signal.
6. Remediation kill switch disabled -> diagnosis only.
Record timings, incident IDs, action count and final Jenkins status.
### H+23 to H+24: hard stop
- Make no new feature changes.
- Confirm the kill switch and rollback path.
- Capture the final working scope and known limitations.
- Ship the highest completed fallback tier.
- Stop at H+24.
## Deadline compromise ladder
When time is at risk, remove functionality in this order:
1. Drop autonomous Gitea/code editing.
2. Limit remediation to `repair_demo_fixture` only.
3. Drop transient rebuild and optional suite-probe actions.
4. Drop additional failure classifications.
5. Replace dynamic OpenSearch construction with one fixed query.
6. Reuse the existing Grafana dashboard instead of adding a polished panel.
7. Fall back from Hermes Agent API to the same Qwen model with the triage skill procedure embedded in the prompt.
8. Disable automatic remediation but keep automatic diagnosis.
9. Preserve detection, evidence and human-required alerting as the minimum shippable system.
The deadline is not extended to preserve a lower-priority feature.
## Delivery tiers at H+24
### Tier A: target
- Automatic detection.
- Jenkins and OpenSearch evidence.
- Hermes Agent skill execution.
- Structured diagnosis.
- One allowlisted repair Job.
- One rebuild.
- Human escalation for unsupported failures.
### Tier B: acceptable compromise
- Automatic detection.
- Jenkins and OpenSearch evidence.
- Hermes Agent skill execution.
- Structured diagnosis and recommendation.
- Human escalation.
- Automatic repair disabled.
### Tier C: minimum shippable
- Automatic detection.
- Jenkins and bounded OpenSearch evidence.
- Ariadne calls Qwen with the triage workflow embedded.
- Structured recommendation.
- Human-required metric/alert.
No tier may silently claim Hermes Agent execution when only Ollama was invoked.
## Testing strategy
### Ariadne unit tests
- OpenSearch query bounds and field selection.
- Correlation by incident ID and time.
- Sanitization and truncation.
- Empty, unavailable and malformed OpenSearch responses.
- Hermes request construction.
- Hermes response schema validation.
- Confidence threshold.
- Action allowlist.
- Incident mismatch rejection.
- Idempotency and maximum action count.
- Repair Job construction from hardcoded values.
- Jenkins rebuild parameter handling.
- Metrics with bounded label cardinality.
### Integration tests
- Mocked Jenkins failure plus mocked OpenSearch evidence.
- Mocked Hermes accepted action.
- Mocked unsupported classification.
- Repair Job success/failure.
- Jenkins rebuild request success/failure.
- Hermes timeout and malformed response.
- OpenSearch timeout while Jenkins evidence remains usable.
### Live read-only preflight
- Query one known `kube-*` time window.
- Confirm the expected Kubernetes field names.
- Confirm Hermes API status and authentication behavior.
- Confirm Jenkins GET access.
- Confirm VictoriaMetrics can query Ariadne metrics.
### End-to-end demonstration
- Arm the demo failure deliberately.
- Confirm one failed Jenkins build.
- Confirm one Ariadne incident.
- Confirm correlated OpenSearch excerpts.
- Confirm the expected Hermes skill and response.
- Confirm one repair Job.
- Confirm one rebuild.
- Confirm green result and resolved metric.
- Repeat once to prove the path is not a one-off race.
## Flux and validation rules
All persistent cluster changes must be represented in Git and reconciled through Flux. No manual `kubectl apply`, patch or ad hoc cluster edit is part of the delivery.
For every affected titan-iac kustomization:
```text
kustomize build <path>
kubectl apply --server-side --dry-run=client -k <path>
flux diff kustomization <name> --path <path>
```
Actual reconciliation occurs only after the intended commits are available to Flux.
Application changes in Ariadne must run the focused tests for new services plus its relevant full test/quality suite. Exact commands should follow the authoritative branch's Jenkinsfile and test configuration rather than assumptions from the currently dirty local checkout.
## Security and safety boundary
- Store all shared credentials in Vault; never Git.
- Restrict automatic triage to an explicit Jenkins job allowlist.
- Restrict OpenSearch to `kube-*`, time bounds and isolated namespaces.
- Sanitize evidence before model use.
- Never send entire indices, artifacts or unrestricted logs to the model.
- Do not add Secret-read permission to Hermes.
- Do not grant Hermes Kubernetes mutation.
- Do not allow model-supplied shell commands or manifests.
- Keep action definitions in tested Ariadne code.
- Permit one action and one rebuild per incident.
- Require human intervention on low confidence or missing decisive evidence.
- Keep a default-off remediation flag until observe mode passes.
## Rollback and kill switches
Immediate behavioral rollback:
```text
ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false
```
Further rollback steps are:
1. Disable automatic Hermes invocation while retaining normal Ariadne schedules.
2. Restore the original Jenkins-weather and testing-triage intervals.
3. Disable the demo Jenkins job.
4. Remove or suspend only the isolated demo resources through Git/Flux.
5. Revoke the shared Hermes API key in Vault.
6. Leave ordinary Jenkins, Fluent Bit, OpenSearch, VictoriaMetrics and Hermes read-only behavior untouched.
The repair action must be safe to run twice even though idempotency should prevent the second run.
## Repository touchpoints
Expected Ariadne application changes:
- Bounded OpenSearch evidence client.
- Diagnosis-bundle enrichment.
- Hermes Agent API client.
- Response schema and validation.
- Incident deduplication/state.
- Allowlisted repair action executor.
- Jenkins rebuild request.
- Triage metrics.
- Unit and integration tests.
Expected titan-iac changes:
- Ariadne environment and Vault injection.
- Hermes shared API-key configuration if required.
- Isolated demo namespace/PVC/RBAC/test-runner resources.
- Predefined repair Job permissions.
- Demo Jenkins JCasC/pipeline definition.
- vmalert rule and minimal Grafana integration.
- `mermaid/TestAutomation.mmd` update.
Do not modify unrelated workloads or fold repository-layout migrations into these changes.
## Optional autonomous-code stretch
This begins only if Tier A passes end to end before H+16 and at least eight hours remain.
The only acceptable stretch is a restricted Gitea PR workflow:
```text
Hermes proposes patch
-> Ariadne validates repository and base branch
-> dedicated service creates a demo branch/PR
-> Jenkins validates the branch
-> human reviews and merges
```
Restrictions:
- One demo repository.
- Dedicated branch prefix.
- No push to `main` or `master`.
- No automatic merge.
- Separate least-privilege Gitea token in Vault.
- Patch size and file-path limits.
- Jenkins validation required.
This stretch is dropped immediately if it threatens rehearsal or the H+24 stop.
## Required architecture-document update
After behavior is known, `mermaid/TestAutomation.mmd` must show these internal steps rather than compressing Hermes into one box:
```text
Trigger
-> terminal Jenkins failure or human request
Detection
-> Ariadne allowlisted polling and incident deduplication
Evidence
-> Jenkins metadata/artifacts
-> bounded OpenSearch excerpts
-> VictoriaMetrics
-> Kubernetes/Flux/Gitea/Grafana context when relevant
Agent invocation
-> Ariadne calls Hermes Agent API
-> explicit triage-titan-test-failures skill
-> local Qwen model through Hermes Ollama
Decision
-> first enforced failure
-> facts versus inference
-> specialist skill routing
-> confidence and action schema
Authorization
-> Ariadne allowlist, incident match, confidence, idempotency and kill switch
Response
-> predefined repair Job and one rebuild
-> or human-required metric and alert
```
The chart must visually distinguish current behavior, newly implemented demo behavior and deferred capabilities. It should continue to keep Jenkins-retained logs separate from OpenSearch pod logs and should not imply direct Hermes OpenSearch access.
## Final handoff record
At H+24, append a short result section containing:
- Delivered tier.
- Exact commits and deployed revisions.
- Demo job/build IDs.
- Incident ID.
- Evidence query window.
- Hermes request/session ID.
- Requested and executed action.
- Repair Job name.
- Rebuild result.
- Alert result.
- Total end-to-end duration.
- Compromises activated.
- Deferred hardening.
- Kill-switch and rollback verification.
The handoff must state plainly whether the full Hermes Agent or the Ollama fallback performed the diagnosis.

9
knowledge/metis.md
View File

@ -1,12 +1,14 @@
# Metis (node recovery)
## Node classes (current map)
- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)
- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent)
- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)
- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)
- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)
- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)
- amd64 agents: titan-22/24 (Debian 13, k3s agent)
- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, future titan-20/21 (when added), plus any newcomers.
- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels)
- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.
## Longhorn disk UUIDs (critical nodes)
- titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)
@ -17,10 +19,9 @@
## Metis repo (~/Development/metis)
- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).
- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).
- `AGENTS.md` in repo is untracked and holds raw notes.
## Next implementation steps
- Add per-class golden image refs and checksums (Harbor or file://) when ready.
- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.
- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.
- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.
- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited.

4
knowledge/runbooks/ci-gitea-jenkins.md
View File

@ -3,7 +3,7 @@ title: "CI: Gitea → Jenkins pipeline"
tags: ["atlas", "ci", "gitea", "jenkins"]
owners: ["brad"]
entrypoints: ["scm.bstein.dev", "ci.bstein.dev"]
source_paths: ["services/gitea", "services/jenkins", "scripts/jenkins_cred_sync.sh", "scripts/gitea_cred_sync.sh"]
source_paths: ["services/gitea", "services/jenkins", "scripts/sync/jenkins_cred_sync.sh", "scripts/sync/gitea_cred_sync.sh"]
---
# CI: Gitea → Jenkins pipeline
@ -14,7 +14,7 @@ Atlas uses Gitea for source control and Jenkins for CI. Authentication is via Ke
## Where it is configured
- Gitea manifests: `services/gitea/`
- Jenkins manifests: `services/jenkins/`
- Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh`
- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh`
## What users do (typical flow)
- Create a repo in Gitea.

20
knowledge/runbooks/cluster-power-recovery.md
View File

@ -14,8 +14,8 @@ Bootstrapping risk to remember
- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.
Script
- `scripts/cluster_power_recovery.sh`
- `scripts/cluster_power_console.sh`
- `scripts/ops/cluster_power_recovery.sh`
- `scripts/ops/cluster_power_console.sh`
- Modes:
- `prepare`
- `shutdown`
@ -26,21 +26,21 @@ Script
Dry-run examples
- Shutdown preview:
- `scripts/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`
- `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`
- Startup preview:
- `scripts/cluster_power_recovery.sh startup`
- `scripts/ops/cluster_power_recovery.sh startup`
- Harbor seed preview:
- `scripts/cluster_power_recovery.sh harbor-seed`
- `scripts/ops/cluster_power_recovery.sh harbor-seed`
Execute examples
- Prepare helper image on every node:
- `scripts/cluster_power_recovery.sh prepare --execute`
- `scripts/ops/cluster_power_recovery.sh prepare --execute`
- Seed Harbor runtime images onto `titan-05` from the control-host bundle:
- `scripts/cluster_power_recovery.sh harbor-seed --execute`
- `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`
- Planned shutdown:
- `scripts/cluster_power_recovery.sh shutdown --execute`
- `scripts/ops/cluster_power_recovery.sh shutdown --execute`
- Planned startup (canonical branch):
- `scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main`
- `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`
Manual remote console examples
- Canonical operator hosts:
@ -129,7 +129,7 @@ Operational notes
- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.
- Harbor is reconciled after the first critical stateful services.
- Harbor bootstrap is now designed around a control-host bundle:
- Build the Harbor bundle locally with `scripts/build_harbor_bootstrap_bundle.sh`.
- Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.
- Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.
- Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.
- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.

2
knowledge/runbooks/kb-authoring.md
View File

@ -3,7 +3,7 @@ title: "KB authoring: what to write (and what not to)"
tags: ["atlas", "kb", "runbooks"]
owners: ["brad"]
entrypoints: []
source_paths: ["knowledge/runbooks", "scripts/knowledge_render_atlas.py"]
source_paths: ["knowledge/runbooks", "scripts/render/knowledge_render_atlas.py"]
---
# KB authoring: what to write (and what not to)

13
knowledge/software/metis.md
View File

@ -1,12 +1,14 @@
# Metis (node recovery)
## Node classes (current map)
- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)
- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent)
- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)
- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)
- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)
- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)
- amd64 agents: titan-22/24 (Debian 13, k3s agent)
- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers.
- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels)
- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.
### Jetson nodes (titan-20/21)
- Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.
@ -22,13 +24,12 @@
## Metis repo (~/Development/metis)
- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).
- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).
- `AGENTS.md` in repo is untracked and holds raw notes.
## Next implementation steps
- Add per-class golden image refs and checksums (Harbor or file://) when ready.
- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.
- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.
- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.
- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited.
## Node OS/Kernel/CRI snapshot (Jan 2026)
- titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64
@ -55,10 +56,10 @@
- titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64
### External hosts
### Dedicated and special-purpose hosts
- titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.
- titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).
- titan-23/oceanus: TODO audit (future).
- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement.
### Control plane Pis (titan-0a/0b/0c)

5
scripts/comms_sync_kb.sh
View File

@ -1,5 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
python scripts/knowledge_render_atlas.py --write
python scripts/knowledge_render_atlas.py --write --out services/comms/knowledge

0
scripts/test_atlas_user_cleanup.py → scripts/manual-tests/test_atlas_user_cleanup.py
View File

0
scripts/test_user_cleanup.py → scripts/manual-tests/test_user_cleanup.py
View File

6
scripts/test_user_cleanup.sh → scripts/manual-tests/test_user_cleanup.sh
View File

@ -1,13 +1,13 @@
#!/usr/bin/env bash
set -euo pipefail
# Manual-only helper to run `scripts/test_user_cleanup.py` inside the portal backend container.
# Manual-only helper to run `scripts/manual-tests/test_user_cleanup.py` inside the portal backend container.
#
# Usage (dry-run):
# scripts/test_user_cleanup.sh --prefix test-
# scripts/manual-tests/test_user_cleanup.sh --prefix test-
#
# Usage (apply):
# scripts/test_user_cleanup.sh --prefix test- --apply --confirm test-
# scripts/manual-tests/test_user_cleanup.sh --prefix test- --apply --confirm test-
NS="${PORTAL_NAMESPACE:-bstein-dev-home}"
TARGET="${PORTAL_BACKEND_EXEC_TARGET:-deploy/bstein-dev-home-backend}"

4
scripts/test_vaultwarden_user_cleanup.py → scripts/manual-tests/test_vaultwarden_user_cleanup.py
View File

@ -11,10 +11,10 @@ conservative:
- Supports a protected email allowlist to prevent catastrophic mistakes.
Example (dry-run):
scripts/test_vaultwarden_user_cleanup.py --prefix e2e-
scripts/manual-tests/test_vaultwarden_user_cleanup.py --prefix e2e-
Example (apply):
scripts/test_vaultwarden_user_cleanup.py --prefix e2e- --apply --confirm e2e-
scripts/manual-tests/test_vaultwarden_user_cleanup.py --prefix e2e- --apply --confirm e2e-
"""
from __future__ import annotations

4
scripts/test_vaultwarden_user_cleanup.sh → scripts/manual-tests/test_vaultwarden_user_cleanup.sh
View File

@ -4,10 +4,10 @@ set -euo pipefail
# Manual-only helper to clean Vaultwarden test users and invites from Postgres.
#
# Usage (dry-run):
# scripts/test_vaultwarden_user_cleanup.sh --prefix e2e-
# scripts/manual-tests/test_vaultwarden_user_cleanup.sh --prefix e2e-
#
# Usage (apply):
# scripts/test_vaultwarden_user_cleanup.sh --prefix e2e- --apply --confirm e2e-
# scripts/manual-tests/test_vaultwarden_user_cleanup.sh --prefix e2e- --apply --confirm e2e-
SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"

2
scripts/build_ananke_node_helper.sh → scripts/ops/build_ananke_node_helper.sh
View File

@ -26,7 +26,7 @@ while [[ $# -gt 0 ]]; do
;;
-h|--help)
cat <<USAGE
Usage: scripts/build_ananke_node_helper.sh [--image <image>] [--docker-config <path>] [--platforms <csv>] [--builder <name>]
Usage: scripts/ops/build_ananke_node_helper.sh [--image <image>] [--docker-config <path>] [--platforms <csv>] [--builder <name>]
USAGE
exit 0
;;

2
scripts/build_harbor_bootstrap_bundle.sh → scripts/ops/build_harbor_bootstrap_bundle.sh
View File

@ -31,7 +31,7 @@ while [[ $# -gt 0 ]]; do
;;
-h|--help)
cat <<USAGE
Usage: scripts/build_harbor_bootstrap_bundle.sh [--images-file <path>] [--bundle-file <path>] [--docker-config <path>] [--platform <linux/arm64>] [--zstd-level <level>]
Usage: scripts/ops/build_harbor_bootstrap_bundle.sh [--images-file <path>] [--bundle-file <path>] [--docker-config <path>] [--platform <linux/arm64>] [--zstd-level <level>]
USAGE
exit 0
;;

12
scripts/cluster_power_console.sh → scripts/ops/cluster_power_console.sh
View File

@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
usage() {
cat <<'USAGE'
Usage:
scripts/cluster_power_console.sh [--repo-dir <path>] [--delegate-host <host>] <shutdown|startup> [recovery-script-options...]
scripts/ops/cluster_power_console.sh [--repo-dir <path>] [--delegate-host <host>] <shutdown|startup> [recovery-script-options...]
Purpose:
Friendly manual entrypoint for running Ananke from a remote console.
@ -17,9 +17,9 @@ Defaults:
--delegate-host titan-db
Examples:
scripts/cluster_power_console.sh shutdown --execute
scripts/cluster_power_console.sh startup --execute --force-flux-branch main
scripts/cluster_power_console.sh --delegate-host titan-24 shutdown --execute
scripts/ops/cluster_power_console.sh shutdown --execute
scripts/ops/cluster_power_console.sh startup --execute --force-flux-branch main
scripts/ops/cluster_power_console.sh --delegate-host titan-24 shutdown --execute
USAGE
}
@ -57,7 +57,7 @@ if [[ $# -lt 1 ]]; then
fi
SIBLING_SCRIPT="${SCRIPT_DIR}/cluster_power_recovery.sh"
REPO_SCRIPT="${REPO_DIR}/scripts/cluster_power_recovery.sh"
REPO_SCRIPT="${REPO_DIR}/scripts/ops/cluster_power_recovery.sh"
LOCAL_SCRIPT=""
if [[ -x "${SIBLING_SCRIPT}" ]]; then
@ -82,6 +82,6 @@ remote_cmd=""
if [[ -n "${REMOTE_REPO_DIR}" ]]; then
remote_cmd+="ANANKE_REPO_DIR=$(printf '%q' "${REMOTE_REPO_DIR}") "
fi
remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi"
remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi"
exec ssh -o BatchMode=yes -o ConnectTimeout=8 "${DELEGATE_HOST}" "${remote_cmd}"

18
scripts/cluster_power_recovery.sh → scripts/ops/cluster_power_recovery.sh
View File

@ -16,7 +16,7 @@ fi
usage() {
cat <<USAGE
Usage:
scripts/cluster_power_recovery.sh <prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup> [options]
scripts/ops/cluster_power_recovery.sh <prepare|status|bootstrap-seed|harbor-seed|longhorn-seed|longhorn-unlock|flux-hold|shutdown|startup> [options]
Options:
--execute Actually run commands (default is dry-run)
@ -77,14 +77,14 @@ Options:
-h, --help Show help
Examples:
scripts/cluster_power_recovery.sh prepare --execute
scripts/cluster_power_recovery.sh bootstrap-seed --execute
scripts/cluster_power_recovery.sh harbor-seed --execute
scripts/cluster_power_recovery.sh longhorn-unlock --execute
scripts/cluster_power_recovery.sh flux-hold --execute
scripts/cluster_power_recovery.sh status
scripts/cluster_power_recovery.sh shutdown --execute
scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main
scripts/ops/cluster_power_recovery.sh prepare --execute
scripts/ops/cluster_power_recovery.sh bootstrap-seed --execute
scripts/ops/cluster_power_recovery.sh harbor-seed --execute
scripts/ops/cluster_power_recovery.sh longhorn-unlock --execute
scripts/ops/cluster_power_recovery.sh flux-hold --execute
scripts/ops/cluster_power_recovery.sh status
scripts/ops/cluster_power_recovery.sh shutdown --execute
scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main
USAGE
}

2
scripts/crypto_wallet_monero_setup.fish → scripts/ops/crypto_wallet_monero_setup.fish Normal file → Executable file
View File

@ -1,3 +1,5 @@
#!/usr/bin/env fish
### ------- helpers ---------------------------------------------------------
function _need --description "ensure a command exists"

2
scripts/crypto_wallet_sui_setup.fish → scripts/ops/crypto_wallet_sui_setup.fish Normal file → Executable file
View File

@ -1,3 +1,5 @@
#!/usr/bin/env fish
### --------- helpers ----------
function _need --description "ensure a command exists"
for c in $argv

0
scripts/flux_hammer.fish → scripts/ops/flux_hammer.fish
View File

6
scripts/jellyfin_manual_load.fish → scripts/ops/jellyfin_manual_load.fish
View File

@ -4,10 +4,10 @@
# Fallback: kubectl port-forward to service (OK for small/medium files).
#
# Usage:
# scripts/jellyfin_manual_load.fish <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]
# scripts/ops/jellyfin_manual_load.fish <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]
# Examples:
# scripts/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN"
# scripts/jellyfin_manual_load.fish "$HOME/Movies/." movies # copy contents-only into /media/movies
# scripts/ops/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN"
# scripts/ops/jellyfin_manual_load.fish "$HOME/Movies/." movies # copy contents-only into /media/movies
function usage
echo "Usage: "(basename (status filename))" <LOCAL_PATH> [REMOTE_SUBDIR] [JELLYFIN_API_KEY]"

2
scripts/k3s_version_update.fish → scripts/ops/k3s_version_update.fish Normal file → Executable file
View File

@ -1,3 +1,5 @@
#!/usr/bin/env fish
# Pick the correct K3s asset for a remote host (arm64 vs x86_64)
function __k3s_asset_for_host
set -l host $argv[1]

0
scripts/longhorn_volume_usage.fish → scripts/ops/longhorn_volume_usage.fish
View File

2
scripts/node_recover.sh → scripts/ops/node_recover.sh
View File

@ -3,7 +3,7 @@ set -euo pipefail
usage() {
cat <<USAGE
Usage: scripts/node_recover.sh <node-name> [options]
Usage: scripts/ops/node_recover.sh <node-name> [options]
Options:
--yes Skip confirmation prompt

0
scripts/styx_kioskification.sh → scripts/ops/styx_kioskification.sh Normal file → Executable file
View File

0
scripts/styx_prep.sh → scripts/ops/styx_prep.sh
View File

0
scripts/styx_prep_nvme_luks.sh → scripts/ops/styx_prep_nvme_luks.sh
View File

0
scripts/verify_jenkins_workspace_cleanup_rollout.sh → scripts/ops/verify_jenkins_workspace_cleanup_rollout.sh
View File

171
scripts/dashboards_render_atlas.py → scripts/render/dashboards_render_atlas.py
View File

@ -2,8 +2,8 @@
"""Generate Atlas Grafana dashboards and render them into ConfigMaps.
Usage:
scripts/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps
scripts/dashboards_render_atlas.py # re-render ConfigMaps from JSON
scripts/render/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps
scripts/render/dashboards_render_atlas.py # re-render ConfigMaps from JSON
"""
import argparse
@ -16,7 +16,14 @@ from pathlib import Path
# Paths, folders, and shared metadata
# ---------------------------------------------------------------------------
ROOT = Path(__file__).resolve().parents[1]
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
ROOT = repo_root()
DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards"
CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path}
@ -428,22 +435,26 @@ STUCK_TERMINATING_EXPR = (
)
UPTIME_WINDOW = "365d"
# vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series.
UPTIME_RECORDING_METRIC = f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas"}}'
UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])"
UPTIME_RECORDING_METRIC = (
f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}'
)
TRAEFIK_READY_EXPR = (
"("
'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
" / clamp_min("
'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)'
")"
" > bool 0)"
)
CONTROL_READY_FRACTION_EXPR = (
f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
f" / {CONTROL_TOTAL})"
" >= bool 2)"
)
UPTIME_AVAIL_EXPR = (
f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
)
UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])"
UPTIME_RECORDING_EXPR = (
f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) "
f"or on() {UPTIME_LIVE_FALLBACK_EXPR})"
)
# Tie-breaker to deterministically pick one node per namespace when shares tie.
NODE_TIEBREAKER = " + ".join(
@ -620,7 +631,7 @@ PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX = (
"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube"
)
PLATFORM_TEST_STANDARD_CHECK_REGEX = (
"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|supply_chain"
"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|semgrep|supply_chain"
)
PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_SUITES = [
"ariadne",
@ -1870,7 +1881,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = {
"Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.",
"Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.",
"Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.",
"Atlas Availability (365d)": "Rolling one-year Atlas availability; higher is better, below target means users saw downtime.",
"Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.",
"Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.",
"CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.",
"Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.",
@ -1952,6 +1963,7 @@ TESTING_PANEL_DESCRIPTIONS = {
"Style Failure Rate": "Percent of style checks currently failing; blue means style/docs gates pass.",
"Gate Glue Failure Rate": "Percent of metric-contract checks failing; blue means dashboard telemetry is trustworthy.",
"SonarQube Failure Rate": "Percent of Sonar checks failing; blue means Sonar quality gates pass.",
"Semgrep Failure Rate": "Percent of Semgrep checks failing; blue means SAST rule scans pass.",
"Supply Chain Failure Rate": "Percent of supply-chain checks failing; blue means artifact/image checks pass.",
"Check Healthy Rates By Suite": "Healthy percent by check family; blue means all selected checks are good.",
"Tests Healthy Rate": "Percent of test checks passing or not applicable; higher is better.",
@ -1960,6 +1972,7 @@ TESTING_PANEL_DESCRIPTIONS = {
"Style Healthy Rate": "Percent of style checks passing or not applicable; higher is better.",
"Gate Glue Healthy Rate": "Percent of telemetry-contract checks passing; higher means cleaner reporting.",
"SonarQube Healthy Rate": "Percent of Sonar checks passing or not applicable; higher is better.",
"Semgrep Healthy Rate": "Percent of Semgrep checks passing or not applicable; higher is better.",
"Supply Chain Healthy Rate": "Percent of supply-chain checks passing or not applicable; higher is better.",
"Test Drilldowns And Problem Tests": "Test-case detail for finding which tests are hurting reliability.",
"Problematic Tests Over Time (Top failures)": "Current outlier tests by rolling 24h failures; tests need repeat failures to stay visible.",
@ -1976,6 +1989,13 @@ TESTING_PANEL_DESCRIPTIONS = {
"Recent Branch Evidence by Suite (7d)": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.",
"Primary Branch Clean by Suite (7d)": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.",
"SonarQube Project Health": "SonarQube availability, projects, fetch errors, and gate status.",
"Public ZAP Baseline": "OWASP ZAP passive baseline results for externally addressed Atlas domains.",
"ZAP Targets Scanned": "Number of public hostnames with a latest OWASP ZAP baseline scan sample.",
"ZAP High/Medium Alerts": "Current high and medium ZAP alert instance count across public targets.",
"ZAP Scan Errors": "Targets whose latest ZAP baseline failed or did not produce a usable report.",
"ZAP Alerts by Risk": "Current ZAP alert distribution by risk level.",
"ZAP Target Health": "Latest ZAP health per hostname from passive baseline scans.",
"ZAP Alerts by Host": "Current high, medium, and low ZAP alert counts by public hostname.",
"SonarQube API Up": "Whether the SonarQube exporter can reach SonarQube; 1 is good.",
"Sonar Projects (Selected)": "Selected SonarQube project count; zero means Sonar is not tracking that suite.",
"Sonar Gate Fetch Errors": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.",
@ -2103,7 +2123,7 @@ def build_overview():
},
{
"id": 27,
"title": "Atlas Availability (365d)",
"title": "Atlas Serving Availability",
"expr": UPTIME_PERCENT_EXPR,
"kind": "stat",
"thresholds": UPTIME_PERCENT_THRESHOLDS,
@ -2111,7 +2131,7 @@ def build_overview():
"decimals": 4,
"text_mode": "value",
"instant": True,
"description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.",
"description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.",
},
{
"id": 4,
@ -2335,10 +2355,10 @@ def build_overview():
}
overview_avg_coverage = f"(avg(({QUALITY_GATE_COVERAGE_BY_SUITE})) or on() vector(0))"
overview_category_health = (
f'avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
f'(avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{'
f'suite=~"{PLATFORM_TEST_SUITE_CANONICAL_MATCHER}",branch!="",branch=~"main|master|origin/main|origin/master",'
f'category=~"{PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"'
"})"
'})) or label_set(vector(0), "category", "none")'
)
for panel_id, title, draw_expr, runtime_expr, y_pos in [
(40, "Pyrphoros UPS Current", ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_RUNTIME_DB, 7),
@ -4163,6 +4183,7 @@ def build_jobs_dashboard():
check_regex_style = "docs|naming|hygiene|lint|docs_naming|style"
check_regex_gate_glue = "gate|glue|gate_glue"
check_regex_sonarqube = "sonarqube|sonar"
check_regex_semgrep = "semgrep|sast"
check_regex_supply_chain = "ironbank|supply_chain|image_compliance|artifact_security"
def _check_state_percent_series(regex: str, failed: bool) -> str:
@ -4206,6 +4227,30 @@ def build_jobs_dashboard():
f"topk by (suite) (1, ({rollup_failed_tests_history}))"
)
worst_test_per_suite = worst_test_per_suite_core
zap_target_health = (
"(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) "
"or on() vector(0))"
)
zap_targets_scanned = (
'(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job="platform-security-zap"})) '
"or on() vector(0))"
)
zap_high_medium_alerts = (
'(sum(platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk=~"high|medium"}) '
"or on() vector(0))"
)
zap_scan_errors = (
'(sum(platform_zap_baseline_scan_status{exported_job="platform-security-zap",status=~"fail|error"}) '
"or on() vector(0))"
)
zap_alerts_by_risk = (
'sum by (risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk!=""}) '
"or on() vector(0)"
)
zap_alerts_by_host_risk = (
'sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",'
'risk=~"high|medium|low"}) or on() vector(0)'
)
def _selected_status_volume(status: str) -> str:
return (
@ -4551,6 +4596,7 @@ def build_jobs_dashboard():
("Style", check_regex_style),
("Gate Glue", check_regex_gate_glue),
("SonarQube", check_regex_sonarqube),
("Semgrep", check_regex_semgrep),
("Supply Chain", check_regex_supply_chain),
]
@ -4574,12 +4620,12 @@ def build_jobs_dashboard():
)
panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
panels.append(panel)
for index, (label, regex) in enumerate(check_dimensions[4:]):
for index, (label, regex) in enumerate(check_dimensions[4:8]):
panel = state_timeline_panel(
start_id + 4 + index,
f"{label} {title_prefix}",
_check_state_percent_series(regex, failed),
{"h": 7, "w": 8, "x": index * 8, "y": y + 7},
{"h": 7, "w": 6, "x": index * 6, "y": y + 7},
thresholds=trend_thresholds,
description=trend_description,
)
@ -4587,7 +4633,7 @@ def build_jobs_dashboard():
panels.append(panel)
_append_check_trends(130, "Failure Rate", True, 29)
_append_check_trends(138, "Healthy Rate", False, 43)
_append_check_trends(160, "Healthy Rate", False, 43)
panels.append(
state_timeline_panel(
145,
@ -4843,6 +4889,79 @@ def build_jobs_dashboard():
)
sonar_gate_project_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
panels.append(sonar_gate_project_panel)
panels.append(
stat_panel(
168,
"ZAP Targets Scanned",
zap_targets_scanned,
{"h": 6, "w": 4, "x": 0, "y": 117},
unit="none",
instant=True,
thresholds=missing_thresholds,
)
)
panels.append(
stat_panel(
169,
"ZAP High/Medium Alerts",
zap_high_medium_alerts,
{"h": 6, "w": 4, "x": 4, "y": 117},
unit="none",
instant=True,
thresholds=failures_thresholds,
)
)
panels.append(
stat_panel(
170,
"ZAP Scan Errors",
zap_scan_errors,
{"h": 6, "w": 4, "x": 8, "y": 117},
unit="none",
instant=True,
thresholds=failures_thresholds,
)
)
zap_risk_panel = pie_panel(
171,
"ZAP Alerts by Risk",
zap_alerts_by_risk,
{"h": 6, "w": 4, "x": 12, "y": 117},
)
zap_risk_panel["targets"][0]["legendFormat"] = "{{risk}}"
panels.append(zap_risk_panel)
zap_target_panel = state_timeline_panel(
172,
"ZAP Target Health",
zap_target_health,
{"h": 6, "w": 8, "x": 16, "y": 117},
thresholds=success_thresholds,
unit="percent",
min_value=0,
max_value=100,
legend="{{host}}",
description=(
"Latest OWASP ZAP passive baseline health per public hostname. "
"High alerts and scan errors drop a target to zero; medium/low alerts mark warning health."
),
)
zap_target_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW
panels.append(zap_target_panel)
panels.append(
bargauge_panel(
173,
"ZAP Alerts by Host",
zap_alerts_by_host_risk,
{"h": 7, "w": 24, "x": 0, "y": 123},
unit="none",
instant=True,
legend="{{host}} · {{risk}}",
sort_order="desc",
thresholds=failures_thresholds,
decimals=0,
limit=20,
)
)
panels.append(
bargauge_panel(
148,
@ -4952,6 +5071,12 @@ def build_jobs_dashboard():
33: {"h": 6, "w": 4, "x": 8, "y": 111},
34: {"h": 6, "w": 4, "x": 12, "y": 111},
35: {"h": 6, "w": 8, "x": 16, "y": 111},
168: {"h": 6, "w": 4, "x": 0, "y": 117},
169: {"h": 6, "w": 4, "x": 4, "y": 117},
170: {"h": 6, "w": 4, "x": 8, "y": 117},
171: {"h": 6, "w": 4, "x": 12, "y": 117},
172: {"h": 6, "w": 8, "x": 16, "y": 117},
173: {"h": 7, "w": 24, "x": 0, "y": 123},
}
for panel_id, grid in row_layout.items():
panel_by_id[panel_id]["gridPos"] = grid
@ -4963,13 +5088,13 @@ def build_jobs_dashboard():
501,
"Check Failure Rates By Suite",
12,
panels=children([130, 131, 132, 133, 134, 135, 136]),
panels=children([130, 131, 132, 133, 134, 135, 136, 137]),
),
row_panel(
502,
"Check Healthy Rates By Suite",
13,
panels=children([138, 139, 140, 141, 142, 143, 144]),
panels=children([160, 161, 162, 163, 164, 165, 166, 167]),
),
row_panel(
503,
@ -4989,6 +5114,12 @@ def build_jobs_dashboard():
16,
panels=children([31, 32, 33, 34, 35]),
),
row_panel(
506,
"Public ZAP Baseline",
17,
panels=children([168, 169, 170, 171, 172, 173]),
),
]
)
panels = compact_panels

15
scripts/dashboards_render_logs.py → scripts/render/dashboards_render_logs.py
View File

@ -2,8 +2,8 @@
"""Generate OpenSearch Dashboards saved objects and render them into ConfigMaps.
Usage:
scripts/dashboards_render_logs.py --build # rebuild NDJSON + ConfigMap
scripts/dashboards_render_logs.py # re-render ConfigMap from NDJSON
scripts/render/dashboards_render_logs.py --build # rebuild NDJSON + ConfigMap
scripts/render/dashboards_render_logs.py # re-render ConfigMap from NDJSON
"""
from __future__ import annotations
@ -14,14 +14,21 @@ import textwrap
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
ROOT = repo_root()
DASHBOARD_DIR = ROOT / "services" / "logging" / "dashboards"
NDJSON_PATH = DASHBOARD_DIR / "logs.ndjson"
CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-dashboards-objects.yaml"
CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path}
# Generated by scripts/dashboards_render_logs.py --build
# Generated by scripts/render/dashboards_render_logs.py --build
apiVersion: v1
kind: ConfigMap
metadata:

85
scripts/render/flux_inventory.py Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Print a compact inventory of Flux Kustomization resources."""
from __future__ import annotations
import argparse
from pathlib import Path
from typing import Any
import yaml
def _iter_docs(path: Path) -> list[dict[str, Any]]:
docs: list[dict[str, Any]] = []
for raw in yaml.safe_load_all(path.read_text(encoding="utf-8")):
if isinstance(raw, dict):
docs.append(raw)
return docs
def _is_flux_kustomization(doc: dict[str, Any]) -> bool:
return (
doc.get("kind") == "Kustomization"
and str(doc.get("apiVersion") or "").startswith("kustomize.toolkit.fluxcd.io/")
)
def _depends_on(spec: dict[str, Any]) -> str:
names = []
for dep in spec.get("dependsOn") or []:
if isinstance(dep, dict) and dep.get("name"):
names.append(str(dep["name"]))
return ",".join(names)
def _row(doc: dict[str, Any], path: Path) -> dict[str, str]:
metadata = doc.get("metadata") or {}
spec = doc.get("spec") or {}
annotations = metadata.get("annotations") or {}
return {
"name": str(metadata.get("name") or ""),
"path": str(spec.get("path") or ""),
"namespace": str(spec.get("targetNamespace") or ""),
"depends": _depends_on(spec),
"prune": str(spec.get("prune", "")),
"wait": str(spec.get("wait", "")),
"suspend": str(spec.get("suspend", False)),
"reason": str(annotations.get("atlas.bstein.dev/suspend-reason") or ""),
"file": str(path),
}
def build_inventory(root: Path) -> list[dict[str, str]]:
rows: list[dict[str, str]] = []
for path in sorted(root.rglob("*.yaml")):
for doc in _iter_docs(path):
if _is_flux_kustomization(doc):
rows.append(_row(doc, path))
return sorted(rows, key=lambda item: item["name"])
def _print_table(rows: list[dict[str, str]]) -> None:
columns = ["name", "path", "namespace", "depends", "prune", "wait", "suspend", "reason"]
widths = {
column: max([len(column), *[len(row[column]) for row in rows]]) for column in columns
}
header = " ".join(column.ljust(widths[column]) for column in columns)
print(header)
print(" ".join("-" * widths[column] for column in columns))
for row in rows:
print(" ".join(row[column].ljust(widths[column]) for column in columns))
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("root", nargs="?", default="clusters/atlas/flux-system")
args = parser.parse_args()
rows = build_inventory(Path(args.root))
_print_table(rows)
return 0
if __name__ == "__main__":
raise SystemExit(main())

11
scripts/knowledge_render_atlas.py → scripts/render/knowledge_render_atlas.py
View File

@ -25,7 +25,14 @@ from typing import Any, Iterable
import yaml
REPO_ROOT = Path(__file__).resolve().parents[1]
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
REPO_ROOT = repo_root()
DASHBOARD_DIR = REPO_ROOT / "services" / "monitoring" / "dashboards"
CLUSTER_SCOPED_KINDS = {
@ -580,7 +587,7 @@ def main() -> int:
catalog_rel = catalog_path.relative_to(REPO_ROOT).as_posix()
catalog_path.write_text(
f"# {catalog_rel}\n"
"# Generated by scripts/knowledge_render_atlas.py (do not edit by hand)\n"
"# Generated by scripts/render/knowledge_render_atlas.py (do not edit by hand)\n"
+ yaml.safe_dump(catalog, sort_keys=False),
encoding="utf-8",
)

15
scripts/logging_render_observability.py → scripts/render/logging_render_observability.py
View File

@ -2,8 +2,8 @@
"""Generate OpenSearch Observability seed objects and render them into ConfigMaps.
Usage:
scripts/logging_render_observability.py --build # rebuild JSON + ConfigMap
scripts/logging_render_observability.py # re-render ConfigMap from JSON
scripts/render/logging_render_observability.py --build # rebuild JSON + ConfigMap
scripts/render/logging_render_observability.py # re-render ConfigMap from JSON
"""
from __future__ import annotations
@ -14,7 +14,14 @@ import textwrap
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
ROOT = repo_root()
OBS_DIR = ROOT / "services" / "logging" / "observability"
APPS_PATH = OBS_DIR / "applications.json"
QUERIES_PATH = OBS_DIR / "saved_queries.json"
@ -23,7 +30,7 @@ CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-observability-objects.
CONFIG_TEMPLATE = textwrap.dedent(
"""# {relative_path}
# Generated by scripts/logging_render_observability.py --build
# Generated by scripts/render/logging_render_observability.py --build
apiVersion: v1
kind: ConfigMap
metadata:

11
scripts/monitoring_render_postmark_exporter.py → scripts/render/monitoring_render_postmark_exporter.py
View File

@ -3,14 +3,21 @@
from pathlib import Path
def repo_root() -> Path:
for path in Path(__file__).resolve().parents:
if (path / "clusters").is_dir() and (path / "services").is_dir():
return path
raise RuntimeError("could not locate repository root") # pragma: no cover
def indent(text: str, spaces: int) -> str:
prefix = " " * spaces
return "".join(prefix + line if line.strip("\n") else line for line in text.splitlines(keepends=True))
def main() -> None:
root = Path(__file__).resolve().parents[1]
source = root / "scripts" / "monitoring_postmark_exporter.py"
root = repo_root()
source = root / "scripts" / "sync" / "monitoring_postmark_exporter.py"
target = root / "services" / "monitoring" / "postmark-exporter-script.yaml"
payload = source.read_text(encoding="utf-8")

5
scripts/sync/comms_sync_kb.sh Executable file
View File

@ -0,0 +1,5 @@
#!/usr/bin/env bash
set -euo pipefail
python scripts/render/knowledge_render_atlas.py --write
python scripts/render/knowledge_render_atlas.py --write --out services/comms/knowledge

0
scripts/gitea_cred_sync.sh → scripts/sync/gitea_cred_sync.sh
View File

0
scripts/gitops_cred_sync.sh → scripts/sync/gitops_cred_sync.sh
View File

0
scripts/jenkins_cred_sync.sh → scripts/sync/jenkins_cred_sync.sh
View File

0
scripts/monitoring_postmark_exporter.py → scripts/sync/monitoring_postmark_exporter.py
View File

50
scripts/tests/test_dashboards_render_atlas.py
View File

@ -3,8 +3,10 @@ import pathlib
def load_module():
path = pathlib.Path(__file__).resolve().parents[1] / "dashboards_render_atlas.py"
spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path)
path = pathlib.Path(__file__).resolve().parents[1] / "render/dashboards_render_atlas.py"
spec = importlib.util.spec_from_file_location(
"scripts.render.dashboards_render_atlas", path
)
module = importlib.util.module_from_spec(spec)
assert spec.loader is not None
spec.loader.exec_module(module)
@ -55,11 +57,20 @@ def test_overview_availability_panel_uses_recorded_365d_rollup():
dashboard = mod.build_overview()
panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27)
assert panel["title"] == "Atlas Availability (365d)"
assert panel["targets"][0]["expr"] == 'last_over_time(atlas:availability:ratio_365d{scope="atlas"}[24h])'
assert panel["title"] == "Atlas Serving Availability"
availability_expr = panel["targets"][0]["expr"]
assert (
'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])'
in availability_expr
)
assert ">= bool 2" in availability_expr
assert "> bool 0" in availability_expr
assert "[1h:5m]" in availability_expr
assert panel["targets"][0]["instant"] is True
assert "precomputed" in panel["description"]
assert "last successful rollup for up to 24h" in panel["description"]
assert "at least one Traefik replica serving" in panel["description"]
assert "partial-capacity samples remain available" in panel["description"]
assert "monitoring gaps are excluded" in panel["description"]
assert "falls back to the live binary serving state" in panel["description"]
def test_overview_uses_readable_quality_power_and_gitops_panels():
@ -148,6 +159,7 @@ def test_overview_uses_readable_quality_power_and_gitops_panels():
assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto"
assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9
assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}"
assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"]
assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"])
pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"]
@ -281,7 +293,7 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"]
assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr
assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr
assert "[30d:15m]" in panels_by_title["CI Run Success Rate (30d)"]["targets"][0]["expr"]
assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"]
assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6
assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6
assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4}
@ -300,7 +312,7 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability():
rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"]
assert rolling_panel["type"] == "state-timeline"
assert "[7d:1m]" in rolling_panel["targets"][0]["expr"]
assert "[7d:1h]" in rolling_panel["targets"][0]["expr"]
category_panel = panels_by_title["Test Category Health History"]
assert category_panel["type"] == "state-timeline"
assert "category" in category_panel["targets"][0]["expr"]
@ -384,7 +396,7 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
for child in row.get("panels", [])
}
assert len(panels) == 18
assert len(panels) == 19
assert len(visible_query_panels) == 12
assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12
assert all(
@ -398,15 +410,18 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
"Test Drilldowns And Problem Tests",
"Telemetry Completeness And Branches",
"SonarQube Project Health",
"Public ZAP Baseline",
]
assert all(row["collapsed"] for row in rows)
assert "Coverage Failure Rate" in nested_panels_by_title
assert "Semgrep Failure Rate" in nested_panels_by_title
assert "Supply Chain Healthy Rate" in nested_panels_by_title
assert "Test Category Health History" in nested_panels_by_title
assert "Selected Test Pass Rate History" in nested_panels_by_title
assert "Coverage Metrics Present by Suite" in nested_panels_by_title
assert "SonarQube API Up" in nested_panels_by_title
assert "ZAP Target Health" in nested_panels_by_title
failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"]
assert failure_rate_panel["type"] == "state-timeline"
@ -457,13 +472,22 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint():
assert sonar_health_panel["type"] == "state-timeline"
assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"]
assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"]
semgrep_panel = nested_panels_by_title["Semgrep Healthy Rate"]
assert semgrep_panel["type"] == "state-timeline"
assert 'check=~"semgrep|sast"' in semgrep_panel["targets"][0]["expr"]
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (30d)"]
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (30d)"]
zap_health_panel = nested_panels_by_title["ZAP Target Health"]
assert zap_health_panel["type"] == "state-timeline"
assert "platform_zap_baseline_target_health_percent" in zap_health_panel["targets"][0]["expr"]
zap_alerts_panel = nested_panels_by_title["ZAP Alerts by Host"]
assert "platform_zap_baseline_alerts_total" in zap_alerts_panel["targets"][0]["expr"]
branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"]
recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"]
assert branch_panel["gridPos"]["x"] == 12
assert recent_branch_panel["gridPos"]["x"] == 18
assert "[30d:15m]" in recent_branch_panel["targets"][0]["expr"]
assert "[30d:15m]" in branch_panel["targets"][0]["expr"]
assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"]
assert "[7d:1h]" in branch_panel["targets"][0]["expr"]
assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent"
assert "unless on(suite)" in branch_panel["targets"][0]["expr"]
assert "> bool 0" in branch_panel["targets"][0]["expr"]

106
scripts/tests/test_monitoring_query_capacity.py Normal file
View File

@ -0,0 +1,106 @@
"""Protect the monitoring backend from known query-starvation regressions."""
from pathlib import Path
import yaml
REPO_ROOT = Path(__file__).resolve().parents[2]
def _documents(path: Path) -> list[dict]:
"""Load every non-empty YAML document from a repository manifest."""
return [document for document in yaml.safe_load_all(path.read_text()) if document]
def test_victoria_metrics_has_dashboard_burst_headroom() -> None:
"""Keep search capacity and pod resources above the proven failure floor."""
manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml")
release = next(
manifest
for manifest in manifests
if manifest.get("kind") == "HelmRelease"
and manifest.get("metadata", {}).get("name") == "victoria-metrics-single"
)
server = release["spec"]["values"]["server"]
assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4
assert server["extraArgs"]["search.maxQueryDuration"] == "1m"
assert server["extraArgs"]["search.maxQueueDuration"] == "30s"
assert server["resources"]["requests"]["memory"] == "2Gi"
assert server["resources"]["limits"]["cpu"] == "2"
assert server["resources"]["limits"]["memory"] == "4Gi"
required_terms = server["affinity"]["nodeAffinity"][
"requiredDuringSchedulingIgnoredDuringExecution"
]["nodeSelectorTerms"]
hostname_rule = next(
expression
for term in required_terms
for expression in term["matchExpressions"]
if expression["key"] == "kubernetes.io/hostname"
)
assert hostname_rule["operator"] == "NotIn"
assert {"titan-14", "titan-18"} <= set(hostname_rule["values"])
def test_yearly_availability_reuses_the_hourly_rollup() -> None:
"""Prevent the yearly rule from rescanning raw cluster metrics for 365 days."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
yearly = next(
rule
for rule in rules["rules"]
if rule["record"] == "atlas:availability:ratio_365d"
)
assert "atlas:availability:ratio_1h" in yearly["expr"]
assert 'definition="serving-v2"' in yearly["expr"]
assert 'definition=""' in yearly["expr"]
assert "share_gt_over_time" in yearly["expr"]
assert "[365d:15m]" in yearly["expr"]
assert "kube_node_status_condition" not in yearly["expr"]
assert "[365d:1h]" not in yearly["expr"]
def test_availability_counts_serving_state_instead_of_replica_capacity() -> None:
"""Treat quorum and any serving ingress replica as available binary states."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0]
hourly = next(
rule
for rule in rules["rules"]
if rule["record"] == "atlas:availability:ratio_1h"
)
assert ">= bool 2" in hourly["expr"]
assert "> bool 0" in hourly["expr"]
assert "/ 3" not in hourly["expr"]
assert hourly["labels"]["definition"] == "serving-v2"
def test_quality_rollups_do_not_run_every_minute() -> None:
"""Keep high-cardinality quality rollups below the backend saturation cadence."""
manifest = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)[0]
quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0]
assert quality["interval"] == "5m"
def test_vmalert_reloads_updated_rule_files() -> None:
"""Make Flux ConfigMap updates take effect without manual pod revision bumps."""
manifests = _documents(
REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml"
)
deployment = next(
manifest for manifest in manifests if manifest.get("kind") == "Deployment"
)
args = deployment["spec"]["template"]["spec"]["containers"][0]["args"]
assert "-configCheckInterval=30s" in args

6
services/bstein-dev-home/kustomization.yaml
View File

@ -16,13 +16,13 @@ resources:
- backend-deployment.yaml
- backend-service.yaml
- vaultwarden-cred-sync-cronjob.yaml
- oneoffs/portal-onboarding-e2e-test-job.yaml
- validation-jobs/portal-onboarding-e2e-test-job.yaml
- ingress.yaml
images:
- name: registry.bstein.dev/bstein/bstein-dev-home-frontend
newTag: 0.1.1-413 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"}
- name: registry.bstein.dev/bstein/bstein-dev-home-backend
newTag: 0.1.1-413 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"}
configMapGenerator:
- name: chat-ai-gateway
namespace: bstein-dev-home

2
services/bstein-dev-home/oneoffs/migrations/kustomization.yaml → services/bstein-dev-home/migration-jobs/kustomization.yaml
View File

@ -1,4 +1,4 @@
# services/bstein-dev-home/oneoffs/migrations/kustomization.yaml
# services/bstein-dev-home/migration-jobs/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: bstein-dev-home

2
services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml → services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml
View File

@ -1,4 +1,4 @@
# services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml
# services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml
# One-off job for bstein-dev-home/bstein-dev-home-portal-migrate-36.
# Purpose: bstein dev home portal migrate 36 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml → services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml
View File

@ -1,4 +1,4 @@
# services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml
# services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml
# One-off job for bstein-dev-home/portal-onboarding-e2e-test-27.
# Purpose: portal onboarding e2e test 27 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/comms-secrets-ensure-job.yaml → services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/comms-secrets-ensure-job.yaml
# services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml
# One-off job for comms/comms-secrets-ensure-7.
# Purpose: comms secrets ensure 7 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml → services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml
# services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml
# One-off job for comms/mas-admin-client-secret-writer.
# Purpose: mas admin client secret writer (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/mas-db-ensure-job.yaml → services/comms/bootstrap-jobs/mas-db-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/mas-db-ensure-job.yaml
# services/comms/bootstrap-jobs/mas-db-ensure-job.yaml
# One-off job for comms/mas-db-ensure-22.
# Purpose: mas db ensure 22 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/mas-local-users-ensure-job.yaml → services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/mas-local-users-ensure-job.yaml
# services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml
# One-off job for comms/mas-local-users-ensure-18.
# Purpose: mas local users ensure 18 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/synapse-admin-ensure-job.yaml → services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/synapse-admin-ensure-job.yaml
# services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml
# One-off job for comms/synapse-admin-ensure-3.
# Purpose: synapse admin ensure 3 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml → services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml
# services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml
# One-off job for comms/synapse-seeder-admin-ensure-9.
# Purpose: synapse seeder admin ensure 9 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/synapse-signingkey-ensure-job.yaml → services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/synapse-signingkey-ensure-job.yaml
# services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml
# One-off job for comms/othrys-synapse-signingkey-ensure-7.
# Purpose: othrys synapse signingkey ensure 7 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

2
services/comms/oneoffs/synapse-user-seed-job.yaml → services/comms/bootstrap-jobs/synapse-user-seed-job.yaml
View File

@ -1,4 +1,4 @@
# services/comms/oneoffs/synapse-user-seed-job.yaml
# services/comms/bootstrap-jobs/synapse-user-seed-job.yaml
# One-off job for comms/synapse-user-seed-9.
# Purpose: synapse user seed 9 (see container args/env in this file).
# Run by setting spec.suspend to false, reconcile, then set it back to true.

Some files were not shown because too many files have changed in this diff Show More