From 0da9e4c82de6f1c720f65438e21ac4f82b4ee341 Mon Sep 17 00:00:00 2001 From: jenkins Date: Wed, 5 Aug 2026 16:21:36 -0300 Subject: [PATCH] refactor: restructure services layout, retire oceanus, add aether scaffolding - Move flat service manifests into structured subdirs (apps/, bootstrap-jobs/, repair-jobs/, migration-jobs/, validation-jobs/, node-ops/, networking/) - Retire oneoffs/ directories across services - Remove oceanus cluster and its host roles; add aether cluster + terraform scaffolding - Reorganize scripts/ into ops/, render/, sync/, manual-tests/ - Add Makefile with render/validate/test/flux targets and repo-structure tests - Update flux-system application CRs to the new paths - Add hermes-automated-triage-24h-plan knowledge doc (+ comms mirror) - Refresh knowledge catalogs, dashboards, vmalert rules, quality contract Co-Authored-By: Claude Fable 5 --- .gitignore | 2 + Jenkinsfile | 55 + Makefile | 105 + ci/Jenkinsfile.titan-iac | 55 + ci/scripts/publish_test_metrics.py | 3 + ci/scripts/publish_test_metrics_quality.py | 17 + ci/scripts/semgrep_report.py | 159 ++ ci/scripts/supply_chain_report.py | 6 +- ci/titan-iac-trivy-waivers.json | 274 ++- .../applications}/kustomization.yaml | 2 +- .../aether/flux-system/kustomization.yaml | 6 + .../flux-system/platform}/kustomization.yaml | 2 +- .../kustomization.yaml | 3 +- .../applications/finance/kustomization.yaml | 1 + .../game-stream/kustomization.yaml | 1 + .../applications/health/kustomization.yaml | 1 + .../applications/jellyfin/kustomization.yaml | 1 + .../applications/jenkins/kustomization.yaml | 1 + .../applications/mailu/kustomization.yaml | 1 + .../nextcloud-mail-sync/kustomization.yaml | 1 + .../applications/nextcloud/kustomization.yaml | 1 + .../applications/outline/kustomization.yaml | 1 + .../applications/planka/kustomization.yaml | 1 + .../applications/quality/kustomization.yaml | 1 + .../applications/typhon/kustomization.yaml | 1 + .../vaultwarden/kustomization.yaml | 1 + .../wallet-monero-temp/kustomization.yaml | 1 + .../applications/xmr-miner/kustomization.yaml | 1 + .../platform/descheduler/kustomization.yaml | 1 + .../platform/gitops-ui/kustomization.yaml | 1 + .../platform/longhorn-ui/kustomization.yaml | 1 + .../resource-guardrails/kustomization.yaml | 1 + .../oceanus/flux-system/kustomization.yaml | 9 - clusters/oceanus/platform/kustomization.yaml | 6 - hosts/inventory/lab.yaml | 6 +- hosts/playbooks/site.yaml | 7 - hosts/roles/oceanus_base/tasks/main.yaml | 6 - .../cert-manager/letsencrypt-prod.yaml | 2 +- .../cert-manager/letsencrypt.yaml | 2 +- infrastructure/core/kustomization.yaml | 4 +- infrastructure/flux-system/kustomization.yaml | 2 +- knowledge/INDEX.md | 2 +- knowledge/catalog/atlas-summary.json | 8 +- knowledge/catalog/atlas.json | 1103 ++++++++- knowledge/catalog/atlas.yaml | 714 +++++- knowledge/catalog/metrics.json | 2144 +++++++++++++---- knowledge/catalog/runbooks.json | 18 +- knowledge/diagrams/atlas-http.mmd | 58 + knowledge/hermes-automated-triage-24h-plan.md | 985 ++++++++ knowledge/metis.md | 9 +- knowledge/runbooks/ci-gitea-jenkins.md | 4 +- knowledge/runbooks/cluster-power-recovery.md | 20 +- knowledge/runbooks/kb-authoring.md | 2 +- knowledge/software/metis.md | 13 +- scripts/comms_sync_kb.sh | 5 - .../test_atlas_user_cleanup.py | 0 .../{ => manual-tests}/test_user_cleanup.py | 0 .../{ => manual-tests}/test_user_cleanup.sh | 6 +- .../test_vaultwarden_user_cleanup.py | 4 +- .../test_vaultwarden_user_cleanup.sh | 4 +- scripts/{ => ops}/build_ananke_node_helper.sh | 2 +- .../build_harbor_bootstrap_bundle.sh | 2 +- scripts/{ => ops}/cluster_power_console.sh | 12 +- scripts/{ => ops}/cluster_power_recovery.sh | 18 +- .../{ => ops}/crypto_wallet_monero_setup.fish | 2 + .../{ => ops}/crypto_wallet_sui_setup.fish | 2 + scripts/{ => ops}/flux_hammer.fish | 0 scripts/{ => ops}/jellyfin_manual_load.fish | 6 +- scripts/{ => ops}/k3s_version_update.fish | 2 + scripts/{ => ops}/longhorn_volume_usage.fish | 0 scripts/{ => ops}/node_recover.sh | 2 +- scripts/{ => ops}/styx_kioskification.sh | 0 scripts/{ => ops}/styx_prep.sh | 0 scripts/{ => ops}/styx_prep_nvme_luks.sh | 0 ...erify_jenkins_workspace_cleanup_rollout.sh | 0 .../{ => render}/dashboards_render_atlas.py | 171 +- .../{ => render}/dashboards_render_logs.py | 15 +- scripts/render/flux_inventory.py | 85 + .../{ => render}/knowledge_render_atlas.py | 11 +- .../logging_render_observability.py | 15 +- .../monitoring_render_postmark_exporter.py | 11 +- scripts/sync/comms_sync_kb.sh | 5 + scripts/{ => sync}/gitea_cred_sync.sh | 0 scripts/{ => sync}/gitops_cred_sync.sh | 0 scripts/{ => sync}/jenkins_cred_sync.sh | 0 .../monitoring_postmark_exporter.py | 0 scripts/tests/test_dashboards_render_atlas.py | 50 +- .../tests/test_monitoring_query_capacity.py | 106 + services/bstein-dev-home/kustomization.yaml | 6 +- .../kustomization.yaml | 2 +- .../portal-migrate-job.yaml | 2 +- .../portal-onboarding-e2e-test-job.yaml | 2 +- .../comms-secrets-ensure-job.yaml | 2 +- .../mas-admin-client-secret-ensure-job.yaml | 2 +- .../mas-db-ensure-job.yaml | 2 +- .../mas-local-users-ensure-job.yaml | 2 +- .../synapse-admin-ensure-job.yaml | 2 +- .../synapse-seeder-admin-ensure-job.yaml | 2 +- .../synapse-signingkey-ensure-job.yaml | 2 +- .../synapse-user-seed-job.yaml | 2 +- services/comms/knowledge/INDEX.md | 2 +- .../knowledge/catalog/atlas-summary.json | 8 +- services/comms/knowledge/catalog/atlas.json | 1103 ++++++++- services/comms/knowledge/catalog/atlas.yaml | 714 +++++- services/comms/knowledge/catalog/metrics.json | 2144 +++++++++++++---- .../comms/knowledge/catalog/runbooks.json | 18 +- .../comms/knowledge/diagrams/atlas-http.mmd | 58 + .../hermes-automated-triage-24h-plan.md | 985 ++++++++ services/comms/knowledge/metis.md | 9 +- .../knowledge/runbooks/ci-gitea-jenkins.md | 4 +- .../runbooks/cluster-power-recovery.md | 152 ++ .../comms/knowledge/runbooks/kb-authoring.md | 2 +- services/comms/knowledge/software/metis.md | 27 +- services/comms/kustomization.yaml | 20 +- .../bstein-force-leave-job.yaml | 2 +- .../othrys-kick-numeric-job.yaml | 2 +- services/comms/scripts/atlasbot/bot.py | 2 +- services/crypto/kustomization.yaml | 1 + services/crypto/monerod/kustomization.yaml | 1 + .../wallet-monero-temp/kustomization.yaml | 1 + services/crypto/xmr-miner/kustomization.yaml | 1 + .../finance-secrets-ensure-job.yaml | 2 +- services/finance/kustomization.yaml | 2 +- services/game-stream/kustomization.yaml | 1 + services/gitea/kustomization.yaml | 1 + services/jellyfin/kustomization.yaml | 1 + .../actual-oidc-secret-ensure-job.yaml | 2 +- ...assandra-gitea-oidc-secret-ensure-job.yaml | 54 + .../cassandra-realm-ensure-job.yaml | 401 +++ .../harbor-oidc-secret-ensure-job.yaml | 2 +- .../hermes-dashboard-oidc-client-job.yaml | 2 +- .../ldap-federation-job.yaml | 2 +- .../logs-oidc-secret-ensure-job.yaml | 2 +- .../mas-secrets-ensure-job.yaml | 2 +- ...etis-node-passwords-secret-ensure-job.yaml | 2 +- .../metis-oidc-secret-ensure-job.yaml | 2 +- .../metis-ssh-keys-secret-ensure-job.yaml | 2 +- ...portal-admin-client-secret-ensure-job.yaml | 2 +- .../portal-groups-scope-ensure-job.yaml | 2 +- .../quality-oidc-secret-ensure-job.yaml | 2 +- .../realm-settings-job.yaml | 2 +- .../soteria-oidc-secret-ensure-job.yaml | 2 +- .../synapse-oidc-secret-ensure-job.yaml | 2 +- .../user-overrides-job.yaml | 2 +- .../vault-oidc-secret-ensure-job.yaml | 2 +- .../veles-gitea-oidc-secret-ensure-job.yaml | 2 +- .../veles-realm-ensure-job.yaml | 2 +- services/keycloak/kustomization.yaml | 53 +- .../cassandra_gitea_oidc_secret_ensure.sh | 157 ++ .../portal-e2e-client-job.yaml | 2 +- ...al-e2e-execute-actions-email-test-job.yaml | 2 +- .../portal-e2e-target-client-job.yaml | 2 +- ...al-e2e-token-exchange-permissions-job.yaml | 2 +- .../portal-e2e-token-exchange-test-job.yaml | 2 +- services/logging/Jenkinsfile.data-prepper | 44 + .../opensearch-dashboards-setup-job.yaml | 2 +- .../opensearch-ism-job.yaml | 2 +- .../opensearch-observability-setup-job.yaml | 2 +- services/logging/kustomization.yaml | 7 +- .../opensearch-dashboards-objects.yaml | 2 +- .../opensearch-observability-objects.yaml | 2 +- .../{ => apps}/ariadne-deployment.yaml | 2 +- .../maintenance/{ => apps}/ariadne-rbac.yaml | 2 +- .../{ => apps}/ariadne-service.yaml | 2 +- .../{ => apps}/ariadne-serviceaccount.yaml | 2 +- services/maintenance/apps/kustomization.yaml | 29 + .../{ => apps}/metis-configmap.yaml | 2 +- .../{ => apps}/metis-data-pvc.yaml | 2 +- .../{ => apps}/metis-deployment.yaml | 2 +- .../{ => apps}/metis-inventory.yaml | 2 +- .../metis-k3s-token-sync-cronjob.yaml | 3 +- .../maintenance/{ => apps}/metis-rbac.yaml | 2 +- .../maintenance/{ => apps}/metis-service.yaml | 2 +- .../{ => apps}/metis-serviceaccount.yaml | 2 +- .../metis-token-sync-serviceaccount.yaml | 2 +- .../{ => apps}/soteria-configmap.yaml | 2 +- .../{ => apps}/soteria-deployment.yaml | 2 +- .../maintenance/{ => apps}/soteria-rbac.yaml | 2 +- .../{ => apps}/soteria-service.yaml | 2 +- .../{ => apps}/soteria-serviceaccount.yaml | 2 +- .../maintenance/{ => bootstrap}/image.yaml | 2 +- .../maintenance/bootstrap/kustomization.yaml | 10 + .../{ => bootstrap}/namespace.yaml | 2 +- .../{ => bootstrap}/secretproviderclass.yaml | 2 +- .../{ => bootstrap}/vault-serviceaccount.yaml | 2 +- .../vault-sync-deployment.yaml | 2 +- services/maintenance/kustomization.yaml | 98 +- .../ariadne-migrate-job.yaml | 2 +- .../migration-jobs/kustomization.yaml | 6 + .../maintenance/networking/kustomization.yaml | 13 + .../{ => networking}/metis-certificate.yaml | 2 +- .../{ => networking}/metis-ingress.yaml | 2 +- .../{ => networking}/oauth2-proxy-metis.yaml | 2 +- .../oauth2-proxy-soteria-networkpolicy.yaml | 2 +- .../oauth2-proxy-soteria.yaml | 2 +- .../{ => networking}/soteria-certificate.yaml | 2 +- .../{ => networking}/soteria-ingress.yaml | 2 +- .../soteria-networkpolicy.yaml | 2 +- .../disable-k3s-traefik-daemonset.yaml | 2 +- .../disable-k3s-traefik-rbac.yaml | 2 +- .../disable-k3s-traefik-serviceaccount.yaml | 2 +- .../{ => node-ops}/image-sweeper-cronjob.yaml | 2 +- .../k3s-agent-restart-daemonset.yaml | 2 +- .../k3s-traefik-cleanup-rbac.yaml | 2 +- .../maintenance/node-ops/kustomization.yaml | 60 + .../metis-sentinel-amd64-daemonset.yaml | 2 +- .../metis-sentinel-arm64-daemonset.yaml | 2 +- .../node-image-sweeper-daemonset.yaml | 2 +- .../node-image-sweeper-serviceaccount.yaml | 2 +- .../{ => node-ops}/node-nofile-daemonset.yaml | 2 +- .../node-nofile-serviceaccount.yaml | 2 +- .../{ => node-ops}/pod-cleaner-cronjob.yaml | 2 +- .../{ => node-ops}/pod-cleaner-rbac.yaml | 2 +- .../rpi-resource-reservation-daemonset.yaml | 2 +- .../scripts/disable_k3s_traefik.sh | 0 .../scripts/k3s_traefik_cleanup.sh | 0 .../scripts/node_image_sweeper.sh | 0 .../{ => node-ops}/scripts/node_nofile.sh | 0 .../{ => node-ops}/scripts/pod_cleaner.sh | 0 .../scripts/rpi_resource_reservation.sh | 0 .../{ => node-ops}/scripts/titan_24_docker.sh | 0 .../titan-22-link-keeper-daemonset.yaml | 2 +- .../titan-24-docker-daemonset.yaml | 2 +- .../titan-24-docker-serviceaccount.yaml | 2 +- .../k3s-traefik-cleanup-job.yaml | 2 +- .../repair-jobs/kustomization.yaml | 7 + .../titan-24-rootfs-sweep-job.yaml | 2 +- .../grafana-org-bootstrap.yaml | 2 +- .../monitoring/dashboards/atlas-overview.json | 8 +- .../monitoring/dashboards/atlas-testing.json | 640 ++++- .../grafana-dashboard-overview.yaml | 8 +- .../monitoring/grafana-dashboard-testing.yaml | 640 ++++- services/monitoring/helmrelease.yaml | 35 +- services/monitoring/kustomization.yaml | 4 +- .../grafana-user-dedupe-job.yaml | 2 +- .../vmalert-atlas-availability.yaml | 119 +- services/pegasus/kustomization.yaml | 1 + services/quality/kustomization.yaml | 3 + services/quality/zap-baseline-configmap.yaml | 180 ++ services/quality/zap-baseline-cronjob.yaml | 77 + .../overlays/atlas/kustomization.yaml | 1 + services/veles/ingress.yaml | 1 + services/veles/kustomization.yaml | 1 + services/veles/redirect-middleware.yaml | 11 + terraform/aether/locals.tf | 22 + terraform/aether/main.tf | 79 + terraform/aether/outputs.tf | 27 + terraform/aether/providers.tf | 9 + terraform/aether/terraform.tfvars.example | 13 + terraform/aether/variables.tf | 74 + terraform/aether/versions.tf | 14 + testing/quality_contract.json | 12 +- testing/tests/test_publish_test_metrics.py | 12 +- .../test_publish_test_metrics_quality.py | 3 + testing/tests/test_repo_structure.py | 126 + testing/tests/test_semgrep_report.py | 170 ++ testing/tests/test_supply_chain_report.py | 22 + 257 files changed, 13256 insertions(+), 1821 deletions(-) create mode 100644 Makefile create mode 100644 ci/scripts/semgrep_report.py rename {infrastructure/modules/profiles/oceanus-validator => clusters/aether/flux-system/applications}/kustomization.yaml (52%) create mode 100644 clusters/aether/flux-system/kustomization.yaml rename clusters/{oceanus/applications => aether/flux-system/platform}/kustomization.yaml (57%) delete mode 100644 clusters/oceanus/flux-system/kustomization.yaml delete mode 100644 clusters/oceanus/platform/kustomization.yaml delete mode 100644 hosts/roles/oceanus_base/tasks/main.yaml rename infrastructure/{sources => core}/cert-manager/letsencrypt-prod.yaml (84%) rename infrastructure/{sources => core}/cert-manager/letsencrypt.yaml (84%) create mode 100644 knowledge/hermes-automated-triage-24h-plan.md delete mode 100755 scripts/comms_sync_kb.sh rename scripts/{ => manual-tests}/test_atlas_user_cleanup.py (100%) rename scripts/{ => manual-tests}/test_user_cleanup.py (100%) rename scripts/{ => manual-tests}/test_user_cleanup.sh (59%) rename scripts/{ => manual-tests}/test_vaultwarden_user_cleanup.py (98%) rename scripts/{ => manual-tests}/test_vaultwarden_user_cleanup.sh (64%) rename scripts/{ => ops}/build_ananke_node_helper.sh (90%) rename scripts/{ => ops}/build_harbor_bootstrap_bundle.sh (95%) rename scripts/{ => ops}/cluster_power_console.sh (70%) rename scripts/{ => ops}/cluster_power_recovery.sh (99%) rename scripts/{ => ops}/crypto_wallet_monero_setup.fish (99%) mode change 100644 => 100755 rename scripts/{ => ops}/crypto_wallet_sui_setup.fish (99%) mode change 100644 => 100755 rename scripts/{ => ops}/flux_hammer.fish (100%) rename scripts/{ => ops}/jellyfin_manual_load.fish (95%) rename scripts/{ => ops}/k3s_version_update.fish (99%) mode change 100644 => 100755 rename scripts/{ => ops}/longhorn_volume_usage.fish (100%) rename scripts/{ => ops}/node_recover.sh (98%) rename scripts/{ => ops}/styx_kioskification.sh (100%) mode change 100644 => 100755 rename scripts/{ => ops}/styx_prep.sh (100%) rename scripts/{ => ops}/styx_prep_nvme_luks.sh (100%) rename scripts/{ => ops}/verify_jenkins_workspace_cleanup_rollout.sh (100%) rename scripts/{ => render}/dashboards_render_atlas.py (97%) rename scripts/{ => render}/dashboards_render_logs.py (96%) create mode 100755 scripts/render/flux_inventory.py rename scripts/{ => render}/knowledge_render_atlas.py (98%) rename scripts/{ => render}/logging_render_observability.py (95%) rename scripts/{ => render}/monitoring_render_postmark_exporter.py (70%) create mode 100755 scripts/sync/comms_sync_kb.sh rename scripts/{ => sync}/gitea_cred_sync.sh (100%) rename scripts/{ => sync}/gitops_cred_sync.sh (100%) rename scripts/{ => sync}/jenkins_cred_sync.sh (100%) rename scripts/{ => sync}/monitoring_postmark_exporter.py (100%) create mode 100644 scripts/tests/test_monitoring_query_capacity.py rename services/bstein-dev-home/{oneoffs/migrations => migration-jobs}/kustomization.yaml (66%) rename services/bstein-dev-home/{oneoffs/migrations => migration-jobs}/portal-migrate-job.yaml (96%) rename services/bstein-dev-home/{oneoffs => validation-jobs}/portal-onboarding-e2e-test-job.yaml (97%) rename services/comms/{oneoffs => bootstrap-jobs}/comms-secrets-ensure-job.yaml (98%) rename services/comms/{oneoffs => bootstrap-jobs}/mas-admin-client-secret-ensure-job.yaml (98%) rename services/comms/{oneoffs => bootstrap-jobs}/mas-db-ensure-job.yaml (98%) rename services/comms/{oneoffs => bootstrap-jobs}/mas-local-users-ensure-job.yaml (99%) rename services/comms/{oneoffs => bootstrap-jobs}/synapse-admin-ensure-job.yaml (99%) rename services/comms/{oneoffs => bootstrap-jobs}/synapse-seeder-admin-ensure-job.yaml (98%) rename services/comms/{oneoffs => bootstrap-jobs}/synapse-signingkey-ensure-job.yaml (97%) rename services/comms/{oneoffs => bootstrap-jobs}/synapse-user-seed-job.yaml (99%) create mode 100644 services/comms/knowledge/hermes-automated-triage-24h-plan.md create mode 100644 services/comms/knowledge/runbooks/cluster-power-recovery.md rename services/comms/{oneoffs => repair-jobs}/bstein-force-leave-job.yaml (99%) rename services/comms/{oneoffs => repair-jobs}/othrys-kick-numeric-job.yaml (99%) rename services/finance/{oneoffs => bootstrap-jobs}/finance-secrets-ensure-job.yaml (96%) rename services/keycloak/{oneoffs => bootstrap-jobs}/actual-oidc-secret-ensure-job.yaml (96%) create mode 100644 services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml create mode 100644 services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml rename services/keycloak/{oneoffs => bootstrap-jobs}/harbor-oidc-secret-ensure-job.yaml (96%) rename services/keycloak/{oneoffs => bootstrap-jobs}/hermes-dashboard-oidc-client-job.yaml (96%) rename services/keycloak/{oneoffs => bootstrap-jobs}/ldap-federation-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/logs-oidc-secret-ensure-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/mas-secrets-ensure-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/metis-node-passwords-secret-ensure-job.yaml (98%) rename services/keycloak/{oneoffs => bootstrap-jobs}/metis-oidc-secret-ensure-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/metis-ssh-keys-secret-ensure-job.yaml (98%) rename services/keycloak/{oneoffs => bootstrap-jobs}/portal-admin-client-secret-ensure-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/portal-groups-scope-ensure-job.yaml (98%) rename services/keycloak/{oneoffs => bootstrap-jobs}/quality-oidc-secret-ensure-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/realm-settings-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/soteria-oidc-secret-ensure-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/synapse-oidc-secret-ensure-job.yaml (98%) rename services/keycloak/{oneoffs => bootstrap-jobs}/user-overrides-job.yaml (99%) rename services/keycloak/{oneoffs => bootstrap-jobs}/vault-oidc-secret-ensure-job.yaml (96%) rename services/keycloak/{oneoffs => bootstrap-jobs}/veles-gitea-oidc-secret-ensure-job.yaml (96%) rename services/keycloak/{oneoffs => bootstrap-jobs}/veles-realm-ensure-job.yaml (99%) create mode 100644 services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh rename services/keycloak/{oneoffs => validation-jobs}/portal-e2e-client-job.yaml (99%) rename services/keycloak/{oneoffs => validation-jobs}/portal-e2e-execute-actions-email-test-job.yaml (97%) rename services/keycloak/{oneoffs => validation-jobs}/portal-e2e-target-client-job.yaml (99%) rename services/keycloak/{oneoffs => validation-jobs}/portal-e2e-token-exchange-permissions-job.yaml (99%) rename services/keycloak/{oneoffs => validation-jobs}/portal-e2e-token-exchange-test-job.yaml (98%) rename services/logging/{oneoffs => bootstrap-jobs}/opensearch-dashboards-setup-job.yaml (97%) rename services/logging/{oneoffs => bootstrap-jobs}/opensearch-ism-job.yaml (98%) rename services/logging/{oneoffs => bootstrap-jobs}/opensearch-observability-setup-job.yaml (95%) rename services/maintenance/{ => apps}/ariadne-deployment.yaml (99%) rename services/maintenance/{ => apps}/ariadne-rbac.yaml (97%) rename services/maintenance/{ => apps}/ariadne-service.yaml (78%) rename services/maintenance/{ => apps}/ariadne-serviceaccount.yaml (69%) create mode 100644 services/maintenance/apps/kustomization.yaml rename services/maintenance/{ => apps}/metis-configmap.yaml (97%) rename services/maintenance/{ => apps}/metis-data-pvc.yaml (82%) rename services/maintenance/{ => apps}/metis-deployment.yaml (98%) rename services/maintenance/{ => apps}/metis-inventory.yaml (98%) rename services/maintenance/{ => apps}/metis-k3s-token-sync-cronjob.yaml (96%) rename services/maintenance/{ => apps}/metis-rbac.yaml (96%) rename services/maintenance/{ => apps}/metis-service.yaml (86%) rename services/maintenance/{ => apps}/metis-serviceaccount.yaml (70%) rename services/maintenance/{ => apps}/metis-token-sync-serviceaccount.yaml (59%) rename services/maintenance/{ => apps}/soteria-configmap.yaml (97%) rename services/maintenance/{ => apps}/soteria-deployment.yaml (98%) rename services/maintenance/{ => apps}/soteria-rbac.yaml (95%) rename services/maintenance/{ => apps}/soteria-service.yaml (87%) rename services/maintenance/{ => apps}/soteria-serviceaccount.yaml (70%) rename services/maintenance/{ => bootstrap}/image.yaml (98%) create mode 100644 services/maintenance/bootstrap/kustomization.yaml rename services/maintenance/{ => bootstrap}/namespace.yaml (55%) rename services/maintenance/{ => bootstrap}/secretproviderclass.yaml (96%) rename services/maintenance/{ => bootstrap}/vault-serviceaccount.yaml (63%) rename services/maintenance/{ => bootstrap}/vault-sync-deployment.yaml (95%) rename services/maintenance/{oneoffs => migration-jobs}/ariadne-migrate-job.yaml (96%) create mode 100644 services/maintenance/migration-jobs/kustomization.yaml create mode 100644 services/maintenance/networking/kustomization.yaml rename services/maintenance/{ => networking}/metis-certificate.yaml (80%) rename services/maintenance/{ => networking}/metis-ingress.yaml (92%) rename services/maintenance/{ => networking}/oauth2-proxy-metis.yaml (98%) rename services/maintenance/{ => networking}/oauth2-proxy-soteria-networkpolicy.yaml (86%) rename services/maintenance/{ => networking}/oauth2-proxy-soteria.yaml (98%) rename services/maintenance/{ => networking}/soteria-certificate.yaml (79%) rename services/maintenance/{ => networking}/soteria-ingress.yaml (92%) rename services/maintenance/{ => networking}/soteria-networkpolicy.yaml (89%) rename services/maintenance/{ => node-ops}/disable-k3s-traefik-daemonset.yaml (95%) rename services/maintenance/{ => node-ops}/disable-k3s-traefik-rbac.yaml (94%) rename services/maintenance/{ => node-ops}/disable-k3s-traefik-serviceaccount.yaml (57%) rename services/maintenance/{ => node-ops}/image-sweeper-cronjob.yaml (96%) rename services/maintenance/{ => node-ops}/k3s-agent-restart-daemonset.yaml (95%) rename services/maintenance/{ => node-ops}/k3s-traefik-cleanup-rbac.yaml (95%) create mode 100644 services/maintenance/node-ops/kustomization.yaml rename services/maintenance/{ => node-ops}/metis-sentinel-amd64-daemonset.yaml (94%) rename services/maintenance/{ => node-ops}/metis-sentinel-arm64-daemonset.yaml (94%) rename services/maintenance/{ => node-ops}/node-image-sweeper-daemonset.yaml (96%) rename services/maintenance/{ => node-ops}/node-image-sweeper-serviceaccount.yaml (57%) rename services/maintenance/{ => node-ops}/node-nofile-daemonset.yaml (95%) rename services/maintenance/{ => node-ops}/node-nofile-serviceaccount.yaml (58%) rename services/maintenance/{ => node-ops}/pod-cleaner-cronjob.yaml (94%) rename services/maintenance/{ => node-ops}/pod-cleaner-rbac.yaml (91%) rename services/maintenance/{ => node-ops}/rpi-resource-reservation-daemonset.yaml (96%) rename services/maintenance/{ => node-ops}/scripts/disable_k3s_traefik.sh (100%) rename services/maintenance/{ => node-ops}/scripts/k3s_traefik_cleanup.sh (100%) rename services/maintenance/{ => node-ops}/scripts/node_image_sweeper.sh (100%) rename services/maintenance/{ => node-ops}/scripts/node_nofile.sh (100%) rename services/maintenance/{ => node-ops}/scripts/pod_cleaner.sh (100%) rename services/maintenance/{ => node-ops}/scripts/rpi_resource_reservation.sh (100%) rename services/maintenance/{ => node-ops}/scripts/titan_24_docker.sh (100%) rename services/maintenance/{ => node-ops}/titan-22-link-keeper-daemonset.yaml (97%) rename services/maintenance/{ => node-ops}/titan-24-docker-daemonset.yaml (95%) rename services/maintenance/{ => node-ops}/titan-24-docker-serviceaccount.yaml (58%) rename services/maintenance/{oneoffs => repair-jobs}/k3s-traefik-cleanup-job.yaml (96%) create mode 100644 services/maintenance/repair-jobs/kustomization.yaml rename services/maintenance/{oneoffs => repair-jobs}/titan-24-rootfs-sweep-job.yaml (96%) rename services/monitoring/{oneoffs => bootstrap-jobs}/grafana-org-bootstrap.yaml (98%) rename services/monitoring/{oneoffs => repair-jobs}/grafana-user-dedupe-job.yaml (98%) create mode 100644 services/quality/zap-baseline-configmap.yaml create mode 100644 services/quality/zap-baseline-cronjob.yaml create mode 100644 services/veles/redirect-middleware.yaml create mode 100644 terraform/aether/locals.tf create mode 100644 terraform/aether/main.tf create mode 100644 terraform/aether/outputs.tf create mode 100644 terraform/aether/providers.tf create mode 100644 terraform/aether/terraform.tfvars.example create mode 100644 terraform/aether/variables.tf create mode 100644 terraform/aether/versions.tf create mode 100644 testing/tests/test_repo_structure.py create mode 100644 testing/tests/test_semgrep_report.py diff --git a/.gitignore b/.gitignore index 136020aaa..09c76a24b 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ __pycache__/ *.py[cod] .pytest_cache +.ruff_cache/ .coverage build/ test-results/ @@ -12,6 +13,7 @@ artifacts/ .venv .venv-ci tmp/ +.mainfix/ .terraform/ **/.terraform/ *.tfvars diff --git a/Jenkinsfile b/Jenkinsfile index e3052fcc8..2ef693a5f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -56,6 +56,11 @@ spec: command: - cat tty: true + - name: semgrep + image: semgrep/semgrep:1.171.0 + command: + - cat + tty: true """ } } @@ -70,6 +75,8 @@ spec: VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428' QUALITY_GATE_SONARQUBE_ENFORCE = '0' QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' + QUALITY_GATE_SEMGREP_ENFORCE = '0' + QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json' QUALITY_GATE_IRONBANK_ENFORCE = '1' QUALITY_GATE_IRONBANK_REQUIRED = '0' QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' @@ -110,6 +117,26 @@ spec: ''' } } + stage('Collect Semgrep evidence') { + steps { + container('semgrep') { + sh '''#!/bin/sh + set -eu + mkdir -p build + set +e + semgrep scan --config auto --metrics=off --json --output build/semgrep.json . + semgrep_rc=$? + set -e + printf '%s\n' "${semgrep_rc}" > build/semgrep.rc + ''' + } + sh ''' + set -eu + semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)" + python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json + ''' + } + } stage('Collect SonarQube evidence') { steps { container('quality-tools') { @@ -126,6 +153,7 @@ spec: "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx" ) [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml") + [ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json") set +e sonar-scanner "${args[@]}" | tee build/sonar-scanner.log rc=${PIPESTATUS[0]} @@ -326,6 +354,33 @@ PY esac fi + if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then + semgrep_status="$(python3 - <<'PY' +import json +from pathlib import Path + +path = Path("build/semgrep-report.json") +if not path.exists(): + print("missing") + raise SystemExit(0) +try: + payload = json.loads(path.read_text(encoding="utf-8")) +except Exception: # noqa: BLE001 + print("error") + raise SystemExit(0) +status = str(payload.get("status") or "").strip().lower() +print(status or "missing") +PY +)" + case "${semgrep_status}" in + ok|pass|passed|success) ;; + *) + echo "semgrep gate failed: ${semgrep_status}" >&2 + fail=1 + ;; + esac + fi + ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}" if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then ironbank_required=1 diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..10df79e1d --- /dev/null +++ b/Makefile @@ -0,0 +1,105 @@ +# Makefile +SHELL := /usr/bin/env bash + +KUSTOMIZE ?= kustomize +PYTHON ?= $(if $(wildcard .venv/bin/python),.venv/bin/python,python3) +TERRAFORM ?= terraform +TRIVY ?= trivy +BUILD_DIR ?= build +ATLAS_FLUX_ROOT := clusters/atlas/flux-system +AETHER_FLUX_ROOT := clusters/aether/flux-system +AETHER_TF_DIR := terraform/aether +FLUX_KUSTOMIZATION ?= flux-system +FLUX_NAMESPACE ?= flux-system +FLUX_PATH ?= $(ATLAS_FLUX_ROOT) +TRIVY_CACHE_DIR ?= $(BUILD_DIR)/trivy-cache +TRIVY_DB_FLAGS ?= --skip-db-update + +.PHONY: help render render-atlas render-aether render-services render-infrastructure validate dashboards knowledge test flux-diff flux-inventory trivy-scan security-report security aether-fmt aether-init aether-validate aether-plan + +help: + @printf '%s\n' \ + 'Targets:' \ + ' render Render Atlas, Aether, services, and infrastructure.' \ + ' render-atlas Render the Atlas Flux root.' \ + ' render-aether Render the Aether Flux root.' \ + ' render-services Render every service kustomization.' \ + ' render-infrastructure Render infrastructure kustomizations.' \ + ' validate Run render checks plus the local quality gate.' \ + ' dashboards Regenerate Grafana dashboard artifacts.' \ + ' knowledge Regenerate Atlas knowledge artifacts and comms mirror.' \ + ' test Run the local quality gate.' \ + ' flux-diff Diff a Flux Kustomization; override FLUX_KUSTOMIZATION/FLUX_PATH.' \ + ' flux-inventory Print Flux Kustomization inventory.' \ + ' trivy-scan Run the local Trivy filesystem scan.' \ + ' security-report Build IronBank report from build/trivy-fs.json.' \ + ' security Run Trivy scan and build the IronBank report.' \ + ' aether-fmt Check Terraform formatting for Aether.' \ + ' aether-init Initialize Aether Terraform providers.' \ + ' aether-validate Validate Aether Terraform after init.' \ + ' aether-plan Plan Aether Terraform from local variables/env.' + +render: render-atlas render-aether render-services render-infrastructure + +render-atlas: + $(KUSTOMIZE) build $(ATLAS_FLUX_ROOT) >/tmp/titan-iac-atlas-flux-system.yaml + +render-aether: + $(KUSTOMIZE) build $(AETHER_FLUX_ROOT) >/tmp/titan-iac-aether-flux-system.yaml + +render-services: + @set -euo pipefail; \ + while IFS= read -r k; do \ + d="$${k%/kustomization.yaml}"; \ + printf 'render %s\n' "$$d"; \ + $(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \ + done < <(find services -name kustomization.yaml | sort) + +render-infrastructure: + @set -euo pipefail; \ + while IFS= read -r k; do \ + d="$${k%/kustomization.yaml}"; \ + printf 'render %s\n' "$$d"; \ + $(KUSTOMIZE) build "$$d" >/tmp/titan-iac-render.yaml; \ + done < <(find infrastructure -name kustomization.yaml | sort) + +validate: render test + +dashboards: + $(PYTHON) scripts/render/dashboards_render_atlas.py --build + $(PYTHON) scripts/render/dashboards_render_logs.py --build + $(PYTHON) scripts/render/logging_render_observability.py --build + +knowledge: + $(PYTHON) scripts/render/knowledge_render_atlas.py --write --sync-comms + +test: + $(PYTHON) -m testing.quality_gate --profile local --build-dir $(BUILD_DIR) + +flux-diff: + flux diff kustomization $(FLUX_KUSTOMIZATION) --namespace $(FLUX_NAMESPACE) --path $(FLUX_PATH) + +flux-inventory: + $(PYTHON) scripts/render/flux_inventory.py $(ATLAS_FLUX_ROOT) + +trivy-scan: + mkdir -p $(BUILD_DIR) + $(TRIVY) fs --cache-dir "$(TRIVY_CACHE_DIR)" $(TRIVY_DB_FLAGS) --skip-files clusters/atlas/flux-system/gotk-components.yaml --timeout 5m --no-progress --format json --output $(BUILD_DIR)/trivy-fs.json --scanners vuln,secret,misconfig --severity HIGH,CRITICAL . + +security-report: + @test -s $(BUILD_DIR)/trivy-fs.json || { printf '%s\n' 'missing build/trivy-fs.json; run make trivy-scan first'; exit 1; } + $(PYTHON) ci/scripts/supply_chain_report.py --trivy-json $(BUILD_DIR)/trivy-fs.json --waivers ci/titan-iac-trivy-waivers.json --output $(BUILD_DIR)/ironbank-compliance.json + +security: trivy-scan security-report + +aether-fmt: + $(TERRAFORM) -chdir=$(AETHER_TF_DIR) fmt -check + +aether-init: + $(TERRAFORM) -chdir=$(AETHER_TF_DIR) init + +aether-validate: + $(TERRAFORM) -chdir=$(AETHER_TF_DIR) validate + +aether-plan: + $(TERRAFORM) -chdir=$(AETHER_TF_DIR) plan diff --git a/ci/Jenkinsfile.titan-iac b/ci/Jenkinsfile.titan-iac index 55c6c2036..8e3e3f133 100644 --- a/ci/Jenkinsfile.titan-iac +++ b/ci/Jenkinsfile.titan-iac @@ -55,6 +55,11 @@ spec: command: - cat tty: true + - name: semgrep + image: semgrep/semgrep:1.171.0 + command: + - cat + tty: true """ } } @@ -69,6 +74,8 @@ spec: VM_URL = 'http://victoria-metrics-single-server.monitoring.svc.cluster.local:8428' QUALITY_GATE_SONARQUBE_ENFORCE = '0' QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' + QUALITY_GATE_SEMGREP_ENFORCE = '0' + QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json' QUALITY_GATE_IRONBANK_ENFORCE = '1' QUALITY_GATE_IRONBANK_REQUIRED = '0' QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' @@ -109,6 +116,26 @@ spec: ''' } } + stage('Collect Semgrep evidence') { + steps { + container('semgrep') { + sh '''#!/bin/sh + set -eu + mkdir -p build + set +e + semgrep scan --config auto --metrics=off --json --output build/semgrep.json . + semgrep_rc=$? + set -e + printf '%s\n' "${semgrep_rc}" > build/semgrep.rc + ''' + } + sh ''' + set -eu + semgrep_rc="$(cat build/semgrep.rc 2>/dev/null || echo 2)" + python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json + ''' + } + } stage('Collect SonarQube evidence') { steps { container('quality-tools') { @@ -125,6 +152,7 @@ spec: "-Dsonar.test.inclusions=**/tests/**,**/testing/**,**/*_test.go,**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx" ) [ -f build/coverage-unit.xml ] && args+=("-Dsonar.python.coverage.reportPaths=build/coverage-unit.xml") + [ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json") set +e sonar-scanner "${args[@]}" | tee build/sonar-scanner.log rc=${PIPESTATUS[0]} @@ -325,6 +353,33 @@ PY esac fi + if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then + semgrep_status="$(python3 - <<'PY' +import json +from pathlib import Path + +path = Path("build/semgrep-report.json") +if not path.exists(): + print("missing") + raise SystemExit(0) +try: + payload = json.loads(path.read_text(encoding="utf-8")) +except Exception: # noqa: BLE001 + print("error") + raise SystemExit(0) +status = str(payload.get("status") or "").strip().lower() +print(status or "missing") +PY +)" + case "${semgrep_status}" in + ok|pass|passed|success) ;; + *) + echo "semgrep gate failed: ${semgrep_status}" >&2 + fail=1 + ;; + esac + fi + ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-0}" if [ "${PUBLISH_IMAGES:-false}" = "true" ]; then ironbank_required=1 diff --git a/ci/scripts/publish_test_metrics.py b/ci/scripts/publish_test_metrics.py index 216655c7e..adafb0188 100644 --- a/ci/scripts/publish_test_metrics.py +++ b/ci/scripts/publish_test_metrics.py @@ -20,6 +20,7 @@ CANONICAL_CHECKS = _quality_helpers.CANONICAL_CHECKS _build_check_statuses = _quality_helpers._build_check_statuses _combine_statuses = _quality_helpers._combine_statuses _infer_sonarqube_status = _quality_helpers._infer_sonarqube_status +_infer_semgrep_status = _quality_helpers._infer_semgrep_status _infer_source_lines_over_500 = _quality_helpers._infer_source_lines_over_500 _infer_supply_chain_status = _quality_helpers._infer_supply_chain_status _infer_workspace_coverage_percent = _quality_helpers._infer_workspace_coverage_percent @@ -286,6 +287,7 @@ def main() -> int: if source_lines_over_500 <= 0: source_lines_over_500 = _infer_source_lines_over_500(summary) sonarqube_report = _load_optional_json(os.getenv("QUALITY_GATE_SONARQUBE_REPORT", "build/sonarqube-quality-gate.json")) + semgrep_report = _load_optional_json(os.getenv("QUALITY_GATE_SEMGREP_REPORT", "build/semgrep-report.json")) supply_chain_report = _load_optional_json(os.getenv("QUALITY_GATE_IRONBANK_REPORT", "build/ironbank-compliance.json")) truthy = {"1", "true", "yes", "on"} supply_chain_required = ( @@ -298,6 +300,7 @@ def main() -> int: workspace_line_coverage_percent=workspace_line_coverage_percent, source_lines_over_500=source_lines_over_500, sonarqube_report=sonarqube_report, + semgrep_report=semgrep_report, supply_chain_report=supply_chain_report, supply_chain_required=supply_chain_required, ) diff --git a/ci/scripts/publish_test_metrics_quality.py b/ci/scripts/publish_test_metrics_quality.py index fca8cfa73..cf40793f2 100644 --- a/ci/scripts/publish_test_metrics_quality.py +++ b/ci/scripts/publish_test_metrics_quality.py @@ -18,6 +18,7 @@ CANONICAL_CHECKS = [ "docs_naming", "gate_glue", "sonarqube", + "semgrep", "supply_chain", ] @@ -135,12 +136,26 @@ def _infer_supply_chain_status(report: dict, required: bool) -> str: return normalized +def _infer_semgrep_status(report: dict) -> str: + """Infer canonical Semgrep check status from its JSON report payload.""" + if not report: + return "not_applicable" + status = report.get("status") + if status is None: + blocking_findings = report.get("blocking_findings") + errors_total = report.get("errors_total") + if isinstance(blocking_findings, int) and isinstance(errors_total, int): + return "failed" if blocking_findings > 0 or errors_total > 0 else "ok" + return _normalize_result_status(str(status) if status is not None else None, default="failed") + + def _build_check_statuses( summary: dict | None, tests: dict[str, int], workspace_line_coverage_percent: float, source_lines_over_500: int, sonarqube_report: dict, + semgrep_report: dict, supply_chain_report: dict, supply_chain_required: bool, ) -> dict[str, str]: @@ -188,6 +203,7 @@ def _build_check_statuses( gate_glue_status = _combine_statuses(candidates) if candidates else "not_applicable" sonarqube_status = status_by_name.get("sonarqube") or _infer_sonarqube_status(sonarqube_report) + semgrep_status = status_by_name.get("semgrep") or _infer_semgrep_status(semgrep_report) supply_chain_status = status_by_name.get("supply_chain") or _infer_supply_chain_status( supply_chain_report, required=supply_chain_required, @@ -200,5 +216,6 @@ def _build_check_statuses( "docs_naming": docs_naming_status, "gate_glue": gate_glue_status, "sonarqube": sonarqube_status, + "semgrep": semgrep_status, "supply_chain": supply_chain_status, } diff --git a/ci/scripts/semgrep_report.py b/ci/scripts/semgrep_report.py new file mode 100644 index 000000000..b4ed5a2dc --- /dev/null +++ b/ci/scripts/semgrep_report.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Build a compact Semgrep gate report from Semgrep JSON output.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +BLOCKING_SEVERITIES = {"ERROR"} +SEVERITIES = ("ERROR", "WARNING", "INFO", "UNKNOWN") +SONAR_SEVERITY_BY_SEMGREP = { + "ERROR": "CRITICAL", + "WARNING": "MAJOR", + "INFO": "MINOR", + "UNKNOWN": "INFO", +} + + +def _read_json(path: Path) -> dict[str, Any]: + """Read a JSON object from disk, returning an error-shaped payload on failure.""" + if not path.exists(): + return {"errors": [{"message": f"report missing: {path}"}], "results": []} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {"errors": [{"message": f"invalid JSON: {exc}"}], "results": []} + if not isinstance(payload, dict): + return {"errors": [{"message": "report payload is not an object"}], "results": []} + return payload + + +def _finding_severity(finding: dict[str, Any]) -> str: + """Return a normalized Semgrep finding severity.""" + extra = finding.get("extra") + severity = extra.get("severity") if isinstance(extra, dict) else None + normalized = str(severity or "UNKNOWN").strip().upper() + return normalized if normalized in SEVERITIES else "UNKNOWN" + + +def build_report( + semgrep_payload: dict[str, Any], + *, + semgrep_exit_code: int, + blocking_severities: set[str] | None = None, +) -> dict[str, Any]: + """Summarize Semgrep evidence into the quality-gate report contract.""" + blocking = blocking_severities or BLOCKING_SEVERITIES + raw_results = semgrep_payload.get("results", []) + raw_errors = semgrep_payload.get("errors", []) + results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] + errors = [item for item in raw_errors if isinstance(item, dict)] if isinstance(raw_errors, list) else [] + + severity_counts = {severity: 0 for severity in SEVERITIES} + blocking_findings = 0 + for finding in results: + severity = _finding_severity(finding) + severity_counts[severity] += 1 + if severity in blocking: + blocking_findings += 1 + + engine_error = semgrep_exit_code not in {0, 1} or bool(errors) + status = "failed" if engine_error or blocking_findings else "ok" + return { + "status": status, + "scanner": "semgrep", + "semgrep_rc": semgrep_exit_code, + "findings_total": len(results), + "blocking_findings": blocking_findings, + "errors_total": len(errors), + "severity_counts": severity_counts, + "blocking_severities": sorted(blocking), + } + + +def _line_number(value: Any, default: int = 1) -> int: + """Return a Sonar-compatible one-indexed line number.""" + try: + line = int(value) + except (TypeError, ValueError): + return default + return max(line, default) + + +def _sonar_issue_from_finding(finding: dict[str, Any]) -> dict[str, Any] | None: + """Convert one Semgrep finding into SonarQube generic issue format.""" + path = str(finding.get("path") or "").strip() + if not path: + return None + extra = finding.get("extra") if isinstance(finding.get("extra"), dict) else {} + severity = _finding_severity(finding) + start = finding.get("start") if isinstance(finding.get("start"), dict) else {} + end = finding.get("end") if isinstance(finding.get("end"), dict) else {} + start_line = _line_number(start.get("line") if isinstance(start, dict) else None) + end_line = _line_number(end.get("line") if isinstance(end, dict) else None, start_line) + if end_line < start_line: + end_line = start_line + return { + "engineId": "semgrep", + "ruleId": str(finding.get("check_id") or "semgrep.unknown"), + "type": "VULNERABILITY" if severity == "ERROR" else "CODE_SMELL", + "severity": SONAR_SEVERITY_BY_SEMGREP[severity], + "primaryLocation": { + "message": str(extra.get("message") or finding.get("check_id") or "Semgrep finding"), + "filePath": path, + "textRange": { + "startLine": start_line, + "endLine": end_line, + }, + }, + } + + +def build_sonar_issues(semgrep_payload: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: + """Build SonarQube generic external issues from Semgrep JSON output.""" + raw_results = semgrep_payload.get("results", []) + results = [item for item in raw_results if isinstance(item, dict)] if isinstance(raw_results, list) else [] + issues = [issue for finding in results if (issue := _sonar_issue_from_finding(finding))] + return {"issues": issues} + + +def main(argv: list[str] | None = None) -> int: + """CLI entrypoint used by Jenkins after Semgrep finishes.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--semgrep-json", required=True) + parser.add_argument("--exit-code", type=int, default=0) + parser.add_argument("--output", required=True) + parser.add_argument("--sonar-issues-output") + parser.add_argument( + "--blocking-severity", + action="append", + default=[], + help="Severity that should mark the report failed. Defaults to ERROR.", + ) + args = parser.parse_args(argv) + + blocking = {item.strip().upper() for item in args.blocking_severity if item.strip()} + payload = _read_json(Path(args.semgrep_json)) + report = build_report( + payload, + semgrep_exit_code=args.exit_code, + blocking_severities=blocking or BLOCKING_SEVERITIES, + ) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n", encoding="utf-8") + if args.sonar_issues_output: + sonar_output = Path(args.sonar_issues_output) + sonar_output.parent.mkdir(parents=True, exist_ok=True) + sonar_output.write_text( + json.dumps(build_sonar_issues(payload), indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/ci/scripts/supply_chain_report.py b/ci/scripts/supply_chain_report.py index ca46783cb..dbb210b50 100644 --- a/ci/scripts/supply_chain_report.py +++ b/ci/scripts/supply_chain_report.py @@ -131,7 +131,11 @@ def build_report( critical = _count_vulnerabilities(trivy_payload, "CRITICAL") high = _count_vulnerabilities(trivy_payload, "HIGH") secrets = _count_secrets(trivy_payload) - status = "ok" if critical == 0 and secrets == 0 and not open_misconfigs else "failed" + status = ( + "ok" + if critical == 0 and secrets == 0 and not open_misconfigs and expired_waivers == 0 + else "failed" + ) return { "status": status, diff --git a/ci/titan-iac-trivy-waivers.json b/ci/titan-iac-trivy-waivers.json index 433a8b70e..9810262e0 100644 --- a/ci/titan-iac-trivy-waivers.json +++ b/ci/titan-iac-trivy-waivers.json @@ -1,7 +1,7 @@ { "version": 1, "generated_from": "Jenkins titan-iac build 225 Trivy filesystem scan", - "default_expires_at": "2026-05-22", + "default_expires_at": "2026-09-30", "ticket": "atlas-quality-wave-k8s-hardening", "default_reason": "Existing Kubernetes manifest hardening baseline accepted only for the first quality-gate rollout; fix or renew explicitly before expiry.", "misconfigurations": [ @@ -15,15 +15,15 @@ "id": "KSV-0009", "targets": [ "services/mailu/vip-controller.yaml", - "services/maintenance/k3s-agent-restart-daemonset.yaml" + "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml" ] }, { "id": "KSV-0010", "targets": [ - "services/maintenance/k3s-agent-restart-daemonset.yaml", - "services/maintenance/metis-sentinel-amd64-daemonset.yaml", - "services/maintenance/metis-sentinel-arm64-daemonset.yaml", + "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", + "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", + "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", "services/monitoring/jetson-tegrastats-exporter.yaml" ] }, @@ -47,8 +47,8 @@ "services/bstein-dev-home/backend-deployment.yaml", "services/bstein-dev-home/chat-ai-gateway-deployment.yaml", "services/bstein-dev-home/frontend-deployment.yaml", - "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml", - "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml", + "services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml", + "services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml", "services/bstein-dev-home/vault-sync-deployment.yaml", "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml", "services/comms/atlasbot-deployment.yaml", @@ -59,16 +59,16 @@ "services/comms/livekit-token-deployment.yaml", "services/comms/livekit.yaml", "services/comms/mas-deployment.yaml", - "services/comms/oneoffs/bstein-force-leave-job.yaml", - "services/comms/oneoffs/comms-secrets-ensure-job.yaml", - "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml", - "services/comms/oneoffs/mas-db-ensure-job.yaml", - "services/comms/oneoffs/mas-local-users-ensure-job.yaml", - "services/comms/oneoffs/othrys-kick-numeric-job.yaml", - "services/comms/oneoffs/synapse-admin-ensure-job.yaml", - "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml", - "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml", - "services/comms/oneoffs/synapse-user-seed-job.yaml", + "services/comms/repair-jobs/bstein-force-leave-job.yaml", + "services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml", + "services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml", + "services/comms/bootstrap-jobs/mas-db-ensure-job.yaml", + "services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml", + "services/comms/repair-jobs/othrys-kick-numeric-job.yaml", + "services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml", + "services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml", + "services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml", + "services/comms/bootstrap-jobs/synapse-user-seed-job.yaml", "services/comms/pin-othrys-job.yaml", "services/comms/reset-othrys-room-job.yaml", "services/comms/seed-othrys-room.yaml", @@ -83,7 +83,7 @@ "services/finance/firefly-cronjob.yaml", "services/finance/firefly-deployment.yaml", "services/finance/firefly-user-sync-cronjob.yaml", - "services/finance/oneoffs/finance-secrets-ensure-job.yaml", + "services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml", "services/gitea/deployment.yaml", "services/harbor/vault-sync-deployment.yaml", "services/health/wger-admin-ensure-cronjob.yaml", @@ -94,63 +94,62 @@ "services/jenkins/deployment.yaml", "services/jenkins/vault-sync-deployment.yaml", "services/keycloak/deployment.yaml", - "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/ldap-federation-job.yaml", - "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml", - "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml", - "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml", - "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml", - "services/keycloak/oneoffs/portal-e2e-client-job.yaml", - "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml", - "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml", - "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml", - "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml", - "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/realm-settings-job.yaml", - "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/user-overrides-job.yaml", - "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/ldap-federation-job.yaml", + "services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-client-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml", + "services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/realm-settings-job.yaml", + "services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/user-overrides-job.yaml", + "services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml", "services/keycloak/vault-sync-deployment.yaml", "services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml", "services/logging/oauth2-proxy.yaml", - "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml", - "services/logging/oneoffs/opensearch-ism-job.yaml", - "services/logging/oneoffs/opensearch-observability-setup-job.yaml", + "services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml", + "services/logging/bootstrap-jobs/opensearch-ism-job.yaml", + "services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml", "services/logging/opensearch-prune-cronjob.yaml", "services/logging/vault-sync-deployment.yaml", "services/mailu/mailu-sync-cronjob.yaml", "services/mailu/mailu-sync-listener.yaml", - "services/mailu/oneoffs/mailu-sync-job.yaml", "services/mailu/vault-sync-deployment.yaml", "services/mailu/vip-controller.yaml", - "services/maintenance/ariadne-deployment.yaml", - "services/maintenance/disable-k3s-traefik-daemonset.yaml", - "services/maintenance/image-sweeper-cronjob.yaml", - "services/maintenance/k3s-agent-restart-daemonset.yaml", - "services/maintenance/metis-deployment.yaml", - "services/maintenance/metis-k3s-token-sync-cronjob.yaml", - "services/maintenance/metis-sentinel-amd64-daemonset.yaml", - "services/maintenance/metis-sentinel-arm64-daemonset.yaml", - "services/maintenance/node-image-sweeper-daemonset.yaml", - "services/maintenance/node-nofile-daemonset.yaml", - "services/maintenance/oauth2-proxy-metis.yaml", - "services/maintenance/oauth2-proxy-soteria.yaml", - "services/maintenance/oneoffs/ariadne-migrate-job.yaml", - "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml", - "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml", - "services/maintenance/pod-cleaner-cronjob.yaml", - "services/maintenance/soteria-deployment.yaml", - "services/maintenance/vault-sync-deployment.yaml", + "services/maintenance/apps/ariadne-deployment.yaml", + "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", + "services/maintenance/node-ops/image-sweeper-cronjob.yaml", + "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", + "services/maintenance/apps/metis-deployment.yaml", + "services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml", + "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", + "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", + "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", + "services/maintenance/node-ops/node-nofile-daemonset.yaml", + "services/maintenance/networking/oauth2-proxy-metis.yaml", + "services/maintenance/networking/oauth2-proxy-soteria.yaml", + "services/maintenance/migration-jobs/ariadne-migrate-job.yaml", + "services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml", + "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml", + "services/maintenance/node-ops/pod-cleaner-cronjob.yaml", + "services/maintenance/apps/soteria-deployment.yaml", + "services/maintenance/bootstrap/vault-sync-deployment.yaml", "services/monitoring/dcgm-exporter.yaml", "services/monitoring/jetson-tegrastats-exporter.yaml", - "services/monitoring/oneoffs/grafana-org-bootstrap.yaml", - "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml", + "services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml", + "services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml", "services/monitoring/platform-quality-gateway-deployment.yaml", "services/monitoring/platform-quality-suite-probe-cronjob.yaml", "services/monitoring/postmark-exporter-deployment.yaml", @@ -188,15 +187,15 @@ "services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml", - "services/maintenance/disable-k3s-traefik-daemonset.yaml", - "services/maintenance/image-sweeper-cronjob.yaml", - "services/maintenance/k3s-agent-restart-daemonset.yaml", - "services/maintenance/metis-deployment.yaml", - "services/maintenance/metis-sentinel-amd64-daemonset.yaml", - "services/maintenance/metis-sentinel-arm64-daemonset.yaml", - "services/maintenance/node-image-sweeper-daemonset.yaml", - "services/maintenance/node-nofile-daemonset.yaml", - "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml", + "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", + "services/maintenance/node-ops/image-sweeper-cronjob.yaml", + "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", + "services/maintenance/apps/metis-deployment.yaml", + "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", + "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", + "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", + "services/maintenance/node-ops/node-nofile-daemonset.yaml", + "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml", "services/monitoring/dcgm-exporter.yaml", "services/monitoring/jetson-tegrastats-exporter.yaml" ] @@ -211,7 +210,7 @@ "services/comms/comms-secrets-ensure-rbac.yaml", "services/comms/mas-db-ensure-rbac.yaml", "services/comms/mas-secrets-ensure-rbac.yaml", - "services/maintenance/soteria-rbac.yaml" + "services/maintenance/apps/soteria-rbac.yaml" ] }, { @@ -226,7 +225,7 @@ "services/comms/comms-secrets-ensure-rbac.yaml", "services/comms/mas-db-ensure-rbac.yaml", "services/jenkins/serviceaccount.yaml", - "services/maintenance/ariadne-rbac.yaml" + "services/maintenance/apps/ariadne-rbac.yaml" ] }, { @@ -235,8 +234,8 @@ "infrastructure/cert-manager/cleanup/cert-manager-cleanup-rbac.yaml", "infrastructure/longhorn/adopt/longhorn-adopt-rbac.yaml", "services/jenkins/serviceaccount.yaml", - "services/maintenance/disable-k3s-traefik-rbac.yaml", - "services/maintenance/k3s-traefik-cleanup-rbac.yaml" + "services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml", + "services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml" ] }, { @@ -266,8 +265,8 @@ "services/bstein-dev-home/backend-deployment.yaml", "services/bstein-dev-home/chat-ai-gateway-deployment.yaml", "services/bstein-dev-home/frontend-deployment.yaml", - "services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml", - "services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml", + "services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml", + "services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml", "services/bstein-dev-home/vault-sync-deployment.yaml", "services/bstein-dev-home/vaultwarden-cred-sync-cronjob.yaml", "services/comms/atlasbot-deployment.yaml", @@ -277,16 +276,16 @@ "services/comms/livekit-token-deployment.yaml", "services/comms/livekit.yaml", "services/comms/mas-deployment.yaml", - "services/comms/oneoffs/bstein-force-leave-job.yaml", - "services/comms/oneoffs/comms-secrets-ensure-job.yaml", - "services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml", - "services/comms/oneoffs/mas-db-ensure-job.yaml", - "services/comms/oneoffs/mas-local-users-ensure-job.yaml", - "services/comms/oneoffs/othrys-kick-numeric-job.yaml", - "services/comms/oneoffs/synapse-admin-ensure-job.yaml", - "services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml", - "services/comms/oneoffs/synapse-signingkey-ensure-job.yaml", - "services/comms/oneoffs/synapse-user-seed-job.yaml", + "services/comms/repair-jobs/bstein-force-leave-job.yaml", + "services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml", + "services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml", + "services/comms/bootstrap-jobs/mas-db-ensure-job.yaml", + "services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml", + "services/comms/repair-jobs/othrys-kick-numeric-job.yaml", + "services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml", + "services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml", + "services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml", + "services/comms/bootstrap-jobs/synapse-user-seed-job.yaml", "services/comms/pin-othrys-job.yaml", "services/comms/reset-othrys-room-job.yaml", "services/comms/seed-othrys-room.yaml", @@ -300,7 +299,7 @@ "services/finance/firefly-cronjob.yaml", "services/finance/firefly-deployment.yaml", "services/finance/firefly-user-sync-cronjob.yaml", - "services/finance/oneoffs/finance-secrets-ensure-job.yaml", + "services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml", "services/gitea/deployment.yaml", "services/harbor/vault-sync-deployment.yaml", "services/health/wger-admin-ensure-cronjob.yaml", @@ -309,63 +308,62 @@ "services/jellyfin/loader.yaml", "services/jenkins/deployment.yaml", "services/jenkins/vault-sync-deployment.yaml", - "services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/ldap-federation-job.yaml", - "services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/mas-secrets-ensure-job.yaml", - "services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml", - "services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml", - "services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml", - "services/keycloak/oneoffs/portal-e2e-client-job.yaml", - "services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml", - "services/keycloak/oneoffs/portal-e2e-target-client-job.yaml", - "services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml", - "services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml", - "services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/realm-settings-job.yaml", - "services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml", - "services/keycloak/oneoffs/user-overrides-job.yaml", - "services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/ldap-federation-job.yaml", + "services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-client-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml", + "services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml", + "services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/realm-settings-job.yaml", + "services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml", + "services/keycloak/bootstrap-jobs/user-overrides-job.yaml", + "services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml", "services/keycloak/vault-sync-deployment.yaml", "services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml", "services/logging/oauth2-proxy.yaml", - "services/logging/oneoffs/opensearch-dashboards-setup-job.yaml", - "services/logging/oneoffs/opensearch-ism-job.yaml", - "services/logging/oneoffs/opensearch-observability-setup-job.yaml", + "services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml", + "services/logging/bootstrap-jobs/opensearch-ism-job.yaml", + "services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml", "services/logging/opensearch-prune-cronjob.yaml", "services/logging/vault-sync-deployment.yaml", "services/mailu/mailu-sync-cronjob.yaml", "services/mailu/mailu-sync-listener.yaml", - "services/mailu/oneoffs/mailu-sync-job.yaml", "services/mailu/vault-sync-deployment.yaml", "services/mailu/vip-controller.yaml", - "services/maintenance/ariadne-deployment.yaml", - "services/maintenance/disable-k3s-traefik-daemonset.yaml", - "services/maintenance/image-sweeper-cronjob.yaml", - "services/maintenance/k3s-agent-restart-daemonset.yaml", - "services/maintenance/metis-deployment.yaml", - "services/maintenance/metis-k3s-token-sync-cronjob.yaml", - "services/maintenance/metis-sentinel-amd64-daemonset.yaml", - "services/maintenance/metis-sentinel-arm64-daemonset.yaml", - "services/maintenance/node-image-sweeper-daemonset.yaml", - "services/maintenance/node-nofile-daemonset.yaml", - "services/maintenance/oauth2-proxy-metis.yaml", - "services/maintenance/oauth2-proxy-soteria.yaml", - "services/maintenance/oneoffs/ariadne-migrate-job.yaml", - "services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml", - "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml", - "services/maintenance/pod-cleaner-cronjob.yaml", - "services/maintenance/soteria-deployment.yaml", - "services/maintenance/vault-sync-deployment.yaml", + "services/maintenance/apps/ariadne-deployment.yaml", + "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", + "services/maintenance/node-ops/image-sweeper-cronjob.yaml", + "services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml", + "services/maintenance/apps/metis-deployment.yaml", + "services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml", + "services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml", + "services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml", + "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", + "services/maintenance/node-ops/node-nofile-daemonset.yaml", + "services/maintenance/networking/oauth2-proxy-metis.yaml", + "services/maintenance/networking/oauth2-proxy-soteria.yaml", + "services/maintenance/migration-jobs/ariadne-migrate-job.yaml", + "services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml", + "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml", + "services/maintenance/node-ops/pod-cleaner-cronjob.yaml", + "services/maintenance/apps/soteria-deployment.yaml", + "services/maintenance/bootstrap/vault-sync-deployment.yaml", "services/monitoring/dcgm-exporter.yaml", "services/monitoring/jetson-tegrastats-exporter.yaml", - "services/monitoring/oneoffs/grafana-org-bootstrap.yaml", - "services/monitoring/oneoffs/grafana-user-dedupe-job.yaml", + "services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml", + "services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml", "services/monitoring/platform-quality-gateway-deployment.yaml", "services/monitoring/platform-quality-suite-probe-cronjob.yaml", "services/monitoring/postmark-exporter-deployment.yaml", @@ -395,12 +393,12 @@ "services/logging/node-image-gc-rpi4-daemonset.yaml", "services/logging/node-image-prune-rpi5-daemonset.yaml", "services/logging/node-log-rotation-daemonset.yaml", - "services/maintenance/disable-k3s-traefik-daemonset.yaml", - "services/maintenance/image-sweeper-cronjob.yaml", - "services/maintenance/metis-deployment.yaml", - "services/maintenance/node-image-sweeper-daemonset.yaml", - "services/maintenance/node-nofile-daemonset.yaml", - "services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml" + "services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml", + "services/maintenance/node-ops/image-sweeper-cronjob.yaml", + "services/maintenance/apps/metis-deployment.yaml", + "services/maintenance/node-ops/node-image-sweeper-daemonset.yaml", + "services/maintenance/node-ops/node-nofile-daemonset.yaml", + "services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml" ] } ] diff --git a/infrastructure/modules/profiles/oceanus-validator/kustomization.yaml b/clusters/aether/flux-system/applications/kustomization.yaml similarity index 52% rename from infrastructure/modules/profiles/oceanus-validator/kustomization.yaml rename to clusters/aether/flux-system/applications/kustomization.yaml index 117c70a31..fad9bd8f3 100644 --- a/infrastructure/modules/profiles/oceanus-validator/kustomization.yaml +++ b/clusters/aether/flux-system/applications/kustomization.yaml @@ -1,4 +1,4 @@ -# infrastructure/modules/profiles/oceanus-validator/kustomization.yaml +# clusters/aether/flux-system/applications/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: [] diff --git a/clusters/aether/flux-system/kustomization.yaml b/clusters/aether/flux-system/kustomization.yaml new file mode 100644 index 000000000..70bc7c555 --- /dev/null +++ b/clusters/aether/flux-system/kustomization.yaml @@ -0,0 +1,6 @@ +# clusters/aether/flux-system/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - platform + - applications diff --git a/clusters/oceanus/applications/kustomization.yaml b/clusters/aether/flux-system/platform/kustomization.yaml similarity index 57% rename from clusters/oceanus/applications/kustomization.yaml rename to clusters/aether/flux-system/platform/kustomization.yaml index dca9ed8a4..5e7f0eb4a 100644 --- a/clusters/oceanus/applications/kustomization.yaml +++ b/clusters/aether/flux-system/platform/kustomization.yaml @@ -1,4 +1,4 @@ -# clusters/oceanus/applications/kustomization.yaml +# clusters/aether/flux-system/platform/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: [] diff --git a/clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml b/clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml index 396a1a5b0..e062626f0 100644 --- a/clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/bstein-dev-home-migrations/kustomization.yaml @@ -6,9 +6,10 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Migration jobs run only during portal schema changes." spec: interval: 10m - path: ./services/bstein-dev-home/oneoffs/migrations + path: ./services/bstein-dev-home/migration-jobs prune: true force: true sourceRef: diff --git a/clusters/atlas/flux-system/applications/finance/kustomization.yaml b/clusters/atlas/flux-system/applications/finance/kustomization.yaml index fde0072cc..5ad59dfcf 100644 --- a/clusters/atlas/flux-system/applications/finance/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/finance/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Finance apps are parked until the next storage and SSO pass." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/game-stream/kustomization.yaml b/clusters/atlas/flux-system/applications/game-stream/kustomization.yaml index d64a9bd30..7057a43bf 100644 --- a/clusters/atlas/flux-system/applications/game-stream/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/game-stream/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Game streaming is optional and resumes only for planned use." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/health/kustomization.yaml b/clusters/atlas/flux-system/applications/health/kustomization.yaml index 121383715..9722bdb3e 100644 --- a/clusters/atlas/flux-system/applications/health/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/health/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Health stack is staged pending account and mail flows." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml b/clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml index 132c640d3..94963ef59 100644 --- a/clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/jellyfin/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Media stack stays paused while auth and storage changes are staged." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml index 4f5ab5da2..dce79ab7f 100644 --- a/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/jenkins/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "CI controller changes are applied only during planned maintenance." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/mailu/kustomization.yaml b/clusters/atlas/flux-system/applications/mailu/kustomization.yaml index 4f3858dde..53b8bdfe9 100644 --- a/clusters/atlas/flux-system/applications/mailu/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/mailu/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Mail stack is staged and resumes only during mail rollout work." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml b/clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml index 2d311e376..c9be5e809 100644 --- a/clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/nextcloud-mail-sync/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Mail sync resumes after Nextcloud and Mailu are active." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml b/clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml index e4a480f8f..9062a74fa 100644 --- a/clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/nextcloud/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Nextcloud is staged pending storage, SSO, and mail validation." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/outline/kustomization.yaml b/clusters/atlas/flux-system/applications/outline/kustomization.yaml index b56bf2d6c..a5a92de29 100644 --- a/clusters/atlas/flux-system/applications/outline/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/outline/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Outline is staged until shared auth and mail are ready." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/planka/kustomization.yaml b/clusters/atlas/flux-system/applications/planka/kustomization.yaml index 3b204cc64..703af7bfd 100644 --- a/clusters/atlas/flux-system/applications/planka/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/planka/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Planka is staged until shared auth and mail are ready." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/quality/kustomization.yaml b/clusters/atlas/flux-system/applications/quality/kustomization.yaml index 8de26f7c9..4fc550a44 100644 --- a/clusters/atlas/flux-system/applications/quality/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/quality/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Quality stack changes resume only during quality-gate rollout work." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/typhon/kustomization.yaml b/clusters/atlas/flux-system/applications/typhon/kustomization.yaml index 9a536b923..4afdfeecb 100644 --- a/clusters/atlas/flux-system/applications/typhon/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/typhon/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Climate automation is staged until sensor and control loops are verified." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml b/clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml index 7842c2620..fad8bbe7a 100644 --- a/clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/vaultwarden/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Vaultwarden stays separate from SSO rollout and resumes by hand." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml b/clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml index baa990f51..87450a746 100644 --- a/clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/wallet-monero-temp/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Temporary wallet RPC stack is opt-in for maintenance windows." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml b/clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml index 5d939a628..87ebd5c3c 100644 --- a/clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/xmr-miner/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Mining workloads are disabled unless explicitly enabled for a short run." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/platform/descheduler/kustomization.yaml b/clusters/atlas/flux-system/platform/descheduler/kustomization.yaml index eef358c1b..29b5cdb00 100644 --- a/clusters/atlas/flux-system/platform/descheduler/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/descheduler/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Descheduler is paused during node recovery and placement stabilization." spec: interval: 30m suspend: true diff --git a/clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml b/clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml index 37a0b2113..fc23ed792 100644 --- a/clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/gitops-ui/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "GitOps UI is optional and resumes only for operator access work." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml b/clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml index 2cfa0f1c1..387251ae3 100644 --- a/clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/longhorn-ui/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Longhorn UI ingress is optional and opened only for storage work." spec: interval: 10m suspend: true diff --git a/clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml b/clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml index 6c8cae1b3..4d08e7bca 100644 --- a/clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/resource-guardrails/kustomization.yaml @@ -6,6 +6,7 @@ metadata: namespace: flux-system annotations: kustomize.toolkit.fluxcd.io/ssa: IfNotPresent + atlas.bstein.dev/suspend-reason: "Guardrail rollout is paused until service resource requests are normalized." spec: interval: 10m suspend: true diff --git a/clusters/oceanus/flux-system/kustomization.yaml b/clusters/oceanus/flux-system/kustomization.yaml deleted file mode 100644 index 0d2942dc9..000000000 --- a/clusters/oceanus/flux-system/kustomization.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# clusters/oceanus/flux-system/kustomization.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - # Populate when oceanus cluster is bootstrapped with Flux. - # - gotk-components.yaml - # - gotk-sync.yaml - - ../platform - - ../applications diff --git a/clusters/oceanus/platform/kustomization.yaml b/clusters/oceanus/platform/kustomization.yaml deleted file mode 100644 index 36d633888..000000000 --- a/clusters/oceanus/platform/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# clusters/oceanus/platform/kustomization.yaml -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - ../../infrastructure/modules/base - - ../../infrastructure/modules/profiles/oceanus-validator diff --git a/hosts/inventory/lab.yaml b/hosts/inventory/lab.yaml index b1859daf5..22e25343f 100644 --- a/hosts/inventory/lab.yaml +++ b/hosts/inventory/lab.yaml @@ -12,7 +12,7 @@ all: ansible_host: REPLACE_ME ansible_user: debian roleset: minipc_gpu - baremetal: + dedicated_hosts: hosts: titan-db: ansible_host: REPLACE_ME @@ -22,7 +22,3 @@ all: ansible_host: REPLACE_ME ansible_user: jump roleset: jumphost - oceanus: - ansible_host: REPLACE_ME - ansible_user: validator - roleset: validator diff --git a/hosts/playbooks/site.yaml b/hosts/playbooks/site.yaml index 04781f539..193a4fd1a 100644 --- a/hosts/playbooks/site.yaml +++ b/hosts/playbooks/site.yaml @@ -14,13 +14,6 @@ - common - titan_jh -- name: Configure oceanus validator host - hosts: oceanus - gather_facts: true - roles: - - common - - oceanus_base - - name: Prepare hybrid tethys node hosts: titan-24 gather_facts: true diff --git a/hosts/roles/oceanus_base/tasks/main.yaml b/hosts/roles/oceanus_base/tasks/main.yaml deleted file mode 100644 index e2b5de79f..000000000 --- a/hosts/roles/oceanus_base/tasks/main.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# hosts/roles/oceanus_base/tasks/main.yaml ---- -- name: Placeholder for oceanus base configuration - ansible.builtin.debug: - msg: "Install validator prerequisites and monitoring exporters here." - tags: ['oceanus'] diff --git a/infrastructure/sources/cert-manager/letsencrypt-prod.yaml b/infrastructure/core/cert-manager/letsencrypt-prod.yaml similarity index 84% rename from infrastructure/sources/cert-manager/letsencrypt-prod.yaml rename to infrastructure/core/cert-manager/letsencrypt-prod.yaml index 5795b0916..f80fdafa1 100644 --- a/infrastructure/sources/cert-manager/letsencrypt-prod.yaml +++ b/infrastructure/core/cert-manager/letsencrypt-prod.yaml @@ -1,4 +1,4 @@ -# infrastructure/sources/cert-manager/letsencrypt-prod.yaml +# infrastructure/core/cert-manager/letsencrypt-prod.yaml apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: diff --git a/infrastructure/sources/cert-manager/letsencrypt.yaml b/infrastructure/core/cert-manager/letsencrypt.yaml similarity index 84% rename from infrastructure/sources/cert-manager/letsencrypt.yaml rename to infrastructure/core/cert-manager/letsencrypt.yaml index 5fbe4e367..90cabed62 100644 --- a/infrastructure/sources/cert-manager/letsencrypt.yaml +++ b/infrastructure/core/cert-manager/letsencrypt.yaml @@ -1,4 +1,4 @@ -# infrastructure/sources/cert-manager/letsencrypt.yaml +# infrastructure/core/cert-manager/letsencrypt.yaml apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: diff --git a/infrastructure/core/kustomization.yaml b/infrastructure/core/kustomization.yaml index dae83588a..689723572 100644 --- a/infrastructure/core/kustomization.yaml +++ b/infrastructure/core/kustomization.yaml @@ -11,5 +11,5 @@ resources: - coredns-deployment.yaml - ntp-sync-daemonset.yaml - workload-profiles.yaml - - ../sources/cert-manager/letsencrypt.yaml - - ../sources/cert-manager/letsencrypt-prod.yaml + - cert-manager/letsencrypt.yaml + - cert-manager/letsencrypt-prod.yaml diff --git a/infrastructure/flux-system/kustomization.yaml b/infrastructure/flux-system/kustomization.yaml index eae7a6573..36bd82d1b 100644 --- a/infrastructure/flux-system/kustomization.yaml +++ b/infrastructure/flux-system/kustomization.yaml @@ -2,4 +2,4 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ../clusters/atlas/flux-system + - ../../clusters/atlas/flux-system diff --git a/knowledge/INDEX.md b/knowledge/INDEX.md index fac915330..28c9b4ebb 100644 --- a/knowledge/INDEX.md +++ b/knowledge/INDEX.md @@ -12,7 +12,7 @@ Layout Regeneration - Update manifests/docs, then regenerate generated artifacts: - - `python scripts/knowledge_render_atlas.py --write` + - `python scripts/render/knowledge_render_atlas.py --write` Authoring rules - Never include secret values. Prefer `secretRef` names or Vault paths like `kv/atlas/...`. diff --git a/knowledge/catalog/atlas-summary.json b/knowledge/catalog/atlas-summary.json index ea825ce74..f56760663 100644 --- a/knowledge/catalog/atlas-summary.json +++ b/knowledge/catalog/atlas-summary.json @@ -1,8 +1,8 @@ { "counts": { - "helmrelease_host_hints": 19, - "http_endpoints": 45, - "services": 47, - "workloads": 74 + "helmrelease_host_hints": 22, + "http_endpoints": 54, + "services": 70, + "workloads": 100 } } diff --git a/knowledge/catalog/atlas.json b/knowledge/catalog/atlas.json index 951c80796..daae83d58 100644 --- a/knowledge/catalog/atlas.json +++ b/knowledge/catalog/atlas.json @@ -13,7 +13,7 @@ }, { "name": "bstein-dev-home-migrations", - "path": "services/bstein-dev-home/migrations", + "path": "services/bstein-dev-home/migration-jobs", "targetNamespace": "bstein-dev-home" }, { @@ -41,6 +41,11 @@ "path": "services/crypto", "targetNamespace": "crypto" }, + { + "name": "descheduler", + "path": "infrastructure/descheduler", + "targetNamespace": "kube-system" + }, { "name": "finance", "path": "services/finance", @@ -51,6 +56,11 @@ "path": "clusters/atlas/flux-system", "targetNamespace": null }, + { + "name": "game-stream", + "path": "services/game-stream", + "targetNamespace": "game-stream" + }, { "name": "gitea", "path": "services/gitea", @@ -76,6 +86,11 @@ "path": "infrastructure/sources/helm", "targetNamespace": "flux-system" }, + { + "name": "hermes", + "path": "services/hermes", + "targetNamespace": "hermes" + }, { "name": "jellyfin", "path": "services/jellyfin", @@ -151,6 +166,11 @@ "path": "services/oauth2-proxy", "targetNamespace": "sso" }, + { + "name": "openclaw", + "path": "services/openclaw", + "targetNamespace": "openclaw" + }, { "name": "openldap", "path": "services/openldap", @@ -176,6 +196,16 @@ "path": "infrastructure/postgres", "targetNamespace": "postgres" }, + { + "name": "quality", + "path": "services/quality", + "targetNamespace": "quality" + }, + { + "name": "resource-guardrails", + "path": "infrastructure/resource-guardrails", + "targetNamespace": null + }, { "name": "sui-metrics", "path": "services/sui-metrics/overlays/atlas", @@ -186,6 +216,11 @@ "path": "infrastructure/traefik", "targetNamespace": "traefik" }, + { + "name": "typhon", + "path": "services/typhon", + "targetNamespace": "climate" + }, { "name": "vault", "path": "services/vault", @@ -206,6 +241,11 @@ "path": "services/vaultwarden", "targetNamespace": "vaultwarden" }, + { + "name": "veles", + "path": "services/veles", + "targetNamespace": "veles" + }, { "name": "wallet-monero-temp", "path": "services/crypto/wallet-monero-temp", @@ -244,7 +284,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-157" + "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-414" ] }, { @@ -260,7 +300,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-157" + "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-414" ] }, { @@ -292,6 +332,35 @@ "python:3.11-slim" ] }, + { + "kind": "Deployment", + "namespace": "climate", + "name": "typhon", + "labels": { + "app": "typhon" + }, + "serviceAccountName": "typhon", + "nodeSelector": { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "registry.bstein.dev/bstein/typhon:main" + ] + }, + { + "kind": "Deployment", + "namespace": "climate", + "name": "typhon-vault-sync", + "labels": { + "app": "typhon-vault-sync" + }, + "serviceAccountName": "typhon", + "nodeSelector": {}, + "images": [ + "alpine:3.20" + ] + }, { "kind": "Deployment", "namespace": "comms", @@ -301,7 +370,8 @@ }, "serviceAccountName": "atlasbot", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "python:3.11-slim" @@ -329,7 +399,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "ghcr.io/coturn/coturn:4.6.2" @@ -344,7 +415,8 @@ }, "serviceAccountName": null, "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "ghcr.io/element-hq/element-call@sha256:e6897c7818331714eae19d83ef8ea94a8b41115f0d8d3f62c2fed2d02c65c9bc" @@ -359,7 +431,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "livekit/livekit-server:v1.9.0" @@ -374,7 +447,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "registry.bstein.dev/tools/lk-jwt-service-vault:0.3.0" @@ -389,7 +463,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "ghcr.io/element-hq/matrix-authentication-service:1.8.0" @@ -416,7 +491,9 @@ "app": "matrix-wellknown" }, "serviceAccountName": null, - "nodeSelector": {}, + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, "images": [ "nginx:1.27-alpine" ] @@ -426,14 +503,16 @@ "namespace": "crypto", "name": "monero-xmrig", "labels": { - "app": "monero-xmrig" + "app": "monero-xmrig", + "atlas.bstein.dev/workload-profile": "scavenger" }, "serviceAccountName": null, "nodeSelector": { + "atlas.bstein.dev/crypto-mining-enabled": "true", "node-role.kubernetes.io/worker": "true" }, "images": [ - "ghcr.io/tari-project/xmrig@sha256:80defbfd0b640d604c91cb5101d3642db7928e1e68ee3c6b011289b3565a39d9" + "ghcr.io/tari-project/xmrig@sha256:d590a41613fea974f155280920095ea10c3710f55ecf16fc38fd3a1c18718129" ] }, { @@ -454,7 +533,8 @@ "namespace": "crypto", "name": "monero-p2pool", "labels": { - "app": "monero-p2pool" + "app": "monero-p2pool", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": "crypto-vault-sync", "nodeSelector": { @@ -469,13 +549,15 @@ "namespace": "crypto", "name": "monerod", "labels": { - "app": "monerod" + "app": "monerod", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": null, "nodeSelector": { "node-role.kubernetes.io/worker": "true" }, "images": [ + "python:3.11-alpine", "registry.bstein.dev/crypto/monerod:0.18.4.1" ] }, @@ -640,12 +722,60 @@ "ghcr.io/fluxcd/source-controller:v1.7.4" ] }, + { + "kind": "DaemonSet", + "namespace": "game-stream", + "name": "wolf-gatekeeper", + "labels": { + "app": "wolf-gatekeeper" + }, + "serviceAccountName": null, + "nodeSelector": { + "kubernetes.io/hostname": "titan-24" + }, + "images": [ + "ghcr.io/games-on-whales/wolf:stable" + ] + }, + { + "kind": "Deployment", + "namespace": "game-stream", + "name": "oauth2-proxy-wolf", + "labels": { + "app": "oauth2-proxy-wolf" + }, + "serviceAccountName": "game-stream-vault", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "StatefulSet", + "namespace": "game-stream", + "name": "wolf", + "labels": { + "app": "wolf", + "atlas.bstein.dev/game-mode": "wolf" + }, + "serviceAccountName": null, + "nodeSelector": { + "kubernetes.io/hostname": "titan-24" + }, + "images": [ + "ghcr.io/games-on-whales/wolf:stable", + "ghcr.io/games-on-whales/wolfmanager/wolfmanager:latest" + ] + }, { "kind": "Deployment", "namespace": "gitea", "name": "gitea", "labels": { - "app": "gitea" + "app": "gitea", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": "gitea-vault", "nodeSelector": { @@ -685,12 +815,39 @@ "wger/server@sha256:710588b78af4e0aa0b4d8a8061e4563e16eae80eeaccfe7f9e0d9cbdd7f0cbc5" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes", + "labels": { + "app": "hermes" + }, + "serviceAccountName": "hermes-triage", + "nodeSelector": {}, + "images": [ + "nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-ollama", + "labels": { + "app": "hermes-ollama" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d" + ] + }, { "kind": "Deployment", "namespace": "jellyfin", "name": "jellyfin", "labels": { - "app": "jellyfin" + "app": "jellyfin", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": "pegasus-vault-sync", "nodeSelector": {}, @@ -723,7 +880,9 @@ "app": "pegasus-vault-sync" }, "serviceAccountName": "pegasus-vault-sync", - "nodeSelector": {}, + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, "images": [ "alpine:3.20" ] @@ -851,7 +1010,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/infra/coredns:1.12.1" + "registry.k8s.io/coredns/coredns:v1.12.1" ] }, { @@ -893,7 +1052,7 @@ }, "serviceAccountName": "node-log-rotation", "nodeSelector": { - "hardware": "rpi5" + "node-role.kubernetes.io/worker": "true" }, "images": [ "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" @@ -1016,6 +1175,38 @@ "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" ] }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "metis-sentinel-amd64", + "labels": { + "app": "metis-sentinel-amd64" + }, + "serviceAccountName": "metis", + "nodeSelector": { + "kubernetes.io/arch": "amd64", + "kubernetes.io/os": "linux" + }, + "images": [ + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64" + ] + }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "metis-sentinel-arm64", + "labels": { + "app": "metis-sentinel-arm64" + }, + "serviceAccountName": "metis", + "nodeSelector": { + "kubernetes.io/arch": "arm64", + "kubernetes.io/os": "linux" + }, + "images": [ + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64" + ] + }, { "kind": "DaemonSet", "namespace": "maintenance", @@ -1044,6 +1235,51 @@ "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" ] }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "rpi-resource-reservation", + "labels": { + "app": "rpi-resource-reservation" + }, + "serviceAccountName": "node-nofile", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" + ] + }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "titan-22-link-keeper", + "labels": { + "app": "titan-22-link-keeper" + }, + "serviceAccountName": null, + "nodeSelector": { + "kubernetes.io/hostname": "titan-22" + }, + "images": [ + "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" + ] + }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "titan-24-docker", + "labels": { + "app": "titan-24-docker" + }, + "serviceAccountName": "titan-24-docker", + "nodeSelector": { + "kubernetes.io/hostname": "titan-24" + }, + "images": [ + "debian:13-slim" + ] + }, { "kind": "Deployment", "namespace": "maintenance", @@ -1057,7 +1293,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/ariadne:0.1.0-49" + "registry.bstein.dev/bstein/ariadne:0.1.0-356" ] }, { @@ -1068,11 +1304,72 @@ "app": "maintenance-vault-sync" }, "serviceAccountName": "maintenance-vault-sync", - "nodeSelector": {}, + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, "images": [ "alpine:3.20" ] }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "metis", + "labels": { + "app": "metis" + }, + "serviceAccountName": "metis", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/metis:0.1.0-243-arm64" + ] + }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "oauth2-proxy-metis", + "labels": { + "app": "oauth2-proxy-metis" + }, + "serviceAccountName": "maintenance-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "oauth2-proxy-soteria", + "labels": { + "app": "oauth2-proxy-soteria" + }, + "serviceAccountName": "maintenance-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "soteria", + "labels": { + "app": "soteria" + }, + "serviceAccountName": "soteria", + "nodeSelector": { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "registry.bstein.dev/bstein/soteria:0.1.0-120" + ] + }, { "kind": "DaemonSet", "namespace": "monitoring", @@ -1101,6 +1398,19 @@ "python:3.10-slim" ] }, + { + "kind": "DaemonSet", + "namespace": "monitoring", + "name": "nvidia-process-exporter", + "labels": { + "app": "nvidia-process-exporter" + }, + "serviceAccountName": "nvidia-process-exporter", + "nodeSelector": {}, + "images": [ + "python:3.12-slim" + ] + }, { "kind": "Deployment", "namespace": "monitoring", @@ -1114,6 +1424,19 @@ "alpine:3.20" ] }, + { + "kind": "Deployment", + "namespace": "monitoring", + "name": "platform-quality-gateway", + "labels": { + "app": "platform-quality-gateway" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "prom/pushgateway:v1.11.2" + ] + }, { "kind": "Deployment", "namespace": "monitoring", @@ -1127,6 +1450,19 @@ "python:3.12-alpine" ] }, + { + "kind": "Deployment", + "namespace": "monitoring", + "name": "vmalert-atlas-availability", + "labels": { + "app": "vmalert-atlas-availability" + }, + "serviceAccountName": "vmalert-atlas-availability", + "nodeSelector": {}, + "images": [ + "victoriametrics/vmalert:v1.113.0" + ] + }, { "kind": "Deployment", "namespace": "nextcloud", @@ -1136,7 +1472,7 @@ }, "serviceAccountName": null, "nodeSelector": { - "hardware": "rpi5" + "kubernetes.io/arch": "amd64" }, "images": [ "collabora/code@sha256:3c58d0e9bae75e4647467d0c7d91cb66f261d3e814709aed590b5c334a04db26" @@ -1151,7 +1487,8 @@ }, "serviceAccountName": "nextcloud-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "nextcloud:29-apache" @@ -1211,6 +1548,7 @@ }, "serviceAccountName": "postgres-vault", "nodeSelector": { + "hardware": "rpi5", "node-role.kubernetes.io/worker": "true" }, "images": [ @@ -1218,6 +1556,51 @@ "quay.io/prometheuscommunity/postgres-exporter:v0.15.0" ] }, + { + "kind": "Deployment", + "namespace": "quality", + "name": "oauth2-proxy-sonarqube", + "labels": { + "app": "oauth2-proxy-sonarqube" + }, + "serviceAccountName": "quality-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "Deployment", + "namespace": "quality", + "name": "sonarqube", + "labels": { + "app": "sonarqube" + }, + "serviceAccountName": "sonarqube", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "sonarqube:lts-community" + ] + }, + { + "kind": "Deployment", + "namespace": "quality", + "name": "sonarqube-exporter", + "labels": { + "app": "sonarqube-exporter" + }, + "serviceAccountName": "quality-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "registry.bstein.dev/bstein/python:3.12-slim" + ] + }, { "kind": "Deployment", "namespace": "sso", @@ -1287,7 +1670,8 @@ }, "serviceAccountName": "sui-metrics", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "victoriametrics/vmagent:v1.103.0" @@ -1302,7 +1686,7 @@ "app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik" }, - "serviceAccountName": "traefik-ingress-controller", + "serviceAccountName": "atlas-traefik-ingress-controller", "nodeSelector": { "node-role.kubernetes.io/worker": "true" }, @@ -1323,7 +1707,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "hashicorp/vault:1.17.6" + "hashicorp/vault:1.21.4" ] }, { @@ -1335,12 +1719,70 @@ }, "serviceAccountName": "vaultwarden-vault", "nodeSelector": { + "hardware": "rpi5", "kubernetes.io/arch": "arm64", "node-role.kubernetes.io/worker": "true" }, "images": [ "vaultwarden/server:1.35.2" ] + }, + { + "kind": "Deployment", + "namespace": "veles", + "name": "veles-backend", + "labels": { + "app": "veles-backend" + }, + "serviceAccountName": "veles-backend", + "nodeSelector": { + "kubernetes.io/arch": "amd64", + "veles.bstein.dev/node-pool": "oceanus" + }, + "images": [ + "registry.bstein.dev/veles/veles-backend:0.7.56" + ] + }, + { + "kind": "Deployment", + "namespace": "veles", + "name": "veles-frontend", + "labels": { + "app": "veles-frontend" + }, + "serviceAccountName": "veles-frontend", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/veles/veles-frontend:0.7.56" + ] + }, + { + "kind": "Deployment", + "namespace": "veles", + "name": "veles-vault-sync", + "labels": { + "app": "veles-vault-sync" + }, + "serviceAccountName": "veles-vault-sync", + "nodeSelector": {}, + "images": [ + "alpine:3.20" + ] + }, + { + "kind": "StatefulSet", + "namespace": "veles", + "name": "veles-postgres", + "labels": { + "app": "veles-postgres" + }, + "serviceAccountName": "veles-postgres", + "nodeSelector": { + "veles.bstein.dev/node-pool": "oceanus" + }, + "images": [ + "postgres:15" + ] } ], "services": [ @@ -1387,7 +1829,7 @@ { "name": "http", "port": 80, - "targetPort": 80, + "targetPort": 8080, "protocol": "TCP" } ] @@ -1408,6 +1850,44 @@ } ] }, + { + "namespace": "climate", + "name": "typhon", + "type": "ClusterIP", + "selector": { + "app": "typhon" + }, + "ports": [ + { + "name": "metrics", + "port": 9108, + "targetPort": "metrics", + "protocol": "TCP" + }, + { + "name": "control", + "port": 9110, + "targetPort": "control", + "protocol": "TCP" + } + ] + }, + { + "namespace": "comms", + "name": "atlasbot", + "type": "ClusterIP", + "selector": { + "app": "atlasbot" + }, + "ports": [ + { + "name": "http", + "port": 8090, + "targetPort": 8090, + "protocol": "TCP" + } + ] + }, { "namespace": "comms", "name": "coturn", @@ -1876,6 +2356,12 @@ "targetPort": 18081, "protocol": "TCP" }, + { + "name": "status", + "port": 18084, + "targetPort": 18084, + "protocol": "TCP" + }, { "name": "p2p", "port": 18080, @@ -2002,6 +2488,132 @@ } ] }, + { + "namespace": "game-stream", + "name": "oauth2-proxy-wolf", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-wolf" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf", + "type": "ClusterIP", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "placeholder", + "port": 47989, + "targetPort": 47989, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf-api", + "type": "ClusterIP", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "http", + "port": 8088, + "targetPort": 8088, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf-gatekeeper", + "type": "ClusterIP", + "selector": { + "app": "wolf-gatekeeper" + }, + "ports": [ + { + "name": "http", + "port": 8087, + "targetPort": 8087, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf-moonlight", + "type": "LoadBalancer", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "tcp-https", + "port": 47984, + "targetPort": 47984, + "protocol": "TCP" + }, + { + "name": "tcp-http", + "port": 47989, + "targetPort": 47989, + "protocol": "TCP" + }, + { + "name": "tcp-rtsp", + "port": 48010, + "targetPort": 48010, + "protocol": "TCP" + }, + { + "name": "udp-control", + "port": 47999, + "targetPort": 47999, + "protocol": "UDP" + }, + { + "name": "udp-video", + "port": 48100, + "targetPort": 48100, + "protocol": "UDP" + }, + { + "name": "udp-audio", + "port": 48200, + "targetPort": 48200, + "protocol": "UDP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolfmanager", + "type": "ClusterIP", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "http", + "port": 8080, + "targetPort": 3000, + "protocol": "TCP" + } + ] + }, { "namespace": "gitea", "name": "gitea", @@ -2050,6 +2662,44 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes", + "type": "ClusterIP", + "selector": { + "app": "hermes" + }, + "ports": [ + { + "name": "api", + "port": 8642, + "targetPort": "api", + "protocol": "TCP" + }, + { + "name": "dashboard", + "port": 9119, + "targetPort": "dashboard", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-ollama", + "type": "ClusterIP", + "selector": { + "app": "hermes-ollama" + }, + "ports": [ + { + "name": "http", + "port": 11434, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "jellyfin", "name": "jellyfin", @@ -2200,6 +2850,70 @@ } ] }, + { + "namespace": "maintenance", + "name": "metis", + "type": "ClusterIP", + "selector": { + "app": "metis" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, + { + "namespace": "maintenance", + "name": "oauth2-proxy-metis", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-metis" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "maintenance", + "name": "oauth2-proxy-soteria", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-soteria" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "maintenance", + "name": "soteria", + "type": "ClusterIP", + "selector": { + "app": "soteria" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "monitoring", "name": "dcgm-exporter", @@ -2232,6 +2946,38 @@ } ] }, + { + "namespace": "monitoring", + "name": "nvidia-process-exporter", + "type": "ClusterIP", + "selector": { + "app": "nvidia-process-exporter" + }, + "ports": [ + { + "name": "metrics", + "port": 9401, + "targetPort": "metrics", + "protocol": "TCP" + } + ] + }, + { + "namespace": "monitoring", + "name": "platform-quality-gateway", + "type": "ClusterIP", + "selector": { + "app": "platform-quality-gateway" + }, + "ports": [ + { + "name": "http", + "port": 9091, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "monitoring", "name": "postmark-exporter", @@ -2248,6 +2994,22 @@ } ] }, + { + "namespace": "monitoring", + "name": "vmalert-atlas-availability", + "type": "ClusterIP", + "selector": { + "app": "vmalert-atlas-availability" + }, + "ports": [ + { + "name": "http", + "port": 8880, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "nextcloud", "name": "collabora", @@ -2350,6 +3112,54 @@ } ] }, + { + "namespace": "quality", + "name": "oauth2-proxy-sonarqube", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-sonarqube" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "quality", + "name": "sonarqube", + "type": "ClusterIP", + "selector": { + "app": "sonarqube" + }, + "ports": [ + { + "name": "http", + "port": 9000, + "targetPort": 9000, + "protocol": "TCP" + } + ] + }, + { + "namespace": "quality", + "name": "sonarqube-exporter", + "type": "ClusterIP", + "selector": { + "app": "sonarqube-exporter" + }, + "ports": [ + { + "name": "metrics", + "port": 9798, + "targetPort": "metrics", + "protocol": "TCP" + } + ] + }, { "namespace": "sso", "name": "keycloak", @@ -2517,9 +3327,77 @@ "protocol": "TCP" } ] + }, + { + "namespace": "veles", + "name": "veles-backend", + "type": "ClusterIP", + "selector": { + "app": "veles-backend" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 8796, + "protocol": "TCP" + } + ] + }, + { + "namespace": "veles", + "name": "veles-frontend", + "type": "ClusterIP", + "selector": { + "app": "veles-frontend" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 8080, + "protocol": "TCP" + } + ] + }, + { + "namespace": "veles", + "name": "veles-postgres", + "type": "ClusterIP", + "selector": { + "app": "veles-postgres" + }, + "ports": [ + { + "name": "postgres", + "port": 5432, + "targetPort": 5432, + "protocol": "TCP" + } + ] } ], "http_endpoints": [ + { + "host": "agent.bstein.dev", + "path": "/", + "backend": { + "namespace": "hermes", + "service": "hermes", + "port": "dashboard", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "agent", + "source": "hermes" + } + }, { "host": "auth.bstein.dev", "path": "/", @@ -2540,6 +3418,26 @@ "source": "oauth2-proxy" } }, + { + "host": "backup.bstein.dev", + "path": "/", + "backend": { + "namespace": "maintenance", + "service": "oauth2-proxy-soteria", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-soteria" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "soteria", + "source": "maintenance" + } + }, { "host": "bstein.dev", "path": "/", @@ -3280,6 +4178,46 @@ "source": "pegasus" } }, + { + "host": "quality.bstein.dev", + "path": "/", + "backend": { + "namespace": "quality", + "service": "oauth2-proxy-sonarqube", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-sonarqube" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "sonarqube", + "source": "quality" + } + }, + { + "host": "recovery.bstein.dev", + "path": "/", + "backend": { + "namespace": "maintenance", + "service": "oauth2-proxy-metis", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-metis" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "metis", + "source": "maintenance" + } + }, { "host": "scm.bstein.dev", "path": "/", @@ -3399,9 +4337,112 @@ "name": "vaultwarden-ingress", "source": "vaultwarden" } + }, + { + "host": "veles.bstein.dev", + "path": "/", + "backend": { + "namespace": "veles", + "service": "veles-frontend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-frontend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "veles.bstein.dev", + "path": "/api", + "backend": { + "namespace": "veles", + "service": "veles-backend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-backend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "veles.bstein.dev", + "path": "/events", + "backend": { + "namespace": "veles", + "service": "veles-backend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-backend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "veles.bstein.dev", + "path": "/ws", + "backend": { + "namespace": "veles", + "service": "veles-backend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-backend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "wolf.bstein.dev", + "path": "/", + "backend": { + "namespace": "game-stream", + "service": "oauth2-proxy-wolf", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-wolf" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "wolf", + "source": "game-stream" + } } ], "helmrelease_host_hints": { + "cert-manager:cert-manager/cert-manager": [ + "atlas.bstein.dev" + ], "comms:comms/othrys-element": [ "call.live.bstein.dev", "live.bstein.dev", @@ -3423,7 +4464,8 @@ "registry.bstein.dev" ], "longhorn:longhorn-system/longhorn": [ - "registry.bstein.dev" + "registry.bstein.dev", + "veles.bstein.dev" ], "mailu:mailu-mailserver/mailu": [ "bstein.dev", @@ -3433,13 +4475,16 @@ "alerts.bstein.dev" ], "monitoring:monitoring/grafana": [ + "atlas.bstein.dev", "bstein.dev", - "mail.bstein.dev", "metrics.bstein.dev", "sso.bstein.dev" ], "monitoring:monitoring/kube-state-metrics": [ "atlas.bstein.dev" + ], + "vault-injector:vault/vault-injector": [ + "atlas.bstein.dev" ] } } diff --git a/knowledge/catalog/atlas.yaml b/knowledge/catalog/atlas.yaml index 637b5f970..0592c2036 100644 --- a/knowledge/catalog/atlas.yaml +++ b/knowledge/catalog/atlas.yaml @@ -1,5 +1,5 @@ # knowledge/catalog/atlas.yaml -# Generated by scripts/knowledge_render_atlas.py (do not edit by hand) +# Generated by scripts/render/knowledge_render_atlas.py (do not edit by hand) cluster: atlas sources: - name: ai-llm @@ -9,7 +9,7 @@ sources: path: services/bstein-dev-home targetNamespace: bstein-dev-home - name: bstein-dev-home-migrations - path: services/bstein-dev-home/migrations + path: services/bstein-dev-home/migration-jobs targetNamespace: bstein-dev-home - name: cert-manager path: infrastructure/cert-manager @@ -26,12 +26,18 @@ sources: - name: crypto path: services/crypto targetNamespace: crypto +- name: descheduler + path: infrastructure/descheduler + targetNamespace: kube-system - name: finance path: services/finance targetNamespace: finance - name: flux-system path: clusters/atlas/flux-system targetNamespace: null +- name: game-stream + path: services/game-stream + targetNamespace: game-stream - name: gitea path: services/gitea targetNamespace: gitea @@ -47,6 +53,9 @@ sources: - name: helm path: infrastructure/sources/helm targetNamespace: flux-system +- name: hermes + path: services/hermes + targetNamespace: hermes - name: jellyfin path: services/jellyfin targetNamespace: jellyfin @@ -92,6 +101,9 @@ sources: - name: oauth2-proxy path: services/oauth2-proxy targetNamespace: sso +- name: openclaw + path: services/openclaw + targetNamespace: openclaw - name: openldap path: services/openldap targetNamespace: sso @@ -107,12 +119,21 @@ sources: - name: postgres path: infrastructure/postgres targetNamespace: postgres +- name: quality + path: services/quality + targetNamespace: quality +- name: resource-guardrails + path: infrastructure/resource-guardrails + targetNamespace: null - name: sui-metrics path: services/sui-metrics/overlays/atlas targetNamespace: sui-metrics - name: traefik path: infrastructure/traefik targetNamespace: traefik +- name: typhon + path: services/typhon + targetNamespace: climate - name: vault path: services/vault targetNamespace: vault @@ -125,6 +146,9 @@ sources: - name: vaultwarden path: services/vaultwarden targetNamespace: vaultwarden +- name: veles + path: services/veles + targetNamespace: veles - name: wallet-monero-temp path: services/crypto/wallet-monero-temp targetNamespace: crypto @@ -151,7 +175,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-157 + - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-414 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-frontend @@ -162,7 +186,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-157 + - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-414 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-vault-sync @@ -183,6 +207,26 @@ workloads: node-role.kubernetes.io/worker: 'true' images: - python:3.11-slim +- kind: Deployment + namespace: climate + name: typhon + labels: + app: typhon + serviceAccountName: typhon + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: 'true' + images: + - registry.bstein.dev/bstein/typhon:main +- kind: Deployment + namespace: climate + name: typhon-vault-sync + labels: + app: typhon-vault-sync + serviceAccountName: typhon + nodeSelector: {} + images: + - alpine:3.20 - kind: Deployment namespace: comms name: atlasbot @@ -191,6 +235,7 @@ workloads: serviceAccountName: atlasbot nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - python:3.11-slim - kind: Deployment @@ -210,6 +255,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - ghcr.io/coturn/coturn:4.6.2 - kind: Deployment @@ -220,6 +266,7 @@ workloads: serviceAccountName: null nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - ghcr.io/element-hq/element-call@sha256:e6897c7818331714eae19d83ef8ea94a8b41115f0d8d3f62c2fed2d02c65c9bc - kind: Deployment @@ -230,6 +277,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - livekit/livekit-server:v1.9.0 - kind: Deployment @@ -240,6 +288,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - registry.bstein.dev/tools/lk-jwt-service-vault:0.3.0 - kind: Deployment @@ -250,6 +299,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - ghcr.io/element-hq/matrix-authentication-service:1.8.0 - kind: Deployment @@ -267,7 +317,8 @@ workloads: labels: app: matrix-wellknown serviceAccountName: null - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/worker: 'true' images: - nginx:1.27-alpine - kind: DaemonSet @@ -275,11 +326,13 @@ workloads: name: monero-xmrig labels: app: monero-xmrig + atlas.bstein.dev/workload-profile: scavenger serviceAccountName: null nodeSelector: + atlas.bstein.dev/crypto-mining-enabled: 'true' node-role.kubernetes.io/worker: 'true' images: - - ghcr.io/tari-project/xmrig@sha256:80defbfd0b640d604c91cb5101d3642db7928e1e68ee3c6b011289b3565a39d9 + - ghcr.io/tari-project/xmrig@sha256:d590a41613fea974f155280920095ea10c3710f55ecf16fc38fd3a1c18718129 - kind: Deployment namespace: crypto name: crypto-vault-sync @@ -294,6 +347,7 @@ workloads: name: monero-p2pool labels: app: monero-p2pool + atlas.bstein.dev/workload-profile: heavy serviceAccountName: crypto-vault-sync nodeSelector: node-role.kubernetes.io/worker: 'true' @@ -304,10 +358,12 @@ workloads: name: monerod labels: app: monerod + atlas.bstein.dev/workload-profile: heavy serviceAccountName: null nodeSelector: node-role.kubernetes.io/worker: 'true' images: + - python:3.11-alpine - registry.bstein.dev/crypto/monerod:0.18.4.1 - kind: Deployment namespace: crypto @@ -425,11 +481,44 @@ workloads: kubernetes.io/os: linux images: - ghcr.io/fluxcd/source-controller:v1.7.4 +- kind: DaemonSet + namespace: game-stream + name: wolf-gatekeeper + labels: + app: wolf-gatekeeper + serviceAccountName: null + nodeSelector: + kubernetes.io/hostname: titan-24 + images: + - ghcr.io/games-on-whales/wolf:stable +- kind: Deployment + namespace: game-stream + name: oauth2-proxy-wolf + labels: + app: oauth2-proxy-wolf + serviceAccountName: game-stream-vault + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: StatefulSet + namespace: game-stream + name: wolf + labels: + app: wolf + atlas.bstein.dev/game-mode: wolf + serviceAccountName: null + nodeSelector: + kubernetes.io/hostname: titan-24 + images: + - ghcr.io/games-on-whales/wolf:stable + - ghcr.io/games-on-whales/wolfmanager/wolfmanager:latest - kind: Deployment namespace: gitea name: gitea labels: app: gitea + atlas.bstein.dev/workload-profile: heavy serviceAccountName: gitea-vault nodeSelector: node-role.kubernetes.io/worker: 'true' @@ -456,11 +545,30 @@ workloads: images: - nginx:1.27.5-alpine@sha256:65645c7bb6a0661892a8b03b89d0743208a18dd2f3f17a54ef4b76fb8e2f2a10 - wger/server@sha256:710588b78af4e0aa0b4d8a8061e4563e16eae80eeaccfe7f9e0d9cbdd7f0cbc5 +- kind: Deployment + namespace: hermes + name: hermes + labels: + app: hermes + serviceAccountName: hermes-triage + nodeSelector: {} + images: + - nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 +- kind: Deployment + namespace: hermes + name: hermes-ollama + labels: + app: hermes-ollama + serviceAccountName: null + nodeSelector: {} + images: + - ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d - kind: Deployment namespace: jellyfin name: jellyfin labels: app: jellyfin + atlas.bstein.dev/workload-profile: heavy serviceAccountName: pegasus-vault-sync nodeSelector: {} images: @@ -483,7 +591,8 @@ workloads: labels: app: pegasus-vault-sync serviceAccountName: pegasus-vault-sync - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/worker: 'true' images: - alpine:3.20 - kind: Deployment @@ -572,7 +681,7 @@ workloads: nodeSelector: kubernetes.io/os: linux images: - - registry.bstein.dev/infra/coredns:1.12.1 + - registry.k8s.io/coredns/coredns:v1.12.1 - kind: DaemonSet namespace: logging name: node-image-gc-rpi4 @@ -600,7 +709,7 @@ workloads: app: node-log-rotation serviceAccountName: node-log-rotation nodeSelector: - hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 - kind: Deployment @@ -682,6 +791,28 @@ workloads: node-role.kubernetes.io/worker: 'true' images: - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: metis-sentinel-amd64 + labels: + app: metis-sentinel-amd64 + serviceAccountName: metis + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux + images: + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64 +- kind: DaemonSet + namespace: maintenance + name: metis-sentinel-arm64 + labels: + app: metis-sentinel-arm64 + serviceAccountName: metis + nodeSelector: + kubernetes.io/arch: arm64 + kubernetes.io/os: linux + images: + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64 - kind: DaemonSet namespace: maintenance name: node-image-sweeper @@ -701,6 +832,36 @@ workloads: nodeSelector: {} images: - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: rpi-resource-reservation + labels: + app: rpi-resource-reservation + serviceAccountName: node-nofile + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: titan-22-link-keeper + labels: + app: titan-22-link-keeper + serviceAccountName: null + nodeSelector: + kubernetes.io/hostname: titan-22 + images: + - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: titan-24-docker + labels: + app: titan-24-docker + serviceAccountName: titan-24-docker + nodeSelector: + kubernetes.io/hostname: titan-24 + images: + - debian:13-slim - kind: Deployment namespace: maintenance name: ariadne @@ -711,16 +872,57 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/ariadne:0.1.0-49 + - registry.bstein.dev/bstein/ariadne:0.1.0-356 - kind: Deployment namespace: maintenance name: maintenance-vault-sync labels: app: maintenance-vault-sync serviceAccountName: maintenance-vault-sync - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/worker: 'true' images: - alpine:3.20 +- kind: Deployment + namespace: maintenance + name: metis + labels: + app: metis + serviceAccountName: metis + nodeSelector: {} + images: + - registry.bstein.dev/bstein/metis:0.1.0-243-arm64 +- kind: Deployment + namespace: maintenance + name: oauth2-proxy-metis + labels: + app: oauth2-proxy-metis + serviceAccountName: maintenance-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: Deployment + namespace: maintenance + name: oauth2-proxy-soteria + labels: + app: oauth2-proxy-soteria + serviceAccountName: maintenance-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: Deployment + namespace: maintenance + name: soteria + labels: + app: soteria + serviceAccountName: soteria + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: 'true' + images: + - registry.bstein.dev/bstein/soteria:0.1.0-120 - kind: DaemonSet namespace: monitoring name: dcgm-exporter @@ -740,6 +942,15 @@ workloads: jetson: 'true' images: - python:3.10-slim +- kind: DaemonSet + namespace: monitoring + name: nvidia-process-exporter + labels: + app: nvidia-process-exporter + serviceAccountName: nvidia-process-exporter + nodeSelector: {} + images: + - python:3.12-slim - kind: Deployment namespace: monitoring name: monitoring-vault-sync @@ -749,6 +960,15 @@ workloads: nodeSelector: {} images: - alpine:3.20 +- kind: Deployment + namespace: monitoring + name: platform-quality-gateway + labels: + app: platform-quality-gateway + serviceAccountName: null + nodeSelector: {} + images: + - prom/pushgateway:v1.11.2 - kind: Deployment namespace: monitoring name: postmark-exporter @@ -758,6 +978,15 @@ workloads: nodeSelector: {} images: - python:3.12-alpine +- kind: Deployment + namespace: monitoring + name: vmalert-atlas-availability + labels: + app: vmalert-atlas-availability + serviceAccountName: vmalert-atlas-availability + nodeSelector: {} + images: + - victoriametrics/vmalert:v1.113.0 - kind: Deployment namespace: nextcloud name: collabora @@ -765,7 +994,7 @@ workloads: app: collabora serviceAccountName: null nodeSelector: - hardware: rpi5 + kubernetes.io/arch: amd64 images: - collabora/code@sha256:3c58d0e9bae75e4647467d0c7d91cb66f261d3e814709aed590b5c334a04db26 - kind: Deployment @@ -776,6 +1005,7 @@ workloads: serviceAccountName: nextcloud-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - nextcloud:29-apache - kind: Deployment @@ -815,10 +1045,41 @@ workloads: app: postgres serviceAccountName: postgres-vault nodeSelector: + hardware: rpi5 node-role.kubernetes.io/worker: 'true' images: - postgres:15 - quay.io/prometheuscommunity/postgres-exporter:v0.15.0 +- kind: Deployment + namespace: quality + name: oauth2-proxy-sonarqube + labels: + app: oauth2-proxy-sonarqube + serviceAccountName: quality-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: Deployment + namespace: quality + name: sonarqube + labels: + app: sonarqube + serviceAccountName: sonarqube + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - sonarqube:lts-community +- kind: Deployment + namespace: quality + name: sonarqube-exporter + labels: + app: sonarqube-exporter + serviceAccountName: quality-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - registry.bstein.dev/bstein/python:3.12-slim - kind: Deployment namespace: sso name: keycloak @@ -868,6 +1129,7 @@ workloads: serviceAccountName: sui-metrics nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - victoriametrics/vmagent:v1.103.0 - kind: Deployment @@ -877,7 +1139,7 @@ workloads: app: traefik app.kubernetes.io/instance: traefik-kube-system app.kubernetes.io/name: traefik - serviceAccountName: traefik-ingress-controller + serviceAccountName: atlas-traefik-ingress-controller nodeSelector: node-role.kubernetes.io/worker: 'true' images: @@ -892,7 +1154,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - hashicorp/vault:1.17.6 + - hashicorp/vault:1.21.4 - kind: Deployment namespace: vaultwarden name: vaultwarden @@ -900,10 +1162,50 @@ workloads: app: vaultwarden serviceAccountName: vaultwarden-vault nodeSelector: + hardware: rpi5 kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - vaultwarden/server:1.35.2 +- kind: Deployment + namespace: veles + name: veles-backend + labels: + app: veles-backend + serviceAccountName: veles-backend + nodeSelector: + kubernetes.io/arch: amd64 + veles.bstein.dev/node-pool: oceanus + images: + - registry.bstein.dev/veles/veles-backend:0.7.56 +- kind: Deployment + namespace: veles + name: veles-frontend + labels: + app: veles-frontend + serviceAccountName: veles-frontend + nodeSelector: {} + images: + - registry.bstein.dev/veles/veles-frontend:0.7.56 +- kind: Deployment + namespace: veles + name: veles-vault-sync + labels: + app: veles-vault-sync + serviceAccountName: veles-vault-sync + nodeSelector: {} + images: + - alpine:3.20 +- kind: StatefulSet + namespace: veles + name: veles-postgres + labels: + app: veles-postgres + serviceAccountName: veles-postgres + nodeSelector: + veles.bstein.dev/node-pool: oceanus + images: + - postgres:15 services: - namespace: ai name: ollama @@ -933,7 +1235,7 @@ services: ports: - name: http port: 80 - targetPort: 80 + targetPort: 8080 protocol: TCP - namespace: bstein-dev-home name: chat-ai-gateway @@ -945,6 +1247,30 @@ services: port: 80 targetPort: 8080 protocol: TCP +- namespace: climate + name: typhon + type: ClusterIP + selector: + app: typhon + ports: + - name: metrics + port: 9108 + targetPort: metrics + protocol: TCP + - name: control + port: 9110 + targetPort: control + protocol: TCP +- namespace: comms + name: atlasbot + type: ClusterIP + selector: + app: atlasbot + ports: + - name: http + port: 8090 + targetPort: 8090 + protocol: TCP - namespace: comms name: coturn type: LoadBalancer @@ -1253,6 +1579,10 @@ services: port: 18081 targetPort: 18081 protocol: TCP + - name: status + port: 18084 + targetPort: 18084 + protocol: TCP - name: p2p port: 18080 targetPort: 18080 @@ -1331,6 +1661,86 @@ services: port: 80 targetPort: http-webhook protocol: TCP +- namespace: game-stream + name: oauth2-proxy-wolf + type: ClusterIP + selector: + app: oauth2-proxy-wolf + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: game-stream + name: wolf + type: ClusterIP + selector: + app: wolf + ports: + - name: placeholder + port: 47989 + targetPort: 47989 + protocol: TCP +- namespace: game-stream + name: wolf-api + type: ClusterIP + selector: + app: wolf + ports: + - name: http + port: 8088 + targetPort: 8088 + protocol: TCP +- namespace: game-stream + name: wolf-gatekeeper + type: ClusterIP + selector: + app: wolf-gatekeeper + ports: + - name: http + port: 8087 + targetPort: 8087 + protocol: TCP +- namespace: game-stream + name: wolf-moonlight + type: LoadBalancer + selector: + app: wolf + ports: + - name: tcp-https + port: 47984 + targetPort: 47984 + protocol: TCP + - name: tcp-http + port: 47989 + targetPort: 47989 + protocol: TCP + - name: tcp-rtsp + port: 48010 + targetPort: 48010 + protocol: TCP + - name: udp-control + port: 47999 + targetPort: 47999 + protocol: UDP + - name: udp-video + port: 48100 + targetPort: 48100 + protocol: UDP + - name: udp-audio + port: 48200 + targetPort: 48200 + protocol: UDP +- namespace: game-stream + name: wolfmanager + type: ClusterIP + selector: + app: wolf + ports: + - name: http + port: 8080 + targetPort: 3000 + protocol: TCP - namespace: gitea name: gitea type: ClusterIP @@ -1361,6 +1771,30 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: hermes + name: hermes + type: ClusterIP + selector: + app: hermes + ports: + - name: api + port: 8642 + targetPort: api + protocol: TCP + - name: dashboard + port: 9119 + targetPort: dashboard + protocol: TCP +- namespace: hermes + name: hermes-ollama + type: ClusterIP + selector: + app: hermes-ollama + ports: + - name: http + port: 11434 + targetPort: http + protocol: TCP - namespace: jellyfin name: jellyfin type: ClusterIP @@ -1457,6 +1891,46 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: maintenance + name: metis + type: ClusterIP + selector: + app: metis + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP +- namespace: maintenance + name: oauth2-proxy-metis + type: ClusterIP + selector: + app: oauth2-proxy-metis + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: maintenance + name: oauth2-proxy-soteria + type: ClusterIP + selector: + app: oauth2-proxy-soteria + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: maintenance + name: soteria + type: ClusterIP + selector: + app: soteria + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP - namespace: monitoring name: dcgm-exporter type: ClusterIP @@ -1477,6 +1951,26 @@ services: port: 9100 targetPort: metrics protocol: TCP +- namespace: monitoring + name: nvidia-process-exporter + type: ClusterIP + selector: + app: nvidia-process-exporter + ports: + - name: metrics + port: 9401 + targetPort: metrics + protocol: TCP +- namespace: monitoring + name: platform-quality-gateway + type: ClusterIP + selector: + app: platform-quality-gateway + ports: + - name: http + port: 9091 + targetPort: http + protocol: TCP - namespace: monitoring name: postmark-exporter type: ClusterIP @@ -1487,6 +1981,16 @@ services: port: 8000 targetPort: http protocol: TCP +- namespace: monitoring + name: vmalert-atlas-availability + type: ClusterIP + selector: + app: vmalert-atlas-availability + ports: + - name: http + port: 8880 + targetPort: http + protocol: TCP - namespace: nextcloud name: collabora type: ClusterIP @@ -1551,6 +2055,36 @@ services: port: 9187 targetPort: 9187 protocol: TCP +- namespace: quality + name: oauth2-proxy-sonarqube + type: ClusterIP + selector: + app: oauth2-proxy-sonarqube + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: quality + name: sonarqube + type: ClusterIP + selector: + app: sonarqube + ports: + - name: http + port: 9000 + targetPort: 9000 + protocol: TCP +- namespace: quality + name: sonarqube-exporter + type: ClusterIP + selector: + app: sonarqube-exporter + ports: + - name: metrics + port: 9798 + targetPort: metrics + protocol: TCP - namespace: sso name: keycloak type: ClusterIP @@ -1657,7 +2191,50 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: veles + name: veles-backend + type: ClusterIP + selector: + app: veles-backend + ports: + - name: http + port: 80 + targetPort: 8796 + protocol: TCP +- namespace: veles + name: veles-frontend + type: ClusterIP + selector: + app: veles-frontend + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP +- namespace: veles + name: veles-postgres + type: ClusterIP + selector: + app: veles-postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 + protocol: TCP http_endpoints: +- host: agent.bstein.dev + path: / + backend: + namespace: hermes + service: hermes + port: dashboard + workloads: + - kind: Deployment + name: hermes + via: + kind: Ingress + name: agent + source: hermes - host: auth.bstein.dev path: / backend: @@ -1671,6 +2248,19 @@ http_endpoints: kind: Ingress name: oauth2-proxy source: oauth2-proxy +- host: backup.bstein.dev + path: / + backend: + namespace: maintenance + service: oauth2-proxy-soteria + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-soteria + via: + kind: Ingress + name: soteria + source: maintenance - host: bstein.dev path: / backend: @@ -2129,6 +2719,32 @@ http_endpoints: kind: Ingress name: pegasus source: pegasus +- host: quality.bstein.dev + path: / + backend: + namespace: quality + service: oauth2-proxy-sonarqube + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-sonarqube + via: + kind: Ingress + name: sonarqube + source: quality +- host: recovery.bstein.dev + path: / + backend: + namespace: maintenance + service: oauth2-proxy-metis + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-metis + via: + kind: Ingress + name: metis + source: maintenance - host: scm.bstein.dev path: / backend: @@ -2207,7 +2823,70 @@ http_endpoints: kind: Ingress name: vaultwarden-ingress source: vaultwarden +- host: veles.bstein.dev + path: / + backend: + namespace: veles + service: veles-frontend + port: 80 + workloads: + - kind: Deployment + name: veles-frontend + via: + kind: Ingress + name: veles + source: veles +- host: veles.bstein.dev + path: /api + backend: + namespace: veles + service: veles-backend + port: 80 + workloads: &id004 + - kind: Deployment + name: veles-backend + via: + kind: Ingress + name: veles + source: veles +- host: veles.bstein.dev + path: /events + backend: + namespace: veles + service: veles-backend + port: 80 + workloads: *id004 + via: + kind: Ingress + name: veles + source: veles +- host: veles.bstein.dev + path: /ws + backend: + namespace: veles + service: veles-backend + port: 80 + workloads: *id004 + via: + kind: Ingress + name: veles + source: veles +- host: wolf.bstein.dev + path: / + backend: + namespace: game-stream + service: oauth2-proxy-wolf + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-wolf + via: + kind: Ingress + name: wolf + source: game-stream helmrelease_host_hints: + cert-manager:cert-manager/cert-manager: + - atlas.bstein.dev comms:comms/othrys-element: - call.live.bstein.dev - live.bstein.dev @@ -2225,15 +2904,18 @@ helmrelease_host_hints: - registry.bstein.dev longhorn:longhorn-system/longhorn: - registry.bstein.dev + - veles.bstein.dev mailu:mailu-mailserver/mailu: - bstein.dev - mail.bstein.dev monitoring:monitoring/alertmanager: - alerts.bstein.dev monitoring:monitoring/grafana: + - atlas.bstein.dev - bstein.dev - - mail.bstein.dev - metrics.bstein.dev - sso.bstein.dev monitoring:monitoring/kube-state-metrics: - atlas.bstein.dev + vault-injector:vault/vault-injector: + - atlas.bstein.dev diff --git a/knowledge/catalog/metrics.json b/knowledge/catalog/metrics.json index e929db582..adb4eea38 100644 --- a/knowledge/catalog/metrics.json +++ b/knowledge/catalog/metrics.json @@ -1,26 +1,181 @@ [ { - "dashboard": "Atlas GPU", - "panel_title": "Namespace GPU Share", + "dashboard": "Atlas GitOps", + "panel_title": "Flux Source", "panel_id": 1, - "panel_type": "piechart", - "description": "Shares are normalized within the selected filter. Switching scope changes the denominator.", + "panel_type": "stat", + "description": "Branch and revision currently reported by Flux's GitRepository source.", "tags": [ "atlas", - "gpu" + "gitops", + "flux" ], "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(100 * (sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) / clamp_min((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)) == 0))" + "max by (branch, revision) (ananke_gitops_flux_source_info{job=\"ananke-power\",namespace=\"flux-system\",name=\"flux-system\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Kustomizations Ready", + "panel_id": 2, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})), 1)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Kustomizations Suspended", + "panel_id": 3, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(max by (namespace, name) (ananke_gitops_kustomization_suspended{job=\"ananke-power\"})) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "HelmReleases Ready", + "panel_id": 4, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})), 1)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "HelmReleases Suspended", + "panel_id": 5, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(max by (namespace, name) (ananke_gitops_helmrelease_suspended{job=\"ananke-power\"})) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "GitOps Exporter", + "panel_id": 6, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "min(ananke_gitops_scrape_success{job=\"ananke-power\"}) or on() vector(0)", + "(time() - max(ananke_gitops_last_scrape_timestamp_seconds{job=\"ananke-power\"})) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Readiness History", + "panel_id": 7, + "panel_type": "state-timeline", + "description": "Ready percentage over time for Flux Kustomizations and HelmReleases.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(100 * sum(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})), 1), \"kind\", \"Kustomizations\", \"__name__\", \".*\") or label_replace(100 * sum(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})), 1), \"kind\", \"HelmReleases\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Flux Sources", + "panel_id": 8, + "panel_type": "table", + "description": "A Value of 1 means Ready; 0 means not Ready.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (namespace, name, url, branch, revision, ready, reason) (ananke_gitops_flux_source_info{job=\"ananke-power\"}) * on(namespace, name) group_left() max by (namespace, name) (ananke_gitops_flux_source_ready{job=\"ananke-power\"})" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Kustomizations", + "panel_id": 9, + "panel_type": "table", + "description": "A Value of 1 means Ready; 0 means not Ready. The ready/reason labels come from Flux status.conditions.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (namespace, name, path, source_namespace, source_name, revision, ready, reason) (ananke_gitops_kustomization_info{job=\"ananke-power\"}) * on(namespace, name) group_left() max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "HelmReleases", + "panel_id": 10, + "panel_type": "table", + "description": "A Value of 1 means Ready; 0 means not Ready. Chart/version/app_version are included when Flux reports them.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (namespace, name, chart, version, app_version, revision, ready, reason) (ananke_gitops_helmrelease_info{job=\"ananke-power\"}) * on(namespace, name) group_left() max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})" ] }, { "dashboard": "Atlas GPU", - "panel_title": "GPU Util by Namespace", - "panel_id": 2, - "panel_type": "timeseries", - "description": "", + "panel_title": "Namespace GPU Utilization", + "panel_id": 1, + "panel_type": "piechart", + "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.", "tags": [ "atlas", "gpu" @@ -28,7 +183,23 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)" + "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))" + ] + }, + { + "dashboard": "Atlas GPU", + "panel_title": "GPU Process Util by Namespace", + "panel_id": 2, + "panel_type": "timeseries", + "description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods.", + "tags": [ + "atlas", + "gpu" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)" ] }, { @@ -44,15 +215,15 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum by (Hostname) (DCGM_FI_DEV_GPU_UTIL{pod!=\"\"})" + "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")" ] }, { "dashboard": "Atlas GPU", - "panel_title": "Top Pods by GPU Util", + "panel_title": "GPU Pods Reporting Device Util", "panel_id": 4, "panel_type": "table", - "description": "", + "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.", "tags": [ "atlas", "gpu" @@ -63,313 +234,6 @@ "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))" ] }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (range)", - "panel_id": 1, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (task) (increase(ariadne_task_runs_total{status=\"error\"}[$__range])))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Attempts / Failures", - "panel_id": 2, - "panel_type": "timeseries", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total[$__interval]))", - "sum(increase(ariadne_task_runs_total{status=\"error\"}[$__interval]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "One-off Job Pods (age hours)", - "panel_id": 3, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(((time() - kube_pod_start_time{pod!=\"\"}) / 3600) * on(namespace,pod) group_left(owner_name) (kube_pod_owner{owner_kind=\"Job\"} unless on(namespace, owner_name) label_replace(kube_job_owner{owner_kind=\"CronJob\"}, \"owner_name\", \"$1\", \"job_name\", \"(.*)\")) * on(namespace,pod) group_left(phase) max by (namespace,pod,phase) (kube_pod_status_phase{phase=~\"Running|Succeeded\"}))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Stale (>36h)", - "panel_id": 4, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "(sum((((time() - (kube_cronjob_status_last_successful_time and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"})) > bool 129600) unless on(namespace,cronjob) (kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1)) + count(((kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"} unless on(namespace,cronjob) kube_cronjob_status_last_successful_time) unless on(namespace,cronjob) (kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1))) or on() vector(0)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Missing Success", - "panel_id": 5, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "count(((kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"} unless on(namespace,cronjob) kube_cronjob_status_last_successful_time) unless on(namespace,cronjob) (kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1)) or on() vector(0)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Suspended", - "panel_id": 6, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum((kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1) or on() vector(0)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (1h)", - "panel_id": 7, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total{status=\"error\"}[1h]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (24h)", - "panel_id": 8, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total{status=\"error\"}[24h]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Runs (1h)", - "panel_id": 9, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total[1h]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Schedule Last Error (hours ago)", - "panel_id": 10, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time(ariadne_schedule_last_error_timestamp_seconds[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Schedule Last Success (hours ago)", - "panel_id": 11, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time(ariadne_schedule_last_success_timestamp_seconds[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Last Success (hours ago)", - "panel_id": 12, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time((kube_cronjob_status_last_successful_time and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"})[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Last Schedule (hours ago)", - "panel_id": 13, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time((kube_cronjob_status_last_schedule_time and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"})[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (1h)", - "panel_id": 14, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (task) (increase(ariadne_task_runs_total{status=\"error\"}[1h])))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (30d)", - "panel_id": 15, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (task) (increase(ariadne_task_runs_total{status=\"error\"}[30d])))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Access Requests", - "panel_id": 16, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(ariadne_access_requests_total)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne CI Coverage (%)", - "panel_id": 17, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "ariadne_ci_coverage_percent{repo=\"ariadne\"}" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne CI Tests (latest)", - "panel_id": 18, - "panel_type": "table", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "ariadne_ci_tests_total{repo=\"ariadne\"}" - ] - }, { "dashboard": "Atlas Mail", "panel_title": "Sent (1d)", @@ -688,7 +552,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -768,7 +632,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"})" + "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})" ] }, { @@ -864,7 +728,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -880,7 +744,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -896,7 +760,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -912,7 +776,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -928,7 +792,23 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + ] + }, + { + "dashboard": "Atlas Nodes", + "panel_title": "Astraios Usage", + "panel_id": 9, + "panel_type": "timeseries", + "description": "", + "tags": [ + "atlas", + "nodes" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -936,7 +816,7 @@ "panel_title": "Control Plane Ready", "panel_id": 2, "panel_type": "gauge", - "description": "", + "description": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "tags": [ "atlas", "overview" @@ -952,7 +832,7 @@ "panel_title": "Control Plane Workloads", "panel_id": 3, "panel_type": "stat", - "description": "", + "description": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "tags": [ "atlas", "overview" @@ -968,7 +848,7 @@ "panel_title": "Stuck Terminating", "panel_id": 5, "panel_type": "stat", - "description": "", + "description": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", "tags": [ "atlas", "overview" @@ -981,10 +861,10 @@ }, { "dashboard": "Atlas Overview", - "panel_title": "Atlas Availability", + "panel_title": "Atlas Availability (365d)", "panel_id": 27, "panel_type": "stat", - "description": "", + "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.", "tags": [ "atlas", "overview" @@ -992,7 +872,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) / 3)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) / clamp_min(sum(kube_deployment_spec_replicas{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}), 1)))))[365d:1h])" + "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\"}[24h])" ] }, { @@ -1000,7 +880,7 @@ "panel_title": "Problem Pods", "panel_id": 4, "panel_type": "stat", - "description": "", + "description": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.", "tags": [ "atlas", "overview" @@ -1016,7 +896,7 @@ "panel_title": "CrashLoop / ImagePull", "panel_id": 6, "panel_type": "stat", - "description": "", + "description": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.", "tags": [ "atlas", "overview" @@ -1032,7 +912,7 @@ "panel_title": "Workers Ready", "panel_id": 1, "panel_type": "gauge", - "description": "", + "description": "Worker nodes currently Ready; full count is good, lower means less place to run services.", "tags": [ "atlas", "overview" @@ -1040,7 +920,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"})" + "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})" ] }, { @@ -1048,7 +928,7 @@ "panel_title": "Hottest node: CPU", "panel_id": 7, "panel_type": "stat", - "description": "", + "description": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.", "tags": [ "atlas", "overview" @@ -1056,7 +936,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" ] }, { @@ -1064,7 +944,7 @@ "panel_title": "Hottest node: RAM", "panel_id": 8, "panel_type": "stat", - "description": "", + "description": "Highest worker memory use right now; lower is safer, high values risk evictions.", "tags": [ "atlas", "overview" @@ -1072,7 +952,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" ] }, { @@ -1080,7 +960,7 @@ "panel_title": "Hottest node: NET (rx+tx)", "panel_id": 9, "panel_type": "stat", - "description": "", + "description": "Busiest node network rate; spikes can reveal traffic concentration or noisy services.", "tags": [ "atlas", "overview" @@ -1088,7 +968,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" ] }, { @@ -1096,7 +976,7 @@ "panel_title": "Hottest node: I/O (r+w)", "panel_id": 10, "panel_type": "stat", - "description": "", + "description": "Busiest node disk I/O rate; high values can explain slow storage-backed apps.", "tags": [ "atlas", "overview" @@ -1104,7 +984,401 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Astreae Usage", + "panel_id": 23, + "panel_type": "stat", + "description": "Percent of Astreae used; lower is safer, high values reduce storage headroom.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) * 100)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Asteria Usage", + "panel_id": 24, + "panel_type": "stat", + "description": "Percent of Asteria used; lower is safer, high values reduce storage headroom.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) * 100)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Astreae Free", + "panel_id": 25, + "panel_type": "stat", + "description": "Free space on Astreae; higher is better for backups and workload growth.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Asteria Free", + "panel_id": 26, + "panel_type": "stat", + "description": "Free space on Asteria; higher is better for backups and workload growth.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Pyrphoros UPS Current", + "panel_id": 40, + "panel_type": "stat", + "description": "Live Pyrphoros UPS draw and runtime; stable runtime means the lab can ride out short outages.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100) or on() vector(0), \"metric\", \"Draw\", \"__name__\", \".*\") or label_replace(max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Pyrphoros\"}) or on() vector(0), \"metric\", \"Runtime\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Statera UPS Current", + "panel_id": 144, + "panel_type": "stat", + "description": "Live Statera UPS draw and runtime; stable runtime means the lab can ride out short outages.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100) or on() vector(0), \"metric\", \"Draw\", \"__name__\", \".*\") or label_replace(max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Statera\"}) or on() vector(0), \"metric\", \"Runtime\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "UPS History (Power Draw)", + "panel_id": 41, + "panel_type": "timeseries", + "description": "UPS power draw over time; steady draw is normal, spikes show sudden load changes.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100)", + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Current Enclosure Temperature", + "panel_id": 42, + "panel_type": "stat", + "description": "Current tent temperature in C and F; moderate values protect hardware and plants.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)) or on() vector(0), \"metric\", \"\u00b0C\", \"__name__\", \".*\") or label_replace(max((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)) * 9 / 5 + 32) or on() vector(0), \"metric\", \"\u00b0F\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Current Enclosure Climate", + "panel_id": 143, + "panel_type": "stat", + "description": "Current humidity and VPD; in-range values mean the enclosure climate is stable.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)) or on() vector(0), \"metric\", \"%RH\", \"__name__\", \".*\") or label_replace(max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)) or on() vector(0), \"metric\", \"kPa\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Enclosure Climate History", + "panel_id": 43, + "panel_type": "timeseries", + "description": "Temperature on left axis, humidity and pressure on right axis with dynamic bound series so small swings remain visible.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)", + "(min_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)[$__range]) - 0.08)", + "(max_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)[$__range]) + 0.08)", + "clamp_min((min_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)[$__range]) - 0.35), 0)", + "clamp_max((max_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)[$__range]) + 0.35), 100)", + "clamp_min((min_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)[$__range]) - 0.03), 0)", + "(max_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)[$__range]) + 0.03)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Fan Intensity History", + "panel_id": 141, + "panel_type": "state-timeline", + "description": "Fan intensity lanes on the 0-10 controller scale. Cooler colors are quiet/low intensity; warmer colors mean the enclosure is pushing harder.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"1\"}), \"fan\", \"Outlet\", \"__name__\", \".*\") or label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"2\"}), \"fan\", \"Inlet - Inside\", \"__name__\", \".*\") or label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"3\"}), \"fan\", \"Inlet - Outside\", \"__name__\", \".*\") or label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"4\"}), \"fan\", \"Tent Interior\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Flux Source", + "panel_id": 140, + "panel_type": "stat", + "description": "Flux GitRepository branch reported by Ananke. Revision and object detail live in Atlas GitOps.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (branch, revision) (ananke_gitops_flux_source_info{job=\"ananke-power\",namespace=\"flux-system\",name=\"flux-system\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Current Gate Health", + "panel_id": 151, + "panel_type": "stat", + "description": "Current gate-check health across suites; skipped or not-applicable checks count as healthy, failures lower it.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((min by (suite) (((100 * sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch=~\"main|master|origin/main|origin/master\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch=~\"main|master|origin/main|origin/master\",check=\"supply_chain\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1) unless on(suite, check) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch=~\"main|master|origin/main|origin/master\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch=~\"main|master|origin/main|origin/master\",check=\"supply_chain\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1))) / clamp_min(sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch=~\"main|master|origin/main|origin/master\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!=\"\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch=~\"main|master|origin/main|origin/master\",check=\"supply_chain\",status!=\"\"})))) > 0), 1)), 1))) or (min by (suite) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"}))))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "CI Run Success (24h)", + "panel_id": 152, + "panel_type": "stat", + "description": "Percent of published quality-gate CI runs that completed successfully in 24h; this is automation health, not raw test pass rate.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * ((sum(platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|pegasus-health|pegasus_health|soteria|titan_iac|titan-iac|bstein_home|bstein-home|data_prepper|data-prepper|lesavka\",status=~\"ok|passed|success\"}) or on() vector(0))) / clamp_min(((sum(platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|pegasus-health|pegasus_health|soteria|titan_iac|titan-iac|bstein_home|bstein-home|data_prepper|data-prepper|lesavka\"}) or on() vector(0))), 1)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Failed Runs (24h)", + "panel_id": 153, + "panel_type": "stat", + "description": "Published quality-gate runs that failed in 24h; zero is good, any value needs a look.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|pegasus-health|pegasus_health|soteria|titan_iac|titan-iac|bstein_home|bstein-home|data_prepper|data-prepper|lesavka\",status!~\"ok|passed|success\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Suites With Runs (24h)", + "panel_id": 154, + "panel_type": "stat", + "description": "Configured suites with at least one published quality-gate run in 24h; full count means the dashboard is fresh.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\"}) > bool 0)) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Avg Coverage", + "panel_id": 155, + "panel_type": "stat", + "description": "Average latest line coverage across suites; higher means code is better protected by tests.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\"}))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "GitOps Health", + "panel_id": 150, + "panel_type": "state-timeline", + "description": "GitOps readiness and suspension health over time. Blue means perfect; warmer colors mean a readiness or suspension problem appeared.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(100 * sum(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})), 1), \"signal\", \"Kustomizations Ready\", \"__name__\", \".*\") or label_replace(100 * sum(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})), 1), \"signal\", \"HelmReleases Ready\", \"__name__\", \".*\") or label_replace(100 * (1 - (sum(max by (namespace, name) (ananke_gitops_kustomization_suspended{job=\"ananke-power\"})) or on() vector(0)) / clamp_min((count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) or on() vector(0)), 1)), \"signal\", \"Kustomizations Not Suspended\", \"__name__\", \".*\") or label_replace(100 * (1 - (sum(max by (namespace, name) (ananke_gitops_helmrelease_suspended{job=\"ananke-power\"})) or on() vector(0)) / clamp_min((count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) or on() vector(0)), 1)), \"signal\", \"HelmReleases Not Suspended\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "One-off Job Pods (age hours)", + "panel_id": 44, + "panel_type": "bargauge", + "description": "Temporary job pods by age; low or empty is good, old pods usually need cleanup.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(((time() - kube_pod_start_time{pod!=\"\"}) / 3600) * on(namespace,pod) group_left(owner_name) (kube_pod_owner{owner_kind=\"Job\"} unless on(namespace, owner_name) label_replace(kube_job_owner{owner_kind=\"CronJob\"}, \"owner_name\", \"$1\", \"job_name\", \"(.*)\")) * on(namespace,pod) group_left(phase) max by (namespace,pod,phase) (kube_pod_status_phase{phase=~\"Running|Succeeded\"}))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Ariadne Run Volume", + "panel_id": 45, + "panel_type": "timeseries", + "description": "Ariadne automation attempts and failures; attempts show activity, failures show work to investigate.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(ariadne_task_runs_total[5m])) or on() vector(0)", + "sum(increase(ariadne_task_runs_total{status=\"error\"}[5m])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Test Category Health", + "panel_id": 46, + "panel_type": "state-timeline", + "description": "Health by major test category across all suites over the last 24 hours. Skipped tests are healthy; failures and errors lower the lane.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|performance|regression|reliability|security|smoke|system|ui\"})" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Jenkins Last Success (h, newest first)", + "panel_id": 142, + "panel_type": "stat", + "description": "Top 6 most recent Jenkins successes by age (newest first). Green means last run succeeded; red means last run did not succeed. Use Atlas Jobs for the full list.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Jenkins Last Failure (h, newest first)", + "panel_id": 243, + "panel_type": "stat", + "description": "Top 6 most recent Jenkins failures by age (newest first). Green means last run succeeded; red means last run did not succeed. Use Atlas Jobs for the full list.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "PVC Backup Health / Age", + "panel_id": 47, + "panel_type": "bargauge", + "description": "Backup age in hours computed from last-success timestamps for restic-managed PVCs (nightly target: <=20h green, <40h yellow, <50h orange, >=50h red). PVCs that have backup history but currently no successful backup (missing/no_completed/error) are pinned to 999h for visibility.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(max by (namespace, pvc) (((time() - pvc_backup_last_success_timestamp_seconds{driver=\"restic\"}) / 3600) or on(namespace,pvc,volume,driver) ((((pvc_backup_health_reason{driver=\"restic\",reason=~\"missing|no_completed|lookup_failed|unknown_timestamp\"} > 0) * (pvc_backup_count{driver=\"restic\"} > bool 0)) * 999))) or on() ((label_replace(label_replace(vector(999), \"namespace\", \"maintenance\", \"__name__\", \".*\"), \"pvc\", \"backup-telemetry-missing\", \"__name__\", \".*\")) unless on() ((count({__name__=~\"pvc_backup_(count|last_success_timestamp_seconds|health_reason)\",driver=\"restic\"})) > 0)))" ] }, { @@ -1112,7 +1386,7 @@ "panel_title": "Mail Sent (1d)", "panel_id": 30, "panel_type": "stat", - "description": "", + "description": "Outbound mail sent in the last day; useful context for mail health and bounce rates.", "tags": [ "atlas", "overview" @@ -1128,7 +1402,7 @@ "panel_title": "Mail Bounces (1d)", "panel_id": 31, "panel_type": "stat", - "description": "", + "description": "Outbound mail bounce rate and count; zero is best, high values risk delivery reputation.", "tags": [ "atlas", "overview" @@ -1145,7 +1419,7 @@ "panel_title": "Mail Success Rate (1d)", "panel_id": 32, "panel_type": "stat", - "description": "", + "description": "Outbound mail success rate; higher is better for user notifications.", "tags": [ "atlas", "overview" @@ -1161,7 +1435,7 @@ "panel_title": "Mail Limit Used (30d)", "panel_id": 33, "panel_type": "stat", - "description": "", + "description": "Postmark monthly send limit used; lower leaves more quota headroom.", "tags": [ "atlas", "overview" @@ -1177,7 +1451,7 @@ "panel_title": "Postgres Connections Used", "panel_id": 34, "panel_type": "stat", - "description": "", + "description": "Current Postgres connections; lower leaves room for apps during spikes.", "tags": [ "atlas", "overview" @@ -1193,7 +1467,7 @@ "panel_title": "Postgres Hottest Connections", "panel_id": 35, "panel_type": "stat", - "description": "", + "description": "Database with the most active connections; high values identify the pressure source.", "tags": [ "atlas", "overview" @@ -1204,135 +1478,6 @@ "topk(1, sum by (datname) (pg_stat_activity_count))" ] }, - { - "dashboard": "Atlas Overview", - "panel_title": "Astreae Usage", - "panel_id": 23, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) * 100)" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Asteria Usage", - "panel_id": 24, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) * 100)" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Astreae Free", - "panel_id": 25, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Asteria Free", - "panel_id": 26, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "One-off Job Pods (age hours)", - "panel_id": 40, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(((time() - kube_pod_start_time{pod!=\"\"}) / 3600) * on(namespace,pod) group_left(owner_name) (kube_pod_owner{owner_kind=\"Job\"} unless on(namespace, owner_name) label_replace(kube_job_owner{owner_kind=\"CronJob\"}, \"owner_name\", \"$1\", \"job_name\", \"(.*)\")) * on(namespace,pod) group_left(phase) max by (namespace,pod,phase) (kube_pod_status_phase{phase=~\"Running|Succeeded\"}))" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Ariadne Attempts / Failures", - "panel_id": 41, - "panel_type": "timeseries", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total[$__interval]))", - "sum(increase(ariadne_task_runs_total{status=\"error\"}[$__interval]))" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Ariadne Test Success Rate", - "panel_id": 42, - "panel_type": "timeseries", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "100 * sum(max_over_time(ariadne_ci_tests_total{repo=\"ariadne\",result=\"passed\"}[1h])) / clamp_min(sum(max_over_time(ariadne_ci_tests_total{repo=\"ariadne\",result=~\"passed|failed|error\"}[1h])), 1)" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Tests with Failures (24h)", - "panel_id": 43, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (result) (max_over_time(ariadne_ci_tests_total{repo=\"ariadne\",result=~\"failed|error\"}[24h])))" - ] - }, { "dashboard": "Atlas Overview", "panel_title": "Namespace CPU Share", @@ -1351,10 +1496,10 @@ }, { "dashboard": "Atlas Overview", - "panel_title": "Namespace GPU Share", + "panel_title": "Namespace GPU Utilization", "panel_id": 12, "panel_type": "piechart", - "description": "Shares are normalized within the selected filter. Switching scope changes the denominator.", + "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.", "tags": [ "atlas", "overview" @@ -1362,7 +1507,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(100 * (sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) / clamp_min((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)) == 0))" + "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))" ] }, { @@ -1386,7 +1531,7 @@ "panel_title": "Worker Node CPU", "panel_id": 14, "panel_type": "timeseries", - "description": "", + "description": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.", "tags": [ "atlas", "overview" @@ -1394,7 +1539,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1402,7 +1547,7 @@ "panel_title": "Worker Node RAM", "panel_id": 15, "panel_type": "timeseries", - "description": "", + "description": "Worker memory over time; lower is safer, sustained high use risks evictions.", "tags": [ "atlas", "overview" @@ -1410,7 +1555,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1418,7 +1563,7 @@ "panel_title": "Control plane CPU", "panel_id": 16, "panel_type": "timeseries", - "description": "", + "description": "Control-plane CPU over time; low steady usage means Kubernetes has control headroom.", "tags": [ "atlas", "overview" @@ -1426,7 +1571,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1434,7 +1579,7 @@ "panel_title": "Control plane RAM", "panel_id": 17, "panel_type": "timeseries", - "description": "", + "description": "Control-plane memory over time; low steady usage means Kubernetes has control headroom.", "tags": [ "atlas", "overview" @@ -1442,7 +1587,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1450,7 +1595,7 @@ "panel_title": "Node Pod Share", "panel_id": 28, "panel_type": "piechart", - "description": "", + "description": "Share of pods per node; uneven share can reveal overloaded workers.", "tags": [ "atlas", "overview" @@ -1466,7 +1611,7 @@ "panel_title": "Top Nodes by Pod Count", "panel_id": 29, "panel_type": "bargauge", - "description": "", + "description": "Nodes with the most pods; lower and balanced is easier to operate.", "tags": [ "atlas", "overview" @@ -1482,7 +1627,7 @@ "panel_title": "Cluster Ingress Throughput", "panel_id": 18, "panel_type": "timeseries", - "description": "", + "description": "Traffic entering the cluster; spikes should line up with expected usage.", "tags": [ "atlas", "overview" @@ -1498,7 +1643,7 @@ "panel_title": "Cluster Egress Throughput", "panel_id": 19, "panel_type": "timeseries", - "description": "", + "description": "Traffic leaving the cluster; spikes should line up with expected usage.", "tags": [ "atlas", "overview" @@ -1514,7 +1659,7 @@ "panel_title": "Intra-Cluster Throughput", "panel_id": 20, "panel_type": "timeseries", - "description": "", + "description": "Traffic inside the cluster; high values can expose chatty services.", "tags": [ "atlas", "overview" @@ -1530,7 +1675,7 @@ "panel_title": "Root Filesystem Usage", "panel_id": 21, "panel_type": "timeseries", - "description": "", + "description": "Node root disk usage; lower is safer, high values can break kubelet.", "tags": [ "atlas", "overview" @@ -1538,15 +1683,15 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { "dashboard": "Atlas Overview", - "panel_title": "Nodes Closest to Full Root Disks", + "panel_title": "Nodes Closest to Full Astraios Disks", "panel_id": 22, - "panel_type": "bargauge", - "description": "", + "panel_type": "timeseries", + "description": "Astraios disk fullness by node; lower is safer for storage reliability.", "tags": [ "atlas", "overview" @@ -1554,7 +1699,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sort_desc(topk(12, avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))))" + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -1714,7 +1859,127 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.025)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.025)))))" + "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "UPS Current Load", + "panel_id": 1, + "panel_type": "stat", + "description": "Per-UPS live snapshot: current draw in watts, estimated battery runtime if discharge started now, and charging/discharging status.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100) or on() vector(0)", + "max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Pyrphoros\"}) or on() vector(0)", + "max(ananke_ups_on_battery{job=\"ananke-power\",source=\"Pyrphoros\"}) or on() vector(0)", + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100) or on() vector(0)", + "max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Statera\"}) or on() vector(0)", + "max(ananke_ups_on_battery{job=\"ananke-power\",source=\"Statera\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "UPS History (Power Draw)", + "panel_id": 2, + "panel_type": "timeseries", + "description": "Historical UPS power consumption in watts for titan-db and tethys.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100)", + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Current Climate", + "panel_id": 3, + "panel_type": "stat", + "description": "Current tent temperature, humidity, VPD, and dew point. These render once Typhon climate telemetry is online.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)) or on() vector(0)", + "max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)) or on() vector(0)", + "max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)) or on() vector(0)", + "max((243.12 * (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))))) / (17.62 - (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)))))) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Climate History", + "panel_id": 4, + "panel_type": "timeseries", + "description": "Two-axis chart: tent temperature/humidity/dew point (left axis) and tent VPD in kPa (right axis).", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)", + "(243.12 * (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))))) / (17.62 - (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)))))" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Fan Activity", + "panel_id": 5, + "panel_type": "stat", + "description": "Current fan activity levels (0-10): inside outlet, inside inlet, outside inlet, and interior fans.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "round(max(typhon_fan_speed_level{fan_group=\"outlet\"}) or on() vector(0))", + "round(max(typhon_fan_speed_level{fan_group=\"inside_inlet\"}) or on() vector(0))", + "round(max(typhon_fan_speed_level{fan_group=\"outside_inlet\"}) or on() vector(0))", + "round(max(typhon_fan_speed_level{fan_group=\"interior\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Fan Intensity History", + "panel_id": 6, + "panel_type": "timeseries", + "description": "Historical fan activity for all four fan groups (0-10 scale).", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"outlet\"})", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"inside_inlet\"})", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"outside_inlet\"})", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"interior\"})" ] }, { @@ -1794,7 +2059,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[24]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1810,7 +2075,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[24]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1876,5 +2141,872 @@ "exprs": [ "time() - max by (cronjob) (kube_cronjob_status_last_successful_time{namespace=\"maintenance\",cronjob=\"image-sweeper\"})" ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Current Gate Health (%)", + "panel_id": 156, + "panel_type": "stat", + "description": "Average latest required gate checks passing across selected suites; this is the current quality state.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((min by (suite) (((100 * (sum by (suite) (((clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1)) unless on(suite, check) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1))))) / clamp_min((sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!=\"\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!=\"\"})))) > 0), 1))), 1))) or (min by (suite) (platform_quality:test_category_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"}))))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success Rate (24h)", + "panel_id": 2, + "panel_type": "stat", + "description": "Percent of selected quality-gate CI runs that completed successfully in 24h; this is run health, not individual test pass rate.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * ((sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status=~\"ok|passed|success\"}) or on() vector(0))) / clamp_min(((sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"}) or on() vector(0))), 1)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success Rate (7d)", + "panel_id": 3, + "panel_type": "stat", + "description": "Percent of selected quality-gate CI runs that completed successfully in 7d; higher means more stable automation.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * ((sum(increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}))[7d:1h])) or on() vector(0))) / clamp_min(((sum(increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))[7d:1h])) or on() vector(0))), 1)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Failed Runs (24h)", + "panel_id": 4, + "panel_type": "stat", + "description": "Selected quality-gate runs that failed in 24h; zero is good and anything else needs a look.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status!~\"ok|passed|success\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Runs (24h)", + "panel_id": 5, + "panel_type": "stat", + "description": "Selected quality-gate CI run count in 24h; zero means the dashboard may be stale.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Suite Freshness (24h)", + "panel_id": 157, + "panel_type": "stat", + "description": "Percent of selected suites with at least one quality-gate CI run in 24h; 100% means inputs are fresh.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * (sum((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"})) > bool 0) or on() vector(0)) / clamp_min(count(((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0))), 1)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Avg Coverage (%)", + "panel_id": 6, + "panel_type": "stat", + "description": "Average latest line coverage for selected suites; higher means better test protection.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"}))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Suites with LOC >500", + "panel_id": 7, + "panel_type": "stat", + "description": "Selected suites with oversized source files; zero is good for maintainability.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(((max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) > bool 0)) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Latest Gate Health by Suite", + "panel_id": 8, + "panel_type": "bargauge", + "description": "Current health by suite from required gate checks, capped by category-level test health. Skipped and not-applicable results are healthy; failures and errors lower the value.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort(((min by (suite) (((100 * (sum by (suite) (((clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1)) unless on(suite, check) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1))))) / clamp_min((sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!=\"\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!=\"\"})))) > 0), 1))), 1))) or (min by (suite) (platform_quality:test_category_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"})))) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success by Suite (24h)", + "panel_id": 9, + "panel_type": "bargauge", + "description": "24h CI run success rate. This is whether automation finished cleanly, so it can stay low after failed or aborted runs even when tests and latest gate checks are green.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort(((100 * (sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status=~\"ok|passed|success\"})) / clamp_min((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"})), 1)) and on(suite) ((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"})) > 0)) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage by Suite (Latest, gate 95)", + "panel_id": 17, + "panel_type": "bargauge", + "description": "Latest suite coverage; 95%+ is acceptable and 100% is strongest.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Files <=500 LOC by Suite (Latest)", + "panel_id": 18, + "panel_type": "bargauge", + "description": "Percent of managed LOC-gated files at or under 500 lines. Older suite payloads fall back to 100%/0% until they emit platform_quality_gate_source_files_total.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort(((100 * clamp_min((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) - (max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 0) / (max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"}))) and on(suite) ((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) > 0)) or on(suite) (100 * (1 - clamp_max((max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 1))) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success by Suite (7d rolling)", + "panel_id": 11, + "panel_type": "state-timeline", + "description": "Seven-day rolling CI run success rate per suite. Each suite gets its own lane, so failed or aborted runs lower the lane color without implying raw test failures.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(100 * sum by (suite) (increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}))[7d:1h])) / (sum by (suite) (increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))[7d:1h])))) and on(suite) ((sum by (suite) (increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))[7d:1h]))) > 0)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Test Category Health History", + "panel_id": 153, + "panel_type": "state-timeline", + "description": "Health by test category from memoized hourly rollups. Use the Suite filter to focus one project; skipped tests are healthy, while failures and errors lower the lane.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"})" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Daily Run Volume (Selected Scope)", + "panel_id": 12, + "panel_type": "timeseries", + "description": "Twenty-four-hour rolling quality-gate run counts for the selected suite/branch scope. This is volume, not a pass-rate percentage.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status=~\"ok|passed|success\"}) or on() vector(0)", + "sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status!~\"ok|passed|success\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage History by Suite", + "panel_id": 13, + "panel_type": "state-timeline", + "description": "Latest reported line coverage per suite over time. Coverage is separate from LOC compliance so one signal cannot hide the other.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Files <=500 LOC History by Suite", + "panel_id": 14, + "panel_type": "state-timeline", + "description": "Percent of LOC-gated source files at or under the 500-line limit. This uses the existing file-count telemetry; longest-file history needs a new publisher metric.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(100 * clamp_min((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) - (max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 0) / (max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"}))) and on(suite) ((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) > 0) or on(suite) (100 * (1 - clamp_max((max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Tests Failure Rate", + "panel_id": 130, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage Failure Rate", + "panel_id": 131, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "LOC Failure Rate", + "panel_id": 132, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Style Failure Rate", + "panel_id": 133, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Gate Glue Failure Rate", + "panel_id": 134, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "SonarQube Failure Rate", + "panel_id": 135, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Supply Chain Failure Rate", + "panel_id": 136, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Tests Healthy Rate", + "panel_id": 138, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage Healthy Rate", + "panel_id": 139, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "LOC Healthy Rate", + "panel_id": 140, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Style Healthy Rate", + "panel_id": 141, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Gate Glue Healthy Rate", + "panel_id": 142, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "SonarQube Healthy Rate", + "panel_id": 143, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Supply Chain Healthy Rate", + "panel_id": 144, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Problematic Tests Over Time (Top failures)", + "panel_id": 145, + "panel_type": "state-timeline", + "description": "Current outlier tests by rolling 24h failure count. A test needs at least two recent failures to appear, then falls off once it quiets down.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum by (suite, test) (sum_over_time(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",status=\"failed\"}[24h:1h]))) and on (suite, test) topk(12, (sum by (suite, test) (sum_over_time(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",status=\"failed\"}[24h:1h] @ end()))) >= 2)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Most Problematic Test by Suite (7d)", + "panel_id": 147, + "panel_type": "bargauge", + "description": "Worst test per suite summed across 7d. This catches repeat offenders while keeping dashboard loads bounded; current hourly top list is quiet.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(topk by (suite) (1, (sum by (suite, test) (sum_over_time(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",status=\"failed\"}[7d:1h])))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Selected Test Pass/Fail History", + "panel_id": 146, + "panel_type": "timeseries", + "description": "Stacked hourly outcome volume for the selected suite/branch/test scope. This uses vmalert rollups only, avoiding expensive raw long-range per-test scans.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\",status=\"passed\"}) or on() vector(0))", + "(sum(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\",status=\"failed\"}) or on() vector(0))", + "(sum(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\",status=\"skipped\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Selected Test Pass Rate History", + "panel_id": 152, + "panel_type": "state-timeline", + "description": "Average pass rate per suite for the selected test filter, using memoized hourly test-case pass-rate rollups instead of raw historical scans.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (suite) (platform_quality:test_case_pass_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\"})" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Tests Metrics Present by Suite", + "panel_id": 27, + "panel_type": "bargauge", + "description": "Whether suite-level test counts are present; 100% means the suite is reporting.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) ({__name__=~\".*_quality_gate_tests_total\",exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Checks Metrics Present by Suite", + "panel_id": 28, + "panel_type": "bargauge", + "description": "Whether gate check metrics are present; 100% means health panels have inputs.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) ({__name__=~\".*_quality_gate_checks_total\",exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage Metrics Present by Suite", + "panel_id": 29, + "panel_type": "bargauge", + "description": "Whether coverage metrics are present; 100% means coverage panels are reliable.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_workspace_line_coverage_percent{exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "LOC Compliance Metrics Present by Suite", + "panel_id": 30, + "panel_type": "bargauge", + "description": "Whether LOC metrics are present; 100% means size panels are reliable.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_source_lines_over_500_total{exported_job=\"platform-quality-ci\"}) and on(suite) count by (suite) (platform_quality_gate_source_files_total{exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Test-Case Metrics Present by Suite", + "panel_id": 148, + "panel_type": "bargauge", + "description": "Whether per-test metrics are present; 100% enables drilldowns.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_test_case_result{exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Real Test Cases Present by Suite", + "panel_id": 151, + "panel_type": "bargauge", + "description": "Whether real test names are present; 100% means not just placeholder telemetry.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_test_case_result{exported_job=\"platform-quality-ci\",test!=\"__no_test_cases__\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Primary Branch Clean by Suite (7d)", + "panel_id": 150, + "panel_type": "bargauge", + "description": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((count by (suite) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}[7d:1h]))) > bool 0) unless on(suite) ((count by (suite) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\",branch!~\"main|master|origin/main|origin/master|unknown\"}[7d:1h]))) > bool 0))) or on(suite) (0 * ((count by (suite) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}[7d:1h]))) > bool 0)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Recent Branch Evidence by Suite (7d)", + "panel_id": 149, + "panel_type": "bargauge", + "description": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(count by (suite, branch) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}[7d:1h])))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "SonarQube API Up", + "panel_id": 31, + "panel_type": "stat", + "description": "Whether the SonarQube exporter can reach SonarQube; 1 is good.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(max(sonarqube_up) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Projects (Selected)", + "panel_id": 32, + "panel_type": "stat", + "description": "Selected SonarQube project count; zero means Sonar is not tracking that suite.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(count(max by (project_key) (sonarqube_project_quality_gate_pass{project_key=~\"${suite:regex}\"})) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Gate Fetch Errors", + "panel_id": 33, + "panel_type": "stat", + "description": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(max(sonarqube_quality_gate_fetch_errors_total) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Gate Status Mix (Selected)", + "panel_id": 34, + "panel_type": "piechart", + "description": "Mix of Sonar gate states; OK is good and non-OK needs cleanup.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "count by (status) (max by (project_key, status) (sonarqube_project_quality_gate_pass{project_key=~\"${suite:regex}\"}))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Gate Health by Project", + "panel_id": 35, + "panel_type": "state-timeline", + "description": "SonarQube gate status over time by project. OK projects render as full healthy lanes; non-OK projects drop to red without disappearing.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "platform_quality:sonar_gate_health_percent:latest_1h{project_key=~\"${suite:regex}\"}" + ] } ] diff --git a/knowledge/catalog/runbooks.json b/knowledge/catalog/runbooks.json index 960510d21..3a34ac883 100644 --- a/knowledge/catalog/runbooks.json +++ b/knowledge/catalog/runbooks.json @@ -15,10 +15,18 @@ "source_paths": [ "services/gitea", "services/jenkins", - "scripts/jenkins_cred_sync.sh", - "scripts/gitea_cred_sync.sh" + "scripts/sync/jenkins_cred_sync.sh", + "scripts/sync/gitea_cred_sync.sh" ], - "body": "# CI: Gitea \u2192 Jenkins pipeline\n\n## What this is\nAtlas uses Gitea for source control and Jenkins for CI. Authentication is via Keycloak (SSO).\n\n## Where it is configured\n- Gitea manifests: `services/gitea/`\n- Jenkins manifests: `services/jenkins/`\n- Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh`\n\n## What users do (typical flow)\n- Create a repo in Gitea.\n- Create/update a Jenkins job/pipeline that can fetch the repo.\n- Configure a webhook (or SCM polling) so pushes trigger builds.\n\n## Troubleshooting (common)\n- \u201cWebhook not firing\u201d: confirm ingress host, webhook URL, and Jenkins job is reachable.\n- \u201cAuth denied cloning\u201d: confirm Keycloak group membership and that Jenkins has a valid token/credential configured." + "body": "# CI: Gitea \u2192 Jenkins pipeline\n\n## What this is\nAtlas uses Gitea for source control and Jenkins for CI. Authentication is via Keycloak (SSO).\n\n## Where it is configured\n- Gitea manifests: `services/gitea/`\n- Jenkins manifests: `services/jenkins/`\n- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh`\n\n## What users do (typical flow)\n- Create a repo in Gitea.\n- Create/update a Jenkins job/pipeline that can fetch the repo.\n- Configure a webhook (or SCM polling) so pushes trigger builds.\n\n## Troubleshooting (common)\n- \u201cWebhook not firing\u201d: confirm ingress host, webhook URL, and Jenkins job is reachable.\n- \u201cAuth denied cloning\u201d: confirm Keycloak group membership and that Jenkins has a valid token/credential configured." + }, + { + "path": "runbooks/cluster-power-recovery.md", + "title": "cluster-power-recovery", + "tags": [], + "entrypoints": [], + "source_paths": [], + "body": "Atlas Cluster Power Recovery (Graceful Shutdown/Startup)\n\nPurpose\n- Provide a safe operator flow for planned power events and cold-boot recovery.\n- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path.\n- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle.\n- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery.\n\nBootstrapping risk to remember\n- Flux source is Git over SSH to `scm.bstein.dev` (Gitea).\n- Gitea itself is a Flux-managed workload and depends on storage + database.\n- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images.\n- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall.\n- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.\n\nScript\n- `scripts/ops/cluster_power_recovery.sh`\n- `scripts/ops/cluster_power_console.sh`\n- Modes:\n - `prepare`\n - `shutdown`\n - `harbor-seed`\n - `startup`\n - `status`\n- Default is dry-run. Add `--execute` to actually perform actions.\n\nDry-run examples\n- Shutdown preview:\n - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`\n- Startup preview:\n - `scripts/ops/cluster_power_recovery.sh startup`\n- Harbor seed preview:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed`\n\nExecute examples\n- Prepare helper image on every node:\n - `scripts/ops/cluster_power_recovery.sh prepare --execute`\n- Seed Harbor runtime images onto `titan-05` from the control-host bundle:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`\n- Planned shutdown:\n - `scripts/ops/cluster_power_recovery.sh shutdown --execute`\n- Planned startup (canonical branch):\n - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`\n\nManual remote console examples\n- Canonical operator hosts:\n - `titan-db`\n - `tethys` (`titan-24`)\n- Both hosts now have:\n - `~/ananke-tools/cluster_power_recovery.sh`\n - `~/ananke-tools/cluster_power_console.sh`\n - `~/ananke-tools/bootstrap/recovery-config.env`\n - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt`\n - `~/ananke-tools/kubeconfig`\n - `~/ananke-cluster-power`\n - `~/bin/ananke-cluster-power`\n - `~/ananke-repo/{infrastructure,services,scripts}`\n- Both hosts also keep the Harbor bootstrap bundle at:\n - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n- Remote usage:\n - `ssh titan-db`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n - `ssh tethys`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n\nUseful options\n- `--shutdown-mode host-poweroff|cluster-only`\n- `--expected-flux-branch main`\n- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-branch main`\n- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only)\n- `--skip-local-bootstrap` (not recommended for cold-start recovery)\n- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred)\n- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node)\n- `--skip-helper-prewarm`\n- `--min-startup-battery 35`\n- `--ups-host pyrphoros@localhost`\n- `--require-ups-battery`\n- `--drain-timeout 180`\n- `--emergency-drain-timeout 45`\n- `--flux-ready-timeout 1200`\n- `--startup-checklist-timeout 900`\n- `--startup-stability-window 180`\n- `--startup-stability-timeout 900`\n- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state`\n- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n\nControlled drill checklist (recommended)\n- Operator host: use `titan-db` as canonical control host for the drill.\n- On-site coordination:\n - Have on-site operator ready before shutdown starts.\n - Confirm they will manually power cluster nodes back on after shutdown completes.\n - Confirm who will announce \"all nodes powered on\" to resume startup.\n- Preflight on `titan-db`:\n - `mkdir -p ~/ananke-logs`\n - `~/ananke-cluster-power status` and verify:\n - `ups_host=pyrphoros@localhost`\n - `ups_battery` is numeric\n - `flux_source_ready=True`\n- Warm helper image just before shutdown:\n - `~/ananke-cluster-power prepare --execute`\n- Run in a persistent shell and capture logs:\n - `tmux new -s ananke-drill`\n - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log`\n- Execute controlled shutdown with telemetry enforcement:\n - `~/ananke-cluster-power shutdown --execute --require-ups-battery`\n- After on-site power-on confirmation, execute startup:\n - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery`\n- Post-check:\n - `~/ananke-cluster-power status`\n - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures.\n\nOperational notes\n- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn.\n- Shutdown behavior is explicit:\n - `host-poweroff` schedules host poweroff after service stop.\n - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off.\n- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted.\n- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`).\n- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo.\n- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`.\n- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.\n- Harbor is reconciled after the first critical stateful services.\n- Harbor bootstrap is now designed around a control-host bundle:\n - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.\n - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.\n - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.\n- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.\n- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls.\n- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap.\n- Startup completion is strict now:\n - all non-optional Flux kustomizations must be `Ready=True`\n - external service checklist must pass (defaults include Gitea, Grafana, Harbor)\n - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`)\n - stability soak must pass with no crashloop/pull-failure churn\n- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile.\n- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster.\n- Dry-run mode no longer mutates outage recovery state.\n- `harbor-seed --execute` was validated by:\n - prewarming the helper image across all nodes\n - streaming the Harbor bootstrap bundle to `titan-05`\n - importing Harbor runtime images into host `containerd`\n - successfully running a Harbor-backed canary pod (`harbor-canary-ok`)\n- After bootstrap, Flux resources are resumed and reconciled.\n- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`." }, { "path": "runbooks/comms-verify.md", @@ -47,7 +55,7 @@ "entrypoints": [], "source_paths": [ "knowledge/runbooks", - "scripts/knowledge_render_atlas.py" + "scripts/render/knowledge_render_atlas.py" ], "body": "# KB authoring: what to write (and what not to)\n\n## The goal\nGive Atlas assistants enough grounded, Atlas-specific context to answer \u201chow do I\u2026?\u201d questions without guessing.\n\n## What to capture (high value)\n- User workflows: \u201cclick here, set X, expected result\u201d\n- Operator workflows: \u201cedit these files, reconcile this kustomization, verify with these commands\u201d\n- Wiring: \u201cthis host routes to this service; this service depends on Postgres/Vault/etc\u201d\n- Failure modes: exact error messages + the 2\u20135 checks that usually resolve them\n- Permissions: Keycloak groups/roles and what they unlock\n\n## What to avoid (low value / fluff)\n- Generic Kubernetes explanations (link to upstream docs instead)\n- Copy-pasting large manifests (prefer file paths + small snippets)\n- Anything that will drift quickly (render it from GitOps instead)\n- Any secret values (reference Secret/Vault locations by name only)\n\n## Document pattern (recommended)\nEach runbook should answer:\n- \u201cWhat is this?\u201d\n- \u201cWhat do users do?\u201d\n- \u201cWhat do operators change (where in Git)?\u201d\n- \u201cHow do we verify it works?\u201d\n- \u201cWhat breaks and how to debug it?\u201d" }, @@ -92,6 +100,6 @@ "tags": [], "entrypoints": [], "source_paths": [], - "body": "# Metis (node recovery)\n\n## Node classes (current map)\n- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)\n- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)\n- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)\n- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)\n- amd64 agents: titan-22/24 (Debian 13, k3s agent)\n- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers.\n\n### Jetson nodes (titan-20/21)\n- Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.\n- Storage: NVMe 232G at / (ext4); onboard mmc partitions present but root on NVMe; 1.9T sda present (unused).\n- k3s agent with drop-in 99-nofile.conf.\n\n## Longhorn disk UUIDs (critical nodes)\n- titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)\n- titan-15: /mnt/astreae UUID=f3362f14-5822-449f-944b-ac570b5cd615 (ext4); /mnt/asteria UUID=9c5316e6-f847-4884-b502-11f2d0d15d6f (ext4)\n- titan-17: /mnt/astreae UUID=1fecdade-08b0-49cb-9ae3-be6c188b0a96 (ext4); /mnt/asteria UUID=2fe9f613-d372-47ca-b84f-82084e4edda0 (ext4)\n- titan-19: /mnt/astreae UUID=4890abb9-dda2-4f4f-9c0f-081ee82849cf (ext4); /mnt/asteria UUID=2b4ea28d-b0e6-4fa3-841b-cd7067ae9153 (ext4)\n\n## Metis repo (~/Development/metis)\n- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).\n- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).\n- `AGENTS.md` in repo is untracked and holds raw notes.\n\n## Next implementation steps\n- Add per-class golden image refs and checksums (Harbor or file://) when ready.\n- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.\n- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.\n- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.\n\n## Node OS/Kernel/CRI snapshot (Jan 2026)\n- titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-05: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-06: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-07: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-08: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-09: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0a: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0b: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0c: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-10: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-11: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-12: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-13: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-14: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-15: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-17: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-18: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-19: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-20: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-21: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-22: Debian 13 (trixie), kernel 6.12.41+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n- titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n\n\n### External hosts\n- titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.\n- titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).\n- titan-23/oceanus: TODO audit (future).\n\n\n### Control plane Pis (titan-0a/0b/0c)\n- Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, containerd 2.0.5-k3s2.\n- Storage: 477G SSD root (/dev/sda2 ext4), /boot/firmware vfat (/dev/sda1). fstab uses LABEL=writable and LABEL=system-boot.\n- k3s server (control-plane taint expected); etcd snapshots not yet cataloged (TODO).\n\n\n## k3s versions\n- rpi5 workers/control-plane: k3s v1.33.3+k3s1 (crictl v1.31.0-k3s2)\n- rpi4 nodes: k3s v1.31.5+k3s1 (crictl v1.31.0-k3s2)\n- Jetson titan-20/21: k3s v1.33.3+k3s1 (per node info), crictl v1.31.0-k3s2" + "body": "# Metis (node recovery)\n\n## Node classes (current map)\n- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent)\n- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)\n- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)\n- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)\n- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)\n- amd64 agents: titan-22/24 (Debian 13, k3s agent)\n- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels)\n- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.\n\n### Jetson nodes (titan-20/21)\n- Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.\n- Storage: NVMe 232G at / (ext4); onboard mmc partitions present but root on NVMe; 1.9T sda present (unused).\n- k3s agent with drop-in 99-nofile.conf.\n\n## Longhorn disk UUIDs (critical nodes)\n- titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)\n- titan-15: /mnt/astreae UUID=f3362f14-5822-449f-944b-ac570b5cd615 (ext4); /mnt/asteria UUID=9c5316e6-f847-4884-b502-11f2d0d15d6f (ext4)\n- titan-17: /mnt/astreae UUID=1fecdade-08b0-49cb-9ae3-be6c188b0a96 (ext4); /mnt/asteria UUID=2fe9f613-d372-47ca-b84f-82084e4edda0 (ext4)\n- titan-19: /mnt/astreae UUID=4890abb9-dda2-4f4f-9c0f-081ee82849cf (ext4); /mnt/asteria UUID=2b4ea28d-b0e6-4fa3-841b-cd7067ae9153 (ext4)\n\n## Metis repo (~/Development/metis)\n- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).\n- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).\n\n## Next implementation steps\n- Add per-class golden image refs and checksums (Harbor or file://) when ready.\n- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.\n- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.\n- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited.\n\n## Node OS/Kernel/CRI snapshot (Jan 2026)\n- titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-05: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-06: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-07: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-08: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-09: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0a: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0b: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0c: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-10: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-11: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-12: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-13: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-14: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-15: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-17: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-18: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-19: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-20: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-21: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-22: Debian 13 (trixie), kernel 6.12.41+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n- titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n\n\n### Dedicated and special-purpose hosts\n- titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.\n- titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).\n- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement.\n\n\n### Control plane Pis (titan-0a/0b/0c)\n- Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, containerd 2.0.5-k3s2.\n- Storage: 477G SSD root (/dev/sda2 ext4), /boot/firmware vfat (/dev/sda1). fstab uses LABEL=writable and LABEL=system-boot.\n- k3s server (control-plane taint expected); etcd snapshots not yet cataloged (TODO).\n\n\n## k3s versions\n- rpi5 workers/control-plane: k3s v1.33.3+k3s1 (crictl v1.31.0-k3s2)\n- rpi4 nodes: k3s v1.31.5+k3s1 (crictl v1.31.0-k3s2)\n- Jetson titan-20/21: k3s v1.33.3+k3s1 (per node info), crictl v1.31.0-k3s2" } ] diff --git a/knowledge/diagrams/atlas-http.mmd b/knowledge/diagrams/atlas-http.mmd index 1aa7ac804..4c4c704cc 100644 --- a/knowledge/diagrams/atlas-http.mmd +++ b/knowledge/diagrams/atlas-http.mmd @@ -1,9 +1,19 @@ flowchart LR + host_agent_bstein_dev["agent.bstein.dev"] + svc_hermes_hermes["hermes/hermes (Service)"] + host_agent_bstein_dev --> svc_hermes_hermes + wl_hermes_hermes["hermes/hermes (Deployment)"] + svc_hermes_hermes --> wl_hermes_hermes host_auth_bstein_dev["auth.bstein.dev"] svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"] host_auth_bstein_dev --> svc_sso_oauth2_proxy wl_sso_oauth2_proxy["sso/oauth2-proxy (Deployment)"] svc_sso_oauth2_proxy --> wl_sso_oauth2_proxy + host_backup_bstein_dev["backup.bstein.dev"] + svc_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Service)"] + host_backup_bstein_dev --> svc_maintenance_oauth2_proxy_soteria + wl_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Deployment)"] + svc_maintenance_oauth2_proxy_soteria --> wl_maintenance_oauth2_proxy_soteria host_bstein_dev["bstein.dev"] svc_bstein_dev_home_bstein_dev_home_frontend["bstein-dev-home/bstein-dev-home-frontend (Service)"] host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_frontend @@ -111,6 +121,16 @@ flowchart LR host_pegasus_bstein_dev --> svc_jellyfin_pegasus wl_jellyfin_pegasus["jellyfin/pegasus (Deployment)"] svc_jellyfin_pegasus --> wl_jellyfin_pegasus + host_quality_bstein_dev["quality.bstein.dev"] + svc_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Service)"] + host_quality_bstein_dev --> svc_quality_oauth2_proxy_sonarqube + wl_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Deployment)"] + svc_quality_oauth2_proxy_sonarqube --> wl_quality_oauth2_proxy_sonarqube + host_recovery_bstein_dev["recovery.bstein.dev"] + svc_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Service)"] + host_recovery_bstein_dev --> svc_maintenance_oauth2_proxy_metis + wl_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Deployment)"] + svc_maintenance_oauth2_proxy_metis --> wl_maintenance_oauth2_proxy_metis host_scm_bstein_dev["scm.bstein.dev"] svc_gitea_gitea["gitea/gitea (Service)"] host_scm_bstein_dev --> svc_gitea_gitea @@ -141,6 +161,20 @@ flowchart LR host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service wl_vaultwarden_vaultwarden["vaultwarden/vaultwarden (Deployment)"] svc_vaultwarden_vaultwarden_service --> wl_vaultwarden_vaultwarden + host_veles_bstein_dev["veles.bstein.dev"] + svc_veles_veles_frontend["veles/veles-frontend (Service)"] + host_veles_bstein_dev --> svc_veles_veles_frontend + wl_veles_veles_frontend["veles/veles-frontend (Deployment)"] + svc_veles_veles_frontend --> wl_veles_veles_frontend + svc_veles_veles_backend["veles/veles-backend (Service)"] + host_veles_bstein_dev --> svc_veles_veles_backend + wl_veles_veles_backend["veles/veles-backend (Deployment)"] + svc_veles_veles_backend --> wl_veles_veles_backend + host_wolf_bstein_dev["wolf.bstein.dev"] + svc_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Service)"] + host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf + wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"] + svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf subgraph bstein_dev_home[bstein-dev-home] svc_bstein_dev_home_bstein_dev_home_frontend @@ -175,6 +209,10 @@ flowchart LR svc_finance_firefly wl_finance_firefly end + subgraph game_stream[game-stream] + svc_game_stream_oauth2_proxy_wolf + wl_game_stream_oauth2_proxy_wolf + end subgraph gitea[gitea] svc_gitea_gitea wl_gitea_gitea @@ -183,6 +221,10 @@ flowchart LR svc_health_wger wl_health_wger end + subgraph hermes[hermes] + svc_hermes_hermes + wl_hermes_hermes + end subgraph jellyfin[jellyfin] svc_jellyfin_pegasus wl_jellyfin_pegasus @@ -204,6 +246,12 @@ flowchart LR subgraph mailu_mailserver[mailu-mailserver] svc_mailu_mailserver_mailu_front end + subgraph maintenance[maintenance] + svc_maintenance_oauth2_proxy_soteria + wl_maintenance_oauth2_proxy_soteria + svc_maintenance_oauth2_proxy_metis + wl_maintenance_oauth2_proxy_metis + end subgraph nextcloud[nextcloud] svc_nextcloud_nextcloud wl_nextcloud_nextcloud @@ -218,6 +266,10 @@ flowchart LR svc_planka_planka wl_planka_planka end + subgraph quality[quality] + svc_quality_oauth2_proxy_sonarqube + wl_quality_oauth2_proxy_sonarqube + end subgraph sso[sso] svc_sso_oauth2_proxy wl_sso_oauth2_proxy @@ -232,3 +284,9 @@ flowchart LR svc_vaultwarden_vaultwarden_service wl_vaultwarden_vaultwarden end + subgraph veles[veles] + svc_veles_veles_frontend + wl_veles_veles_frontend + svc_veles_veles_backend + wl_veles_veles_backend + end diff --git a/knowledge/hermes-automated-triage-24h-plan.md b/knowledge/hermes-automated-triage-24h-plan.md new file mode 100644 index 000000000..2daf232dc --- /dev/null +++ b/knowledge/hermes-automated-triage-24h-plan.md @@ -0,0 +1,985 @@ +# Hermes Automated Triage: 24-Hour Delivery Plan + +- Status: planning only +- Created: 2026-08-05 +- Hard delivery limit: 24 elapsed hours from execution authorization +- Priority: deadline first, then the most complete safe automation that fits + +## Executive decision + +The first delivery will automate one narrow, real, repeatable failure-to-repair path. It will not attempt general autonomous coding before that path works end to end. + +The target demonstration is: + +```text +Controlled test failure + -> Jenkins build fails + -> Ariadne detects the terminal failure automatically + -> Ariadne gathers Jenkins evidence and bounded OpenSearch logs + -> Ariadne automatically invokes the Hermes Agent + -> Hermes selects its Titan test-triage skills + -> Hermes returns a structured diagnosis and requested action + -> Ariadne validates the request against an exact allowlist + -> Ariadne creates one predefined repair Job + -> Ariadne triggers one Jenkins rebuild + -> the rebuild passes and the incident resolves +``` + +A second path will demonstrate safe escalation: + +```text +Unsupported, ambiguous or unsafe failure + -> Hermes returns human_required + -> Ariadne publishes a triage metric + -> VictoriaMetrics and vmalert evaluate it + -> Grafana and Alertmanager provide the human-facing signal +``` + +At H+24 work stops. Lower-priority functionality is removed or deferred rather than extending the deadline. + +## Definition of done + +The delivery is fully successful when all of the following are demonstrated: + +- A deliberately seeded test failure causes a Jenkins build to fail. +- Ariadne detects that exact terminal build once, preferably within two minutes. +- Ariadne assigns a stable incident ID based on the Jenkins job and build number. +- The diagnosis bundle includes the first failed stage, retained Jenkins evidence and provenance-bearing OpenSearch excerpts. +- Ariadne invokes the Hermes Agent without a human prompt. +- The invocation explicitly selects the Titan test-triage skill. +- Hermes returns schema-valid facts, inferences, classification, confidence and action request. +- Ariadne accepts only the predefined demo action and executes it once. +- A predefined repair Job corrects only the isolated demo fixture. +- Ariadne triggers one rebuild with failure seeding disabled. +- The rebuilt job passes and Ariadne records the incident as resolved. +- An unsupported classification produces a human-required metric and no mutation. +- No secret values are stored in evidence, model prompts, responses or logs. +- No path can automatically write to `main`, reconcile Flux, or execute model-supplied commands. +- A single Flux-managed setting can disable all automatic actions immediately. + +Target demo timings are: + +- Failure detection: less than two minutes. +- Evidence collection and Hermes response: less than three minutes. +- Repair launch and rebuild request: less than two minutes. +- Complete failure-to-green loop: less than ten minutes, excluding image pulls during a cold cluster start. + +## Current system context + +This section records the facts needed to execute the plan without relying on prior conversation. + +### Jenkins and the homegrown suites + +Jenkins runs the homegrown repositories on ephemeral Kubernetes agent pods. The active quality scope includes Ariadne, Metis, Ananke, Atlasbot, Pegasus, Soteria, titan-iac, bstein-dev-home, data-prepper and Lesavka, with additional build/test jobs for projects such as Arcanagon, Typhon and Veles. + +The normalized quality evidence includes, as applicable: + +```text +build + -> style/docs + -> LOC/naming + -> coverage + -> tests + -> gate glue + -> SonarQube + -> Semgrep + -> Trivy/supply-chain evidence + -> gate enforcement + -> image build and Harbor push +``` + +Jenkins retains build metadata, console output, stage results, JUnit, coverage and quality reports. These retained artifacts are the authoritative source for the exact failed test or gate. + +### Logging and OpenSearch + +OpenSearch does not collect logs by itself. Fluent Bit tails Kubernetes container logs and writes them directly into `kube-*` indices. It also writes node journal records into `journald-*` indices. + +The relevant path is: + +```text +Kubernetes pod stdout/stderr + -> Fluent Bit DaemonSet + -> OpenSearch kube-* indices +``` + +Jenkins-retained console logs and OpenSearch pod logs are separate evidence paths. Jenkins agent command output must not be assumed to exist in CRI container logs. The demo must use Jenkins artifacts for the exact assertion and must ensure that a dedicated demo test-runner or related workload writes a correlated incident message to pod stdout/stderr for OpenSearch. + +Data Prepper is not part of the ordinary log path. It receives OpenTelemetry traces, transforms them and writes trace/service-map indices into OpenSearch. It is irrelevant to the Jenkins log integration except as another separately tested workload. + +Relevant IaC sources include: + +- `services/logging/fluent-bit-helmrelease.yaml` +- `services/logging/opensearch-observability-objects.yaml` +- `services/logging/Jenkinsfile.data-prepper` + +### Metrics and human-facing status + +Jenkins publishers and runtime probes send normalized quality metrics to `platform-quality-gateway`. VictoriaMetrics scrapes Prometheus-compatible endpoints, vmalert evaluates rules, Grafana displays the Atlas Testing dashboards, and Alertmanager owns notification delivery. + +The intended human-escalation path is therefore: + +```text +Ariadne triage metric + -> VictoriaMetrics + -> vmalert + -> Alertmanager notification and Grafana status +``` + +Hermes does not send an alert "to Grafana." Grafana visualizes the metric and alert state. + +Relevant IaC sources include: + +- `services/monitoring/platform-quality-gateway-deployment.yaml` +- `services/monitoring/platform-quality-suite-probe-cronjob.yaml` +- `services/monitoring/vmalert-atlas-availability.yaml` +- `services/monitoring/dashboards/atlas-testing.json` +- `services/quality/sonarqube-exporter-deployment.yaml` + +### Ariadne today + +Ariadne is the deterministic automation and evidence-collection service in the `maintenance` namespace. Its current deployment has access to: + +- Jenkins API credentials and `JENKINS_BASE_URL`. +- VictoriaMetrics through `ARIADNE_VM_URL`. +- OpenSearch through `OPENSEARCH_URL`. +- Kubernetes through its service account. +- The local model through `ARIADNE_TESTING_TRIAGE_MODEL_URL`. + +The deployment schedules Jenkins build-weather collection every ten minutes and testing triage every fifteen minutes. Those intervals are too slow for a smooth demo and should be reduced only for the allowlisted demo path. + +Ariadne's current OpenSearch integration manages index retention. It does not currently retrieve historical log content for testing diagnosis. + +Ariadne currently points model-assisted testing triage at the Hermes Ollama service. Calling Ollama uses the same Qwen model but bypasses the Hermes Agent, its tool loop, skills, memory and approval model. This is not equivalent to invoking Hermes. + +Ariadne already has patterns for launching a platform quality probe Job and for performing Jenkins workspace maintenance. These patterns should be reused for the predefined demo repair rather than introducing a general job-execution system. + +Relevant IaC source: + +- `services/maintenance/apps/ariadne-deployment.yaml` + +Relevant application repository: + +- Gitea: `bstein/Ariadne` +- Local checkout currently at `/home/brad/Development/Ariadne` + +### Hermes today + +Hermes runs as a single read-only triage agent in the `hermes` namespace. Its current configuration provides: + +- Hermes Agent API on port `8642`. +- Hermes dashboard on port `9119`. +- Local Qwen 2.5 `7b-instruct-q4_0` inference through Hermes Ollama. +- Ariadne, Jenkins, VictoriaMetrics, Gitea and Grafana base URLs. +- `kubectl` installed as a read-only cluster inspection tool. +- A persistent home PVC containing memories and runtime-installed skills. + +Hermes does not currently receive `OPENSEARCH_URL`. In the desired design it does not need direct OpenSearch access: Ariadne will provide bounded, sanitized excerpts in the diagnosis bundle. + +Hermes's Flux-managed instructions say to start with Ariadne as the source of truth, then use Jenkins, quality metrics, Flux, Grafana and Kubernetes read-only evidence. Its RBAC allows reads of workloads, events, pod logs, services, storage objects and Flux resources. It cannot read Kubernetes Secret values. + +Hermes explicitly denies or lacks permission for actions such as: + +- `kubectl apply`, delete, patch, scale, cordon, drain or rollout restart. +- Flux suspend, resume or reconcile. +- Vault secret reads. +- Kubernetes Secret reads. +- Jenkins rebuilds. +- Git writes. + +This boundary remains in place. Hermes will request actions; Ariadne will validate and execute exact predefined actions. + +Relevant IaC sources include: + +- `services/hermes/deployment.yaml` +- `services/hermes/configmap.yaml` +- `services/hermes/rbac.yaml` + +### Hermes's current Atlas skills + +A read-only inventory of the live Hermes PVC on 2026-08-05 showed the Atlas triage entry skills: + +- `triage-titan-test-failures` +- `triage-atlas-service-health` +- `tune-atlas-alerts` +- `master-hermes-on-atlas` + +The test-triage skill pack includes: + +- `titan-triage-orchestrator` +- `jenkins-retained-evidence` +- `platform-quality-metrics` +- `kubernetes-readonly-failure-classifier` +- `flux-git-change-correlation` +- `grafana-metric-provenance` +- `triage-evidence-reporting` +- `approved-triage-actions` +- `soteria-backup-health` + +The orchestrator already defines the needed evidence order: Ariadne first, then the smallest relevant specialist, followed by a fact-versus-inference report. The demo invocation should explicitly request `$triage-titan-test-failures`; a new general triage framework is unnecessary. + +These skills currently live on the runtime PVC and are not all declared in `titan-iac`. The demo should not depend on mutating the live PVC manually. Any essential prompt or output contract must be supplied by Ariadne or added through Flux-managed Hermes configuration. + +### Current gaps + +The desired closed loop does not exist today because: + +- Ariadne does not query OpenSearch for triage log evidence. +- Ariadne calls Ollama rather than the Hermes Agent API. +- There is no shared Ariadne-to-Hermes API authentication contract. +- There is no schema for a Hermes triage response. +- There is no allowlisted Hermes-to-Ariadne action request. +- Ariadne does not automatically execute a Hermes-selected repair. +- There is no incident-level idempotency preventing duplicate repair attempts. +- There is no dedicated human-required triage metric and alert. +- The current architecture chart compresses Hermes's internal trigger, routing and decision steps into a single box. + +## Worktree and source-control safety + +The existing working directories must not be used as clean implementation bases. + +As of 2026-08-05: + +- `/home/brad/Development/titan-iac` has extensive unrelated modified, deleted and untracked user work, including an ongoing repository-layout migration and untracked Mermaid files. +- `/home/brad/Development/Ariadne` is dirty on branch `codex/ariadne-metrics-hotfix`. + +Execution must: + +1. Inspect the actual revisions deployed by Flux and Jenkins. +2. Fetch without altering either existing worktree. +3. Create dedicated clean Git worktrees for the demo branches. +4. Never reset, clean, stash, rebase or overwrite either existing dirty worktree. +5. Keep Ariadne application changes and titan-iac/Flux changes in separately reviewable commits. +6. Bring over only the intentional planning/diagram change when appropriate. + +Suggested isolated paths are: + +```text +/home/brad/Development/_worktrees/ariadne-hermes-triage-demo +/home/brad/Development/_worktrees/titan-iac-hermes-triage-demo +``` + +The implementation branch point must be the current authoritative primary/deployed revision, not an assumed local branch. + +## Deadline-driven scope + +### Required scope + +- One dedicated demo Jenkins job. +- One deterministic failure signature. +- One isolated demo fixture. +- One bounded OpenSearch search contract. +- One automatic Hermes invocation. +- One structured decision contract. +- One exact repair action. +- One rebuild attempt. +- One human-required metric and alert path. +- One complete source-level Mermaid update. + +### Explicit non-goals + +- General conversational OpenSearch access. +- Arbitrary index selection or unbounded log retrieval. +- General autonomous repository editing. +- Direct commits or merges to `main`. +- Model-generated Kubernetes manifests or commands. +- Automatic Flux reconciliation. +- Automatic SonarQube, Semgrep or Trivy waivers. +- Secret inspection. +- Multiple repair retries. +- Multiple repositories or failure signatures before the primary loop is green. +- A permanent generalized incident-management platform. + +## Demo design + +### Isolated test fixture + +Use a dedicated `hermes-triage-demo` namespace, Jenkins job and fixture. The preferred deterministic design is: + +1. A small PVC stores a single state value such as `healthy` or `unhealthy`. +2. The demo is explicitly armed through a Jenkins parameter such as `SEED_FAILURE=true`. +3. A dedicated Kubernetes test-runner Job reads the fixture. +4. When the fixture is unhealthy, it writes a structured incident line to stdout/stderr and exits nonzero. +5. Jenkins waits for the Job, records the test failure and archives a small JUnit result. +6. Fluent Bit forwards the test-runner Job output to OpenSearch. +7. The test-runner Job name and message include the Jenkins job/build incident ID. + +Example structured log: + +```json +{ + "event": "hermes_demo_test_failure", + "incident_id": "hermes-triage-demo/42", + "classification_hint": "demo_fixture_unhealthy", + "message": "expected fixture state healthy; found unhealthy" +} +``` + +The Jenkins rebuild must set `SEED_FAILURE=false`, otherwise every rebuild would deliberately recreate the failure. + +The namespace isolates the test-runner permissions and prevents a demo repair from touching production workloads. + +### Predefined repair Job + +Ariadne will contain one exact action mapping: + +```text +repair_demo_fixture + -> create hermes-demo-repair- Job + -> mount only the demo fixture PVC + -> write healthy state + -> emit a structured repair result + -> exit +``` + +Hermes cannot supply the image, command, namespace, PVC, labels or arbitrary parameters. It can request only the action ID. + +After the Job succeeds, Ariadne requests one Jenkins rebuild with failure seeding disabled. + +### Why a repair Job is the critical path + +A source-code test failure generally requires a Git change, but safe autonomous code editing requires repository credentials, checkout isolation, patch generation, tests, branch creation and PR lifecycle handling. Making that the primary demo would threaten the 24-hour limit. + +The fixture repair demonstrates a real closed loop: + +- Test failure is real. +- Jenkins and OpenSearch evidence are real. +- Hermes classification is real. +- Hermes action selection is real. +- Ariadne authorization and execution are real. +- The repair and green rebuild are real. + +Autonomous Gitea PR creation remains a stretch goal that begins only after the required loop is stable. + +## Evidence contract + +### Incident identity + +Use a stable ID: + +```text +/ +``` + +Every log line, bundle, Hermes request, action, repair Job and metric must carry that ID. + +### Jenkins evidence + +The bundle should include: + +- Job and build number. +- Branch and commit when available. +- Build URL. +- Start and end timestamps. +- Terminal result. +- First failed stage. +- Console tail. +- Failed JUnit cases. +- Selected quality reports when relevant. +- Evidence source and retrieval timestamp. + +### OpenSearch evidence + +The demo query should be intentionally fixed and bounded: + +- Indices: `kube-*` only. +- Time range: build start minus five minutes through build end plus five minutes. +- Namespace: `hermes-triage-demo`, plus `jenkins` only if it adds useful context. +- Correlation: exact incident ID first; namespace and time fallback second. +- Size: at most 50 records initially, never more than 100. +- Fields: timestamp, namespace, pod, container, log/message and selected safe labels. +- Timeout: approximately five seconds. +- Response byte cap: small enough to prevent prompt flooding. +- Pagination: none in the demo. + +Each excerpt must retain: + +- Index or source identifier. +- Timestamp. +- Namespace. +- Pod. +- Container. +- Sanitized message. + +Basic sanitization must remove or mask: + +- `Authorization` headers. +- Bearer tokens. +- Cookies and session IDs. +- Common password, token and secret assignments. +- Private key blocks. + +This is demo-grade sanitization, not a claim of comprehensive data-loss prevention. Queries remain restricted to the isolated namespace wherever possible. + +### Ariadne diagnosis bundle + +Add a `log_evidence` section without replacing the existing bundle fields: + +```json +{ + "incident_id": "hermes-triage-demo/42", + "generated_at": "...", + "jenkins": {}, + "quality_metrics": {}, + "runtime_context": {}, + "log_evidence": { + "query_window": {}, + "records": [], + "truncated": false, + "error": null + } +} +``` + +Empty or unavailable OpenSearch results must be represented explicitly. They must not block Jenkins-based diagnosis. + +## Automatic Hermes invocation + +### Required behavior + +Ariadne must call the Hermes Agent API, not only the Ollama completion endpoint. The invocation prompt should be stable and small: + +```text +Use $triage-titan-test-failures. +Analyze incident . +Treat the attached Ariadne bundle as the source of truth. +Identify the first enforced failure. +Distinguish facts from inference. +Return only the required triage response schema. +Do not perform mutations. +``` + +The bundle is attached as structured content. Ariadne records the Hermes request ID or session ID for audit and idempotency. + +### Authentication + +No API key may be committed to Git. + +The preferred path is: + +1. Generate one internal Hermes API key. +2. Store it in an approved Vault path. +3. Inject it into Hermes as `API_SERVER_KEY`. +4. Inject it into Ariadne as `HERMES_API_KEY`. +5. Restrict network access to the required namespaces where practical. + +The exact Hermes Agent API path and authentication behavior must be verified in the first hour. The current Hermes init container generates a key in its persistent `.env` if none exists; the integration must not scrape that file or copy it into Git. + +### Response schema + +Hermes must return a bounded machine-readable response: + +```json +{ + "incident_id": "hermes-triage-demo/42", + "classification": "known_demo_fixture_failure", + "confidence": 0.95, + "facts": [ + { + "statement": "...", + "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", + "reference": "..." + } + ], + "inferences": [], + "first_failed_gate": "tests", + "requested_action": { + "type": "run_ariadne_job", + "id": "repair_demo_fixture" + }, + "human_required": false, + "reason": "" +} +``` + +Invalid JSON, unknown fields, mismatched incident IDs, low confidence and unknown actions all become human-required results. + +## Detection and deduplication + +For the demo, Ariadne should detect only an allowlisted job such as `hermes-triage-demo`. Reducing the global scheduler interval must not cause automatic action against unrelated failures. + +Suggested configuration: + +```text +ARIADNE_HERMES_AUTOTRIAGE_ENABLED=true +ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST=hermes-triage-demo +ARIADNE_HERMES_AUTOTRIAGE_INTERVAL_SECONDS=60 +ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false +ARIADNE_HERMES_ALLOWED_ACTIONS=repair_demo_fixture +ARIADNE_HERMES_MIN_CONFIDENCE=0.85 +ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT=1 +``` + +The remediation flag remains false until the evidence-and-Hermes path passes in observe mode. + +Ariadne must persist or otherwise reliably track: + +- Incident observed. +- Hermes invocation started/completed. +- Action requested/accepted/rejected. +- Repair Job created/completed. +- Rebuild requested. +- Final build result. + +The same Jenkins build must never invoke a second repair after its action record exists. + +## Action authorization + +Ariadne may accept the demo action only when: + +- The Jenkins build is terminal and failed. +- The job is allowlisted. +- The incident ID matches the current record. +- The classification is exactly `known_demo_fixture_failure`. +- The action is exactly `repair_demo_fixture`. +- Confidence is at least the configured threshold. +- The Jenkins/JUnit or log evidence contains the expected failure signature. +- No action has already run for the incident. +- Automatic remediation is enabled. + +Hermes must not provide: + +- Shell commands. +- Kubernetes YAML. +- Container images. +- Namespaces. +- PVC names. +- Git commands. +- Retry counts. + +These remain hardcoded, tested Ariadne behavior. + +## Failure-class policy + +| Failure class | Automatic behavior in the 24-hour delivery | +|---|---| +| Known demo fixture failure | Run `hermes-demo-repair`, then rebuild once | +| Stale runtime quality telemetry | Optionally run the existing platform quality suite probe | +| Clearly transient demo Jenkins failure | At most one allowlisted rebuild if time permits | +| Ordinary source/test defect | Diagnose and recommend; human required | +| SonarQube failure | Human required; never waive automatically | +| Semgrep failure | Human required; never waive automatically | +| Trivy/supply-chain failure | Human required; never waive automatically | +| Kubernetes or Flux defect | Read-only diagnosis; human-required repo/Flux change | +| Missing or stale decisive evidence | Human required | +| Authentication, Secret or Vault issue | Human required | +| Storage-destructive or hardware issue | Human required | +| Unknown class or low confidence | Human required | + +## Triage metrics and escalation + +Ariadne should expose a small stable metric set, for example: + +```text +ariadne_hermes_triage_incident{job="...",build="...",status="detected|diagnosed|repairing|resolved|human_required"} 1 +ariadne_hermes_triage_action_total{action="...",result="accepted|rejected|success|failed"} 1 +ariadne_hermes_triage_last_success_timestamp_seconds ... +ariadne_hermes_triage_duration_seconds ... +``` + +Avoid unbounded labels such as full error messages, commit messages or model text. + +The minimum alert is: + +```text +human_required incident remains active for a short demo-safe interval + -> vmalert fires + -> Alertmanager handles notification + -> Grafana links to Jenkins and the Ariadne incident report +``` + +If dashboard changes threaten the deadline, the metric and alert rule take precedence over a polished panel. + +## 24-hour execution schedule + +### H+0 to H+1: establish authoritative state + +- Confirm the deployed Flux revision and Ariadne image/source revision. +- Confirm the Jenkins job definitions and branch behavior. +- Create clean dedicated worktrees without touching existing dirty worktrees. +- Verify the Hermes Agent API route, request format and authentication. +- Verify Ariadne-to-Hermes network reachability. +- Verify Ariadne-to-OpenSearch reachability. +- Verify the maintenance service account can create/watch the intended isolated Job or identify the minimal RBAC change. +- Freeze the exact demo job, namespace, failure signature and repair action. + +Gate at H+1: + +- No additional repositories or failure classes enter scope. +- If the Hermes Agent API path is still unknown, allocate no more than one additional hour before activating the fallback. + +### H+1 to H+4: bounded OpenSearch evidence + +- Add the fixed OpenSearch query client to Ariadne. +- Filter by the demo namespace, incident ID and build time window. +- Add response size, count and timeout limits. +- Add basic sanitization. +- Add `log_evidence` to the diagnosis bundle. +- Add unit tests for success, empty result, timeout, malformed payload, truncation and sanitization. +- Perform one live read-only query. + +Compromise at H+4: + +- Use a single fixed `kube-*` query rather than a general query builder. +- Retain raw chronological excerpts rather than semantic grouping. +- OpenSearch failure becomes explicit missing evidence and does not block Jenkins diagnosis. + +### H+4 to H+7: automatic Hermes invocation + +- Configure the shared internal API credential through Vault. +- Implement Ariadne's Hermes Agent client. +- Invoke the explicit Titan triage skill with the incident bundle. +- Require and validate structured JSON. +- Add a bounded timeout and one retry. +- Record request/session ID and duration. +- Deduplicate by incident ID. + +Compromise at H+7: + +- If the Hermes Agent API remains blocked, call the existing Hermes Ollama endpoint with the triage workflow embedded in the prompt. +- Clearly label this fallback as model-assisted Ariadne triage, not full Hermes Agent skill execution. +- Preserve the same response schema so the rest of the loop remains usable. + +### H+7 to H+10: decision and authorization contract + +- Freeze the response schema. +- Implement incident, class, confidence and action validation. +- Add idempotency and maximum-action checks. +- Add the global action kill switch. +- Reject arbitrary action content. +- Emit requested, accepted and rejected action metrics. +- Test invalid JSON, mismatched incident, low confidence, unknown action and duplicate incident. + +Gate at H+10: + +- Response schema and action list freeze. +- No additional automatic action type is added. + +### H+10 to H+14: isolated repair loop + +- Add the isolated namespace, fixture and test-runner resources. +- Add the dedicated Jenkins demo job or pipeline definition. +- Ensure the test-runner emits the incident ID to pod stdout/stderr. +- Add the predefined repair Job builder/executor to Ariadne. +- Wait for repair completion. +- Trigger one Jenkins rebuild with `SEED_FAILURE=false`. +- Record the rebuilt job and final status. +- Stop after one failed repair or rebuild. + +Compromise at H+14: + +- If repair execution is not repeatable, disable remediation. +- Ship automatic detection, evidence, Hermes diagnosis and human escalation. +- Do not consume remaining time attempting general repair logic. + +### H+14 to H+16: human-required signal + +- Add incident/action metrics. +- Add one vmalert rule for unresolved human-required incidents. +- Add Jenkins and Ariadne report links where the current dashboard model supports them. +- Confirm the signal clears on a resolved incident. + +Compromise at H+16: + +- Use the existing Atlas Testing dashboard and direct VictoriaMetrics query if a new panel is not ready. +- Preserve the metric and alert rule over dashboard polish. + +### H+16 to H+18: Flux integration and documentation + +- Add only required environment, Vault injection, network and RBAC changes. +- Keep the repair Job namespace and permissions isolated. +- Add the automatic-action kill switch to Flux-managed configuration. +- Rewrite `mermaid/TestAutomation.mmd` to show the actual trigger, evidence collection, Agent invocation, skill routing, decision schema, action gate, Ariadne execution and human escalation. +- Update this plan with final deviations and deferred hardening. +- Do not render SVG output unless separately requested. + +### H+18 to H+21: validation and first rehearsal + +- Run focused and complete relevant Ariadne tests. +- Run repository quality checks required by its Jenkins pipeline. +- Run `kustomize build` for each affected titan-iac kustomization. +- Run client-side dry-run validation without mutating the cluster. +- Deliver through Git and Flux only. +- Seed the controlled failure and observe the full loop. +- Fix only critical-path failures. + +Gate at H+21: + +- Feature freeze. +- No autonomous-code stretch work starts after this point. + +### H+21 to H+23: repeatability rehearsal + +Run these scenarios: + +1. Known fixture failure -> repair Job -> green rebuild. +2. Unsupported classification -> human-required signal and no repair. +3. Duplicate observation -> no duplicate repair. +4. OpenSearch timeout -> Jenkins-based diagnosis continues. +5. Hermes timeout or invalid response -> human-required signal. +6. Remediation kill switch disabled -> diagnosis only. + +Record timings, incident IDs, action count and final Jenkins status. + +### H+23 to H+24: hard stop + +- Make no new feature changes. +- Confirm the kill switch and rollback path. +- Capture the final working scope and known limitations. +- Ship the highest completed fallback tier. +- Stop at H+24. + +## Deadline compromise ladder + +When time is at risk, remove functionality in this order: + +1. Drop autonomous Gitea/code editing. +2. Limit remediation to `repair_demo_fixture` only. +3. Drop transient rebuild and optional suite-probe actions. +4. Drop additional failure classifications. +5. Replace dynamic OpenSearch construction with one fixed query. +6. Reuse the existing Grafana dashboard instead of adding a polished panel. +7. Fall back from Hermes Agent API to the same Qwen model with the triage skill procedure embedded in the prompt. +8. Disable automatic remediation but keep automatic diagnosis. +9. Preserve detection, evidence and human-required alerting as the minimum shippable system. + +The deadline is not extended to preserve a lower-priority feature. + +## Delivery tiers at H+24 + +### Tier A: target + +- Automatic detection. +- Jenkins and OpenSearch evidence. +- Hermes Agent skill execution. +- Structured diagnosis. +- One allowlisted repair Job. +- One rebuild. +- Human escalation for unsupported failures. + +### Tier B: acceptable compromise + +- Automatic detection. +- Jenkins and OpenSearch evidence. +- Hermes Agent skill execution. +- Structured diagnosis and recommendation. +- Human escalation. +- Automatic repair disabled. + +### Tier C: minimum shippable + +- Automatic detection. +- Jenkins and bounded OpenSearch evidence. +- Ariadne calls Qwen with the triage workflow embedded. +- Structured recommendation. +- Human-required metric/alert. + +No tier may silently claim Hermes Agent execution when only Ollama was invoked. + +## Testing strategy + +### Ariadne unit tests + +- OpenSearch query bounds and field selection. +- Correlation by incident ID and time. +- Sanitization and truncation. +- Empty, unavailable and malformed OpenSearch responses. +- Hermes request construction. +- Hermes response schema validation. +- Confidence threshold. +- Action allowlist. +- Incident mismatch rejection. +- Idempotency and maximum action count. +- Repair Job construction from hardcoded values. +- Jenkins rebuild parameter handling. +- Metrics with bounded label cardinality. + +### Integration tests + +- Mocked Jenkins failure plus mocked OpenSearch evidence. +- Mocked Hermes accepted action. +- Mocked unsupported classification. +- Repair Job success/failure. +- Jenkins rebuild request success/failure. +- Hermes timeout and malformed response. +- OpenSearch timeout while Jenkins evidence remains usable. + +### Live read-only preflight + +- Query one known `kube-*` time window. +- Confirm the expected Kubernetes field names. +- Confirm Hermes API status and authentication behavior. +- Confirm Jenkins GET access. +- Confirm VictoriaMetrics can query Ariadne metrics. + +### End-to-end demonstration + +- Arm the demo failure deliberately. +- Confirm one failed Jenkins build. +- Confirm one Ariadne incident. +- Confirm correlated OpenSearch excerpts. +- Confirm the expected Hermes skill and response. +- Confirm one repair Job. +- Confirm one rebuild. +- Confirm green result and resolved metric. +- Repeat once to prove the path is not a one-off race. + +## Flux and validation rules + +All persistent cluster changes must be represented in Git and reconciled through Flux. No manual `kubectl apply`, patch or ad hoc cluster edit is part of the delivery. + +For every affected titan-iac kustomization: + +```text +kustomize build +kubectl apply --server-side --dry-run=client -k +flux diff kustomization --path +``` + +Actual reconciliation occurs only after the intended commits are available to Flux. + +Application changes in Ariadne must run the focused tests for new services plus its relevant full test/quality suite. Exact commands should follow the authoritative branch's Jenkinsfile and test configuration rather than assumptions from the currently dirty local checkout. + +## Security and safety boundary + +- Store all shared credentials in Vault; never Git. +- Restrict automatic triage to an explicit Jenkins job allowlist. +- Restrict OpenSearch to `kube-*`, time bounds and isolated namespaces. +- Sanitize evidence before model use. +- Never send entire indices, artifacts or unrestricted logs to the model. +- Do not add Secret-read permission to Hermes. +- Do not grant Hermes Kubernetes mutation. +- Do not allow model-supplied shell commands or manifests. +- Keep action definitions in tested Ariadne code. +- Permit one action and one rebuild per incident. +- Require human intervention on low confidence or missing decisive evidence. +- Keep a default-off remediation flag until observe mode passes. + +## Rollback and kill switches + +Immediate behavioral rollback: + +```text +ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false +``` + +Further rollback steps are: + +1. Disable automatic Hermes invocation while retaining normal Ariadne schedules. +2. Restore the original Jenkins-weather and testing-triage intervals. +3. Disable the demo Jenkins job. +4. Remove or suspend only the isolated demo resources through Git/Flux. +5. Revoke the shared Hermes API key in Vault. +6. Leave ordinary Jenkins, Fluent Bit, OpenSearch, VictoriaMetrics and Hermes read-only behavior untouched. + +The repair action must be safe to run twice even though idempotency should prevent the second run. + +## Repository touchpoints + +Expected Ariadne application changes: + +- Bounded OpenSearch evidence client. +- Diagnosis-bundle enrichment. +- Hermes Agent API client. +- Response schema and validation. +- Incident deduplication/state. +- Allowlisted repair action executor. +- Jenkins rebuild request. +- Triage metrics. +- Unit and integration tests. + +Expected titan-iac changes: + +- Ariadne environment and Vault injection. +- Hermes shared API-key configuration if required. +- Isolated demo namespace/PVC/RBAC/test-runner resources. +- Predefined repair Job permissions. +- Demo Jenkins JCasC/pipeline definition. +- vmalert rule and minimal Grafana integration. +- `mermaid/TestAutomation.mmd` update. + +Do not modify unrelated workloads or fold repository-layout migrations into these changes. + +## Optional autonomous-code stretch + +This begins only if Tier A passes end to end before H+16 and at least eight hours remain. + +The only acceptable stretch is a restricted Gitea PR workflow: + +```text +Hermes proposes patch + -> Ariadne validates repository and base branch + -> dedicated service creates a demo branch/PR + -> Jenkins validates the branch + -> human reviews and merges +``` + +Restrictions: + +- One demo repository. +- Dedicated branch prefix. +- No push to `main` or `master`. +- No automatic merge. +- Separate least-privilege Gitea token in Vault. +- Patch size and file-path limits. +- Jenkins validation required. + +This stretch is dropped immediately if it threatens rehearsal or the H+24 stop. + +## Required architecture-document update + +After behavior is known, `mermaid/TestAutomation.mmd` must show these internal steps rather than compressing Hermes into one box: + +```text +Trigger + -> terminal Jenkins failure or human request + +Detection + -> Ariadne allowlisted polling and incident deduplication + +Evidence + -> Jenkins metadata/artifacts + -> bounded OpenSearch excerpts + -> VictoriaMetrics + -> Kubernetes/Flux/Gitea/Grafana context when relevant + +Agent invocation + -> Ariadne calls Hermes Agent API + -> explicit triage-titan-test-failures skill + -> local Qwen model through Hermes Ollama + +Decision + -> first enforced failure + -> facts versus inference + -> specialist skill routing + -> confidence and action schema + +Authorization + -> Ariadne allowlist, incident match, confidence, idempotency and kill switch + +Response + -> predefined repair Job and one rebuild + -> or human-required metric and alert +``` + +The chart must visually distinguish current behavior, newly implemented demo behavior and deferred capabilities. It should continue to keep Jenkins-retained logs separate from OpenSearch pod logs and should not imply direct Hermes OpenSearch access. + +## Final handoff record + +At H+24, append a short result section containing: + +- Delivered tier. +- Exact commits and deployed revisions. +- Demo job/build IDs. +- Incident ID. +- Evidence query window. +- Hermes request/session ID. +- Requested and executed action. +- Repair Job name. +- Rebuild result. +- Alert result. +- Total end-to-end duration. +- Compromises activated. +- Deferred hardening. +- Kill-switch and rollback verification. + +The handoff must state plainly whether the full Hermes Agent or the Ollama fallback performed the diagnosis. diff --git a/knowledge/metis.md b/knowledge/metis.md index 5b0d06be5..4f3d471a3 100644 --- a/knowledge/metis.md +++ b/knowledge/metis.md @@ -1,12 +1,14 @@ # Metis (node recovery) ## Node classes (current map) -- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent) +- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent) - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint) - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks) - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent) +- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent) - amd64 agents: titan-22/24 (Debian 13, k3s agent) -- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, future titan-20/21 (when added), plus any newcomers. +- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels) +- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers. ## Longhorn disk UUIDs (critical nodes) - titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4) @@ -17,10 +19,9 @@ ## Metis repo (~/Development/metis) - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`). - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints). -- `AGENTS.md` in repo is untracked and holds raw notes. ## Next implementation steps - Add per-class golden image refs and checksums (Harbor or file://) when ready. - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths. - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection. -- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited. +- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited. diff --git a/knowledge/runbooks/ci-gitea-jenkins.md b/knowledge/runbooks/ci-gitea-jenkins.md index 48dc91fde..14f76d59d 100644 --- a/knowledge/runbooks/ci-gitea-jenkins.md +++ b/knowledge/runbooks/ci-gitea-jenkins.md @@ -3,7 +3,7 @@ title: "CI: Gitea → Jenkins pipeline" tags: ["atlas", "ci", "gitea", "jenkins"] owners: ["brad"] entrypoints: ["scm.bstein.dev", "ci.bstein.dev"] -source_paths: ["services/gitea", "services/jenkins", "scripts/jenkins_cred_sync.sh", "scripts/gitea_cred_sync.sh"] +source_paths: ["services/gitea", "services/jenkins", "scripts/sync/jenkins_cred_sync.sh", "scripts/sync/gitea_cred_sync.sh"] --- # CI: Gitea → Jenkins pipeline @@ -14,7 +14,7 @@ Atlas uses Gitea for source control and Jenkins for CI. Authentication is via Ke ## Where it is configured - Gitea manifests: `services/gitea/` - Jenkins manifests: `services/jenkins/` -- Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh` +- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh` ## What users do (typical flow) - Create a repo in Gitea. diff --git a/knowledge/runbooks/cluster-power-recovery.md b/knowledge/runbooks/cluster-power-recovery.md index 67a10d3d9..86436d4c2 100644 --- a/knowledge/runbooks/cluster-power-recovery.md +++ b/knowledge/runbooks/cluster-power-recovery.md @@ -14,8 +14,8 @@ Bootstrapping risk to remember - Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back. Script -- `scripts/cluster_power_recovery.sh` -- `scripts/cluster_power_console.sh` +- `scripts/ops/cluster_power_recovery.sh` +- `scripts/ops/cluster_power_console.sh` - Modes: - `prepare` - `shutdown` @@ -26,21 +26,21 @@ Script Dry-run examples - Shutdown preview: - - `scripts/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain` + - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain` - Startup preview: - - `scripts/cluster_power_recovery.sh startup` + - `scripts/ops/cluster_power_recovery.sh startup` - Harbor seed preview: - - `scripts/cluster_power_recovery.sh harbor-seed` + - `scripts/ops/cluster_power_recovery.sh harbor-seed` Execute examples - Prepare helper image on every node: - - `scripts/cluster_power_recovery.sh prepare --execute` + - `scripts/ops/cluster_power_recovery.sh prepare --execute` - Seed Harbor runtime images onto `titan-05` from the control-host bundle: - - `scripts/cluster_power_recovery.sh harbor-seed --execute` + - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute` - Planned shutdown: - - `scripts/cluster_power_recovery.sh shutdown --execute` + - `scripts/ops/cluster_power_recovery.sh shutdown --execute` - Planned startup (canonical branch): - - `scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main` + - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main` Manual remote console examples - Canonical operator hosts: @@ -129,7 +129,7 @@ Operational notes - Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer. - Harbor is reconciled after the first critical stateful services. - Harbor bootstrap is now designed around a control-host bundle: - - Build the Harbor bundle locally with `scripts/build_harbor_bootstrap_bundle.sh`. + - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`. - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`. - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`. - The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations. diff --git a/knowledge/runbooks/kb-authoring.md b/knowledge/runbooks/kb-authoring.md index 9378d1d4d..9b872354d 100644 --- a/knowledge/runbooks/kb-authoring.md +++ b/knowledge/runbooks/kb-authoring.md @@ -3,7 +3,7 @@ title: "KB authoring: what to write (and what not to)" tags: ["atlas", "kb", "runbooks"] owners: ["brad"] entrypoints: [] -source_paths: ["knowledge/runbooks", "scripts/knowledge_render_atlas.py"] +source_paths: ["knowledge/runbooks", "scripts/render/knowledge_render_atlas.py"] --- # KB authoring: what to write (and what not to) diff --git a/knowledge/software/metis.md b/knowledge/software/metis.md index 7ca3b3991..044a75e70 100644 --- a/knowledge/software/metis.md +++ b/knowledge/software/metis.md @@ -1,12 +1,14 @@ # Metis (node recovery) ## Node classes (current map) -- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent) +- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent) - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint) - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks) - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent) +- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent) - amd64 agents: titan-22/24 (Debian 13, k3s agent) -- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers. +- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels) +- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers. ### Jetson nodes (titan-20/21) - Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64. @@ -22,13 +24,12 @@ ## Metis repo (~/Development/metis) - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`). - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints). -- `AGENTS.md` in repo is untracked and holds raw notes. ## Next implementation steps - Add per-class golden image refs and checksums (Harbor or file://) when ready. - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths. - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection. -- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited. +- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited. ## Node OS/Kernel/CRI snapshot (Jan 2026) - titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64 @@ -55,10 +56,10 @@ - titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64 -### External hosts +### Dedicated and special-purpose hosts - titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled. - titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q). -- titan-23/oceanus: TODO audit (future). +- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement. ### Control plane Pis (titan-0a/0b/0c) diff --git a/scripts/comms_sync_kb.sh b/scripts/comms_sync_kb.sh deleted file mode 100755 index 16f93322f..000000000 --- a/scripts/comms_sync_kb.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -python scripts/knowledge_render_atlas.py --write -python scripts/knowledge_render_atlas.py --write --out services/comms/knowledge diff --git a/scripts/test_atlas_user_cleanup.py b/scripts/manual-tests/test_atlas_user_cleanup.py similarity index 100% rename from scripts/test_atlas_user_cleanup.py rename to scripts/manual-tests/test_atlas_user_cleanup.py diff --git a/scripts/test_user_cleanup.py b/scripts/manual-tests/test_user_cleanup.py similarity index 100% rename from scripts/test_user_cleanup.py rename to scripts/manual-tests/test_user_cleanup.py diff --git a/scripts/test_user_cleanup.sh b/scripts/manual-tests/test_user_cleanup.sh similarity index 59% rename from scripts/test_user_cleanup.sh rename to scripts/manual-tests/test_user_cleanup.sh index 346aedc48..39e9f10bf 100755 --- a/scripts/test_user_cleanup.sh +++ b/scripts/manual-tests/test_user_cleanup.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash set -euo pipefail -# Manual-only helper to run `scripts/test_user_cleanup.py` inside the portal backend container. +# Manual-only helper to run `scripts/manual-tests/test_user_cleanup.py` inside the portal backend container. # # Usage (dry-run): -# scripts/test_user_cleanup.sh --prefix test- +# scripts/manual-tests/test_user_cleanup.sh --prefix test- # # Usage (apply): -# scripts/test_user_cleanup.sh --prefix test- --apply --confirm test- +# scripts/manual-tests/test_user_cleanup.sh --prefix test- --apply --confirm test- NS="${PORTAL_NAMESPACE:-bstein-dev-home}" TARGET="${PORTAL_BACKEND_EXEC_TARGET:-deploy/bstein-dev-home-backend}" diff --git a/scripts/test_vaultwarden_user_cleanup.py b/scripts/manual-tests/test_vaultwarden_user_cleanup.py similarity index 98% rename from scripts/test_vaultwarden_user_cleanup.py rename to scripts/manual-tests/test_vaultwarden_user_cleanup.py index 93b7aa53a..2dad95b65 100755 --- a/scripts/test_vaultwarden_user_cleanup.py +++ b/scripts/manual-tests/test_vaultwarden_user_cleanup.py @@ -11,10 +11,10 @@ conservative: - Supports a protected email allowlist to prevent catastrophic mistakes. Example (dry-run): - scripts/test_vaultwarden_user_cleanup.py --prefix e2e- + scripts/manual-tests/test_vaultwarden_user_cleanup.py --prefix e2e- Example (apply): - scripts/test_vaultwarden_user_cleanup.py --prefix e2e- --apply --confirm e2e- + scripts/manual-tests/test_vaultwarden_user_cleanup.py --prefix e2e- --apply --confirm e2e- """ from __future__ import annotations diff --git a/scripts/test_vaultwarden_user_cleanup.sh b/scripts/manual-tests/test_vaultwarden_user_cleanup.sh similarity index 64% rename from scripts/test_vaultwarden_user_cleanup.sh rename to scripts/manual-tests/test_vaultwarden_user_cleanup.sh index f21f85e6c..b12e964db 100755 --- a/scripts/test_vaultwarden_user_cleanup.sh +++ b/scripts/manual-tests/test_vaultwarden_user_cleanup.sh @@ -4,10 +4,10 @@ set -euo pipefail # Manual-only helper to clean Vaultwarden test users and invites from Postgres. # # Usage (dry-run): -# scripts/test_vaultwarden_user_cleanup.sh --prefix e2e- +# scripts/manual-tests/test_vaultwarden_user_cleanup.sh --prefix e2e- # # Usage (apply): -# scripts/test_vaultwarden_user_cleanup.sh --prefix e2e- --apply --confirm e2e- +# scripts/manual-tests/test_vaultwarden_user_cleanup.sh --prefix e2e- --apply --confirm e2e- SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" diff --git a/scripts/build_ananke_node_helper.sh b/scripts/ops/build_ananke_node_helper.sh similarity index 90% rename from scripts/build_ananke_node_helper.sh rename to scripts/ops/build_ananke_node_helper.sh index 8d03b22d5..e8f16eab7 100755 --- a/scripts/build_ananke_node_helper.sh +++ b/scripts/ops/build_ananke_node_helper.sh @@ -26,7 +26,7 @@ while [[ $# -gt 0 ]]; do ;; -h|--help) cat <] [--docker-config ] [--platforms ] [--builder ] +Usage: scripts/ops/build_ananke_node_helper.sh [--image ] [--docker-config ] [--platforms ] [--builder ] USAGE exit 0 ;; diff --git a/scripts/build_harbor_bootstrap_bundle.sh b/scripts/ops/build_harbor_bootstrap_bundle.sh similarity index 95% rename from scripts/build_harbor_bootstrap_bundle.sh rename to scripts/ops/build_harbor_bootstrap_bundle.sh index 5b590cf21..475e15be6 100755 --- a/scripts/build_harbor_bootstrap_bundle.sh +++ b/scripts/ops/build_harbor_bootstrap_bundle.sh @@ -31,7 +31,7 @@ while [[ $# -gt 0 ]]; do ;; -h|--help) cat <] [--bundle-file ] [--docker-config ] [--platform ] [--zstd-level ] +Usage: scripts/ops/build_harbor_bootstrap_bundle.sh [--images-file ] [--bundle-file ] [--docker-config ] [--platform ] [--zstd-level ] USAGE exit 0 ;; diff --git a/scripts/cluster_power_console.sh b/scripts/ops/cluster_power_console.sh similarity index 70% rename from scripts/cluster_power_console.sh rename to scripts/ops/cluster_power_console.sh index 0b2814fc8..8a1d8fa9a 100755 --- a/scripts/cluster_power_console.sh +++ b/scripts/ops/cluster_power_console.sh @@ -6,7 +6,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" usage() { cat <<'USAGE' Usage: - scripts/cluster_power_console.sh [--repo-dir ] [--delegate-host ] [recovery-script-options...] + scripts/ops/cluster_power_console.sh [--repo-dir ] [--delegate-host ] [recovery-script-options...] Purpose: Friendly manual entrypoint for running Ananke from a remote console. @@ -17,9 +17,9 @@ Defaults: --delegate-host titan-db Examples: - scripts/cluster_power_console.sh shutdown --execute - scripts/cluster_power_console.sh startup --execute --force-flux-branch main - scripts/cluster_power_console.sh --delegate-host titan-24 shutdown --execute + scripts/ops/cluster_power_console.sh shutdown --execute + scripts/ops/cluster_power_console.sh startup --execute --force-flux-branch main + scripts/ops/cluster_power_console.sh --delegate-host titan-24 shutdown --execute USAGE } @@ -57,7 +57,7 @@ if [[ $# -lt 1 ]]; then fi SIBLING_SCRIPT="${SCRIPT_DIR}/cluster_power_recovery.sh" -REPO_SCRIPT="${REPO_DIR}/scripts/cluster_power_recovery.sh" +REPO_SCRIPT="${REPO_DIR}/scripts/ops/cluster_power_recovery.sh" LOCAL_SCRIPT="" if [[ -x "${SIBLING_SCRIPT}" ]]; then @@ -82,6 +82,6 @@ remote_cmd="" if [[ -n "${REMOTE_REPO_DIR}" ]]; then remote_cmd+="ANANKE_REPO_DIR=$(printf '%q' "${REMOTE_REPO_DIR}") " fi -remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi" +remote_cmd+="if [ -x ~/ananke-tools/cluster_power_recovery.sh ]; then ~/ananke-tools/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/ops/cluster_power_recovery.sh ${quoted_args}; elif [ -x ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ]; then ${quoted_repo_dir}/scripts/cluster_power_recovery.sh ${quoted_args}; else echo 'cluster-power-console: remote recovery script not found' >&2; exit 1; fi" exec ssh -o BatchMode=yes -o ConnectTimeout=8 "${DELEGATE_HOST}" "${remote_cmd}" diff --git a/scripts/cluster_power_recovery.sh b/scripts/ops/cluster_power_recovery.sh similarity index 99% rename from scripts/cluster_power_recovery.sh rename to scripts/ops/cluster_power_recovery.sh index 41f38d1de..157512cea 100755 --- a/scripts/cluster_power_recovery.sh +++ b/scripts/ops/cluster_power_recovery.sh @@ -16,7 +16,7 @@ fi usage() { cat < [options] + scripts/ops/cluster_power_recovery.sh [options] Options: --execute Actually run commands (default is dry-run) @@ -77,14 +77,14 @@ Options: -h, --help Show help Examples: - scripts/cluster_power_recovery.sh prepare --execute - scripts/cluster_power_recovery.sh bootstrap-seed --execute - scripts/cluster_power_recovery.sh harbor-seed --execute - scripts/cluster_power_recovery.sh longhorn-unlock --execute - scripts/cluster_power_recovery.sh flux-hold --execute - scripts/cluster_power_recovery.sh status - scripts/cluster_power_recovery.sh shutdown --execute - scripts/cluster_power_recovery.sh startup --execute --force-flux-branch main + scripts/ops/cluster_power_recovery.sh prepare --execute + scripts/ops/cluster_power_recovery.sh bootstrap-seed --execute + scripts/ops/cluster_power_recovery.sh harbor-seed --execute + scripts/ops/cluster_power_recovery.sh longhorn-unlock --execute + scripts/ops/cluster_power_recovery.sh flux-hold --execute + scripts/ops/cluster_power_recovery.sh status + scripts/ops/cluster_power_recovery.sh shutdown --execute + scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main USAGE } diff --git a/scripts/crypto_wallet_monero_setup.fish b/scripts/ops/crypto_wallet_monero_setup.fish old mode 100644 new mode 100755 similarity index 99% rename from scripts/crypto_wallet_monero_setup.fish rename to scripts/ops/crypto_wallet_monero_setup.fish index 0b1f43a1c..03f17ec33 --- a/scripts/crypto_wallet_monero_setup.fish +++ b/scripts/ops/crypto_wallet_monero_setup.fish @@ -1,3 +1,5 @@ +#!/usr/bin/env fish + ### ------- helpers --------------------------------------------------------- function _need --description "ensure a command exists" diff --git a/scripts/crypto_wallet_sui_setup.fish b/scripts/ops/crypto_wallet_sui_setup.fish old mode 100644 new mode 100755 similarity index 99% rename from scripts/crypto_wallet_sui_setup.fish rename to scripts/ops/crypto_wallet_sui_setup.fish index 09d81e1ba..28778d318 --- a/scripts/crypto_wallet_sui_setup.fish +++ b/scripts/ops/crypto_wallet_sui_setup.fish @@ -1,3 +1,5 @@ +#!/usr/bin/env fish + ### --------- helpers ---------- function _need --description "ensure a command exists" for c in $argv diff --git a/scripts/flux_hammer.fish b/scripts/ops/flux_hammer.fish similarity index 100% rename from scripts/flux_hammer.fish rename to scripts/ops/flux_hammer.fish diff --git a/scripts/jellyfin_manual_load.fish b/scripts/ops/jellyfin_manual_load.fish similarity index 95% rename from scripts/jellyfin_manual_load.fish rename to scripts/ops/jellyfin_manual_load.fish index b4919e4a2..b7f4f37b0 100755 --- a/scripts/jellyfin_manual_load.fish +++ b/scripts/ops/jellyfin_manual_load.fish @@ -4,10 +4,10 @@ # Fallback: kubectl port-forward to service (OK for small/medium files). # # Usage: -# scripts/jellyfin_manual_load.fish [REMOTE_SUBDIR] [JELLYFIN_API_KEY] +# scripts/ops/jellyfin_manual_load.fish [REMOTE_SUBDIR] [JELLYFIN_API_KEY] # Examples: -# scripts/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN" -# scripts/jellyfin_manual_load.fish "$HOME/Movies/." movies # copy contents-only into /media/movies +# scripts/ops/jellyfin_manual_load.fish "$HOME/Downloads/Avatar - The Last Airbender (2005 - 2008) [1080p]" kids_tv "$JELLYFIN_API_TOKEN" +# scripts/ops/jellyfin_manual_load.fish "$HOME/Movies/." movies # copy contents-only into /media/movies function usage echo "Usage: "(basename (status filename))" [REMOTE_SUBDIR] [JELLYFIN_API_KEY]" diff --git a/scripts/k3s_version_update.fish b/scripts/ops/k3s_version_update.fish old mode 100644 new mode 100755 similarity index 99% rename from scripts/k3s_version_update.fish rename to scripts/ops/k3s_version_update.fish index 0a5f7101f..dffe5c7c2 --- a/scripts/k3s_version_update.fish +++ b/scripts/ops/k3s_version_update.fish @@ -1,3 +1,5 @@ +#!/usr/bin/env fish + # Pick the correct K3s asset for a remote host (arm64 vs x86_64) function __k3s_asset_for_host set -l host $argv[1] diff --git a/scripts/longhorn_volume_usage.fish b/scripts/ops/longhorn_volume_usage.fish similarity index 100% rename from scripts/longhorn_volume_usage.fish rename to scripts/ops/longhorn_volume_usage.fish diff --git a/scripts/node_recover.sh b/scripts/ops/node_recover.sh similarity index 98% rename from scripts/node_recover.sh rename to scripts/ops/node_recover.sh index 44e656f32..c65958bae 100755 --- a/scripts/node_recover.sh +++ b/scripts/ops/node_recover.sh @@ -3,7 +3,7 @@ set -euo pipefail usage() { cat < [options] +Usage: scripts/ops/node_recover.sh [options] Options: --yes Skip confirmation prompt diff --git a/scripts/styx_kioskification.sh b/scripts/ops/styx_kioskification.sh old mode 100644 new mode 100755 similarity index 100% rename from scripts/styx_kioskification.sh rename to scripts/ops/styx_kioskification.sh diff --git a/scripts/styx_prep.sh b/scripts/ops/styx_prep.sh similarity index 100% rename from scripts/styx_prep.sh rename to scripts/ops/styx_prep.sh diff --git a/scripts/styx_prep_nvme_luks.sh b/scripts/ops/styx_prep_nvme_luks.sh similarity index 100% rename from scripts/styx_prep_nvme_luks.sh rename to scripts/ops/styx_prep_nvme_luks.sh diff --git a/scripts/verify_jenkins_workspace_cleanup_rollout.sh b/scripts/ops/verify_jenkins_workspace_cleanup_rollout.sh similarity index 100% rename from scripts/verify_jenkins_workspace_cleanup_rollout.sh rename to scripts/ops/verify_jenkins_workspace_cleanup_rollout.sh diff --git a/scripts/dashboards_render_atlas.py b/scripts/render/dashboards_render_atlas.py similarity index 97% rename from scripts/dashboards_render_atlas.py rename to scripts/render/dashboards_render_atlas.py index 1eece5ab3..67c6f7f00 100644 --- a/scripts/dashboards_render_atlas.py +++ b/scripts/render/dashboards_render_atlas.py @@ -2,8 +2,8 @@ """Generate Atlas Grafana dashboards and render them into ConfigMaps. Usage: - scripts/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps - scripts/dashboards_render_atlas.py # re-render ConfigMaps from JSON + scripts/render/dashboards_render_atlas.py --build # rebuild JSON + ConfigMaps + scripts/render/dashboards_render_atlas.py # re-render ConfigMaps from JSON """ import argparse @@ -16,7 +16,14 @@ from pathlib import Path # Paths, folders, and shared metadata # --------------------------------------------------------------------------- -ROOT = Path(__file__).resolve().parents[1] +def repo_root() -> Path: + for path in Path(__file__).resolve().parents: + if (path / "clusters").is_dir() and (path / "services").is_dir(): + return path + raise RuntimeError("could not locate repository root") # pragma: no cover + + +ROOT = repo_root() DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards" CONFIG_TEMPLATE = textwrap.dedent( """# {relative_path} @@ -428,22 +435,26 @@ STUCK_TERMINATING_EXPR = ( ) UPTIME_WINDOW = "365d" # vmalert precomputes the expensive long-window rollup so Grafana only reads one compact series. -UPTIME_RECORDING_METRIC = f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas"}}' -UPTIME_RECORDING_EXPR = f"last_over_time({UPTIME_RECORDING_METRIC}[24h])" +UPTIME_RECORDING_METRIC = ( + f'atlas:availability:ratio_{UPTIME_WINDOW}{{scope="atlas",definition="serving-v2"}}' +) TRAEFIK_READY_EXPR = ( "(" 'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})' - " / clamp_min(" - 'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)' - ")" + " > bool 0)" ) CONTROL_READY_FRACTION_EXPR = ( f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})" - f" / {CONTROL_TOTAL})" + " >= bool 2)" ) UPTIME_AVAIL_EXPR = ( f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))" ) +UPTIME_LIVE_FALLBACK_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[1h:5m])" +UPTIME_RECORDING_EXPR = ( + f"(last_over_time({UPTIME_RECORDING_METRIC}[24h]) " + f"or on() {UPTIME_LIVE_FALLBACK_EXPR})" +) # Tie-breaker to deterministically pick one node per namespace when shares tie. NODE_TIEBREAKER = " + ".join( @@ -620,7 +631,7 @@ PLATFORM_TEST_ALWAYS_REQUIRED_CHECK_REGEX = ( "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube" ) PLATFORM_TEST_STANDARD_CHECK_REGEX = ( - "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|supply_chain" + "tests|coverage|loc|style|docs_naming|gate_glue|sonarqube|semgrep|supply_chain" ) PLATFORM_TEST_SUPPLY_CHAIN_REQUIRED_SUITES = [ "ariadne", @@ -1870,7 +1881,7 @@ OVERVIEW_PANEL_DESCRIPTIONS = { "Control Plane Ready": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "Control Plane Workloads": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "Stuck Terminating": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", - "Atlas Availability (365d)": "Rolling one-year Atlas availability; higher is better, below target means users saw downtime.", + "Atlas Serving Availability": "Observed availability with control-plane quorum and at least one Traefik replica serving; partial replica capacity remains available.", "Problem Pods": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.", "CrashLoop / ImagePull": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.", "Workers Ready": "Worker nodes currently Ready; full count is good, lower means less place to run services.", @@ -1952,6 +1963,7 @@ TESTING_PANEL_DESCRIPTIONS = { "Style Failure Rate": "Percent of style checks currently failing; blue means style/docs gates pass.", "Gate Glue Failure Rate": "Percent of metric-contract checks failing; blue means dashboard telemetry is trustworthy.", "SonarQube Failure Rate": "Percent of Sonar checks failing; blue means Sonar quality gates pass.", + "Semgrep Failure Rate": "Percent of Semgrep checks failing; blue means SAST rule scans pass.", "Supply Chain Failure Rate": "Percent of supply-chain checks failing; blue means artifact/image checks pass.", "Check Healthy Rates By Suite": "Healthy percent by check family; blue means all selected checks are good.", "Tests Healthy Rate": "Percent of test checks passing or not applicable; higher is better.", @@ -1960,6 +1972,7 @@ TESTING_PANEL_DESCRIPTIONS = { "Style Healthy Rate": "Percent of style checks passing or not applicable; higher is better.", "Gate Glue Healthy Rate": "Percent of telemetry-contract checks passing; higher means cleaner reporting.", "SonarQube Healthy Rate": "Percent of Sonar checks passing or not applicable; higher is better.", + "Semgrep Healthy Rate": "Percent of Semgrep checks passing or not applicable; higher is better.", "Supply Chain Healthy Rate": "Percent of supply-chain checks passing or not applicable; higher is better.", "Test Drilldowns And Problem Tests": "Test-case detail for finding which tests are hurting reliability.", "Problematic Tests Over Time (Top failures)": "Current outlier tests by rolling 24h failures; tests need repeat failures to stay visible.", @@ -1976,6 +1989,13 @@ TESTING_PANEL_DESCRIPTIONS = { "Recent Branch Evidence by Suite (7d)": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.", "Primary Branch Clean by Suite (7d)": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.", "SonarQube Project Health": "SonarQube availability, projects, fetch errors, and gate status.", + "Public ZAP Baseline": "OWASP ZAP passive baseline results for externally addressed Atlas domains.", + "ZAP Targets Scanned": "Number of public hostnames with a latest OWASP ZAP baseline scan sample.", + "ZAP High/Medium Alerts": "Current high and medium ZAP alert instance count across public targets.", + "ZAP Scan Errors": "Targets whose latest ZAP baseline failed or did not produce a usable report.", + "ZAP Alerts by Risk": "Current ZAP alert distribution by risk level.", + "ZAP Target Health": "Latest ZAP health per hostname from passive baseline scans.", + "ZAP Alerts by Host": "Current high, medium, and low ZAP alert counts by public hostname.", "SonarQube API Up": "Whether the SonarQube exporter can reach SonarQube; 1 is good.", "Sonar Projects (Selected)": "Selected SonarQube project count; zero means Sonar is not tracking that suite.", "Sonar Gate Fetch Errors": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.", @@ -2103,7 +2123,7 @@ def build_overview(): }, { "id": 27, - "title": "Atlas Availability (365d)", + "title": "Atlas Serving Availability", "expr": UPTIME_PERCENT_EXPR, "kind": "stat", "thresholds": UPTIME_PERCENT_THRESHOLDS, @@ -2111,7 +2131,7 @@ def build_overview(): "decimals": 4, "text_mode": "value", "instant": True, - "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.", + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available.", }, { "id": 4, @@ -2335,10 +2355,10 @@ def build_overview(): } overview_avg_coverage = f"(avg(({QUALITY_GATE_COVERAGE_BY_SUITE})) or on() vector(0))" overview_category_health = ( - f'avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{' + f'(avg by (category) ({PLATFORM_TEST_CATEGORY_HEALTH_ROLLUP}{{' f'suite=~"{PLATFORM_TEST_SUITE_CANONICAL_MATCHER}",branch!="",branch=~"main|master|origin/main|origin/master",' f'category=~"{PLATFORM_TEST_OVERVIEW_CATEGORY_REGEX}"' - "})" + '})) or label_set(vector(0), "category", "none")' ) for panel_id, title, draw_expr, runtime_expr, y_pos in [ (40, "Pyrphoros UPS Current", ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_RUNTIME_DB, 7), @@ -4163,6 +4183,7 @@ def build_jobs_dashboard(): check_regex_style = "docs|naming|hygiene|lint|docs_naming|style" check_regex_gate_glue = "gate|glue|gate_glue" check_regex_sonarqube = "sonarqube|sonar" + check_regex_semgrep = "semgrep|sast" check_regex_supply_chain = "ironbank|supply_chain|image_compliance|artifact_security" def _check_state_percent_series(regex: str, failed: bool) -> str: @@ -4206,6 +4227,30 @@ def build_jobs_dashboard(): f"topk by (suite) (1, ({rollup_failed_tests_history}))" ) worst_test_per_suite = worst_test_per_suite_core + zap_target_health = ( + "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) " + "or on() vector(0))" + ) + zap_targets_scanned = ( + '(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job="platform-security-zap"})) ' + "or on() vector(0))" + ) + zap_high_medium_alerts = ( + '(sum(platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk=~"high|medium"}) ' + "or on() vector(0))" + ) + zap_scan_errors = ( + '(sum(platform_zap_baseline_scan_status{exported_job="platform-security-zap",status=~"fail|error"}) ' + "or on() vector(0))" + ) + zap_alerts_by_risk = ( + 'sum by (risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",risk!=""}) ' + "or on() vector(0)" + ) + zap_alerts_by_host_risk = ( + 'sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job="platform-security-zap",' + 'risk=~"high|medium|low"}) or on() vector(0)' + ) def _selected_status_volume(status: str) -> str: return ( @@ -4551,6 +4596,7 @@ def build_jobs_dashboard(): ("Style", check_regex_style), ("Gate Glue", check_regex_gate_glue), ("SonarQube", check_regex_sonarqube), + ("Semgrep", check_regex_semgrep), ("Supply Chain", check_regex_supply_chain), ] @@ -4574,12 +4620,12 @@ def build_jobs_dashboard(): ) panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW panels.append(panel) - for index, (label, regex) in enumerate(check_dimensions[4:]): + for index, (label, regex) in enumerate(check_dimensions[4:8]): panel = state_timeline_panel( start_id + 4 + index, f"{label} {title_prefix}", _check_state_percent_series(regex, failed), - {"h": 7, "w": 8, "x": index * 8, "y": y + 7}, + {"h": 7, "w": 6, "x": index * 6, "y": y + 7}, thresholds=trend_thresholds, description=trend_description, ) @@ -4587,7 +4633,7 @@ def build_jobs_dashboard(): panels.append(panel) _append_check_trends(130, "Failure Rate", True, 29) - _append_check_trends(138, "Healthy Rate", False, 43) + _append_check_trends(160, "Healthy Rate", False, 43) panels.append( state_timeline_panel( 145, @@ -4843,6 +4889,79 @@ def build_jobs_dashboard(): ) sonar_gate_project_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW panels.append(sonar_gate_project_panel) + panels.append( + stat_panel( + 168, + "ZAP Targets Scanned", + zap_targets_scanned, + {"h": 6, "w": 4, "x": 0, "y": 117}, + unit="none", + instant=True, + thresholds=missing_thresholds, + ) + ) + panels.append( + stat_panel( + 169, + "ZAP High/Medium Alerts", + zap_high_medium_alerts, + {"h": 6, "w": 4, "x": 4, "y": 117}, + unit="none", + instant=True, + thresholds=failures_thresholds, + ) + ) + panels.append( + stat_panel( + 170, + "ZAP Scan Errors", + zap_scan_errors, + {"h": 6, "w": 4, "x": 8, "y": 117}, + unit="none", + instant=True, + thresholds=failures_thresholds, + ) + ) + zap_risk_panel = pie_panel( + 171, + "ZAP Alerts by Risk", + zap_alerts_by_risk, + {"h": 6, "w": 4, "x": 12, "y": 117}, + ) + zap_risk_panel["targets"][0]["legendFormat"] = "{{risk}}" + panels.append(zap_risk_panel) + zap_target_panel = state_timeline_panel( + 172, + "ZAP Target Health", + zap_target_health, + {"h": 6, "w": 8, "x": 16, "y": 117}, + thresholds=success_thresholds, + unit="percent", + min_value=0, + max_value=100, + legend="{{host}}", + description=( + "Latest OWASP ZAP passive baseline health per public hostname. " + "High alerts and scan errors drop a target to zero; medium/low alerts mark warning health." + ), + ) + zap_target_panel["timeFrom"] = PLATFORM_TEST_HISTORY_WINDOW + panels.append(zap_target_panel) + panels.append( + bargauge_panel( + 173, + "ZAP Alerts by Host", + zap_alerts_by_host_risk, + {"h": 7, "w": 24, "x": 0, "y": 123}, + unit="none", + instant=True, + legend="{{host}} · {{risk}}", + sort_order="desc", + thresholds=failures_thresholds, + decimals=0, + limit=20, + ) + ) panels.append( bargauge_panel( 148, @@ -4952,6 +5071,12 @@ def build_jobs_dashboard(): 33: {"h": 6, "w": 4, "x": 8, "y": 111}, 34: {"h": 6, "w": 4, "x": 12, "y": 111}, 35: {"h": 6, "w": 8, "x": 16, "y": 111}, + 168: {"h": 6, "w": 4, "x": 0, "y": 117}, + 169: {"h": 6, "w": 4, "x": 4, "y": 117}, + 170: {"h": 6, "w": 4, "x": 8, "y": 117}, + 171: {"h": 6, "w": 4, "x": 12, "y": 117}, + 172: {"h": 6, "w": 8, "x": 16, "y": 117}, + 173: {"h": 7, "w": 24, "x": 0, "y": 123}, } for panel_id, grid in row_layout.items(): panel_by_id[panel_id]["gridPos"] = grid @@ -4963,13 +5088,13 @@ def build_jobs_dashboard(): 501, "Check Failure Rates By Suite", 12, - panels=children([130, 131, 132, 133, 134, 135, 136]), + panels=children([130, 131, 132, 133, 134, 135, 136, 137]), ), row_panel( 502, "Check Healthy Rates By Suite", 13, - panels=children([138, 139, 140, 141, 142, 143, 144]), + panels=children([160, 161, 162, 163, 164, 165, 166, 167]), ), row_panel( 503, @@ -4989,6 +5114,12 @@ def build_jobs_dashboard(): 16, panels=children([31, 32, 33, 34, 35]), ), + row_panel( + 506, + "Public ZAP Baseline", + 17, + panels=children([168, 169, 170, 171, 172, 173]), + ), ] ) panels = compact_panels diff --git a/scripts/dashboards_render_logs.py b/scripts/render/dashboards_render_logs.py similarity index 96% rename from scripts/dashboards_render_logs.py rename to scripts/render/dashboards_render_logs.py index 48b592d9b..ecb1144bd 100755 --- a/scripts/dashboards_render_logs.py +++ b/scripts/render/dashboards_render_logs.py @@ -2,8 +2,8 @@ """Generate OpenSearch Dashboards saved objects and render them into ConfigMaps. Usage: - scripts/dashboards_render_logs.py --build # rebuild NDJSON + ConfigMap - scripts/dashboards_render_logs.py # re-render ConfigMap from NDJSON + scripts/render/dashboards_render_logs.py --build # rebuild NDJSON + ConfigMap + scripts/render/dashboards_render_logs.py # re-render ConfigMap from NDJSON """ from __future__ import annotations @@ -14,14 +14,21 @@ import textwrap from dataclasses import dataclass from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] +def repo_root() -> Path: + for path in Path(__file__).resolve().parents: + if (path / "clusters").is_dir() and (path / "services").is_dir(): + return path + raise RuntimeError("could not locate repository root") # pragma: no cover + + +ROOT = repo_root() DASHBOARD_DIR = ROOT / "services" / "logging" / "dashboards" NDJSON_PATH = DASHBOARD_DIR / "logs.ndjson" CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-dashboards-objects.yaml" CONFIG_TEMPLATE = textwrap.dedent( """# {relative_path} -# Generated by scripts/dashboards_render_logs.py --build +# Generated by scripts/render/dashboards_render_logs.py --build apiVersion: v1 kind: ConfigMap metadata: diff --git a/scripts/render/flux_inventory.py b/scripts/render/flux_inventory.py new file mode 100755 index 000000000..7b0b575f7 --- /dev/null +++ b/scripts/render/flux_inventory.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +"""Print a compact inventory of Flux Kustomization resources.""" + +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Any + +import yaml + + +def _iter_docs(path: Path) -> list[dict[str, Any]]: + docs: list[dict[str, Any]] = [] + for raw in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if isinstance(raw, dict): + docs.append(raw) + return docs + + +def _is_flux_kustomization(doc: dict[str, Any]) -> bool: + return ( + doc.get("kind") == "Kustomization" + and str(doc.get("apiVersion") or "").startswith("kustomize.toolkit.fluxcd.io/") + ) + + +def _depends_on(spec: dict[str, Any]) -> str: + names = [] + for dep in spec.get("dependsOn") or []: + if isinstance(dep, dict) and dep.get("name"): + names.append(str(dep["name"])) + return ",".join(names) + + +def _row(doc: dict[str, Any], path: Path) -> dict[str, str]: + metadata = doc.get("metadata") or {} + spec = doc.get("spec") or {} + annotations = metadata.get("annotations") or {} + return { + "name": str(metadata.get("name") or ""), + "path": str(spec.get("path") or ""), + "namespace": str(spec.get("targetNamespace") or ""), + "depends": _depends_on(spec), + "prune": str(spec.get("prune", "")), + "wait": str(spec.get("wait", "")), + "suspend": str(spec.get("suspend", False)), + "reason": str(annotations.get("atlas.bstein.dev/suspend-reason") or ""), + "file": str(path), + } + + +def build_inventory(root: Path) -> list[dict[str, str]]: + rows: list[dict[str, str]] = [] + for path in sorted(root.rglob("*.yaml")): + for doc in _iter_docs(path): + if _is_flux_kustomization(doc): + rows.append(_row(doc, path)) + return sorted(rows, key=lambda item: item["name"]) + + +def _print_table(rows: list[dict[str, str]]) -> None: + columns = ["name", "path", "namespace", "depends", "prune", "wait", "suspend", "reason"] + widths = { + column: max([len(column), *[len(row[column]) for row in rows]]) for column in columns + } + header = " ".join(column.ljust(widths[column]) for column in columns) + print(header) + print(" ".join("-" * widths[column] for column in columns)) + for row in rows: + print(" ".join(row[column].ljust(widths[column]) for column in columns)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("root", nargs="?", default="clusters/atlas/flux-system") + args = parser.parse_args() + + rows = build_inventory(Path(args.root)) + _print_table(rows) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/knowledge_render_atlas.py b/scripts/render/knowledge_render_atlas.py similarity index 98% rename from scripts/knowledge_render_atlas.py rename to scripts/render/knowledge_render_atlas.py index 1e305cbbe..6e65e46b9 100644 --- a/scripts/knowledge_render_atlas.py +++ b/scripts/render/knowledge_render_atlas.py @@ -25,7 +25,14 @@ from typing import Any, Iterable import yaml -REPO_ROOT = Path(__file__).resolve().parents[1] +def repo_root() -> Path: + for path in Path(__file__).resolve().parents: + if (path / "clusters").is_dir() and (path / "services").is_dir(): + return path + raise RuntimeError("could not locate repository root") # pragma: no cover + + +REPO_ROOT = repo_root() DASHBOARD_DIR = REPO_ROOT / "services" / "monitoring" / "dashboards" CLUSTER_SCOPED_KINDS = { @@ -580,7 +587,7 @@ def main() -> int: catalog_rel = catalog_path.relative_to(REPO_ROOT).as_posix() catalog_path.write_text( f"# {catalog_rel}\n" - "# Generated by scripts/knowledge_render_atlas.py (do not edit by hand)\n" + "# Generated by scripts/render/knowledge_render_atlas.py (do not edit by hand)\n" + yaml.safe_dump(catalog, sort_keys=False), encoding="utf-8", ) diff --git a/scripts/logging_render_observability.py b/scripts/render/logging_render_observability.py similarity index 95% rename from scripts/logging_render_observability.py rename to scripts/render/logging_render_observability.py index 679e34090..b76c2dc45 100755 --- a/scripts/logging_render_observability.py +++ b/scripts/render/logging_render_observability.py @@ -2,8 +2,8 @@ """Generate OpenSearch Observability seed objects and render them into ConfigMaps. Usage: - scripts/logging_render_observability.py --build # rebuild JSON + ConfigMap - scripts/logging_render_observability.py # re-render ConfigMap from JSON + scripts/render/logging_render_observability.py --build # rebuild JSON + ConfigMap + scripts/render/logging_render_observability.py # re-render ConfigMap from JSON """ from __future__ import annotations @@ -14,7 +14,14 @@ import textwrap from dataclasses import dataclass from pathlib import Path -ROOT = Path(__file__).resolve().parents[1] +def repo_root() -> Path: + for path in Path(__file__).resolve().parents: + if (path / "clusters").is_dir() and (path / "services").is_dir(): + return path + raise RuntimeError("could not locate repository root") # pragma: no cover + + +ROOT = repo_root() OBS_DIR = ROOT / "services" / "logging" / "observability" APPS_PATH = OBS_DIR / "applications.json" QUERIES_PATH = OBS_DIR / "saved_queries.json" @@ -23,7 +30,7 @@ CONFIG_PATH = ROOT / "services" / "logging" / "opensearch-observability-objects. CONFIG_TEMPLATE = textwrap.dedent( """# {relative_path} -# Generated by scripts/logging_render_observability.py --build +# Generated by scripts/render/logging_render_observability.py --build apiVersion: v1 kind: ConfigMap metadata: diff --git a/scripts/monitoring_render_postmark_exporter.py b/scripts/render/monitoring_render_postmark_exporter.py similarity index 70% rename from scripts/monitoring_render_postmark_exporter.py rename to scripts/render/monitoring_render_postmark_exporter.py index b0a458a50..c841d7221 100644 --- a/scripts/monitoring_render_postmark_exporter.py +++ b/scripts/render/monitoring_render_postmark_exporter.py @@ -3,14 +3,21 @@ from pathlib import Path +def repo_root() -> Path: + for path in Path(__file__).resolve().parents: + if (path / "clusters").is_dir() and (path / "services").is_dir(): + return path + raise RuntimeError("could not locate repository root") # pragma: no cover + + def indent(text: str, spaces: int) -> str: prefix = " " * spaces return "".join(prefix + line if line.strip("\n") else line for line in text.splitlines(keepends=True)) def main() -> None: - root = Path(__file__).resolve().parents[1] - source = root / "scripts" / "monitoring_postmark_exporter.py" + root = repo_root() + source = root / "scripts" / "sync" / "monitoring_postmark_exporter.py" target = root / "services" / "monitoring" / "postmark-exporter-script.yaml" payload = source.read_text(encoding="utf-8") diff --git a/scripts/sync/comms_sync_kb.sh b/scripts/sync/comms_sync_kb.sh new file mode 100755 index 000000000..894f06b8f --- /dev/null +++ b/scripts/sync/comms_sync_kb.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -euo pipefail + +python scripts/render/knowledge_render_atlas.py --write +python scripts/render/knowledge_render_atlas.py --write --out services/comms/knowledge diff --git a/scripts/gitea_cred_sync.sh b/scripts/sync/gitea_cred_sync.sh similarity index 100% rename from scripts/gitea_cred_sync.sh rename to scripts/sync/gitea_cred_sync.sh diff --git a/scripts/gitops_cred_sync.sh b/scripts/sync/gitops_cred_sync.sh similarity index 100% rename from scripts/gitops_cred_sync.sh rename to scripts/sync/gitops_cred_sync.sh diff --git a/scripts/jenkins_cred_sync.sh b/scripts/sync/jenkins_cred_sync.sh similarity index 100% rename from scripts/jenkins_cred_sync.sh rename to scripts/sync/jenkins_cred_sync.sh diff --git a/scripts/monitoring_postmark_exporter.py b/scripts/sync/monitoring_postmark_exporter.py similarity index 100% rename from scripts/monitoring_postmark_exporter.py rename to scripts/sync/monitoring_postmark_exporter.py diff --git a/scripts/tests/test_dashboards_render_atlas.py b/scripts/tests/test_dashboards_render_atlas.py index 3740589e4..1e3b09c15 100644 --- a/scripts/tests/test_dashboards_render_atlas.py +++ b/scripts/tests/test_dashboards_render_atlas.py @@ -3,8 +3,10 @@ import pathlib def load_module(): - path = pathlib.Path(__file__).resolve().parents[1] / "dashboards_render_atlas.py" - spec = importlib.util.spec_from_file_location("scripts.dashboards_render_atlas", path) + path = pathlib.Path(__file__).resolve().parents[1] / "render/dashboards_render_atlas.py" + spec = importlib.util.spec_from_file_location( + "scripts.render.dashboards_render_atlas", path + ) module = importlib.util.module_from_spec(spec) assert spec.loader is not None spec.loader.exec_module(module) @@ -55,11 +57,20 @@ def test_overview_availability_panel_uses_recorded_365d_rollup(): dashboard = mod.build_overview() panel = next(panel for panel in flatten_panels(dashboard["panels"]) if panel["id"] == 27) - assert panel["title"] == "Atlas Availability (365d)" - assert panel["targets"][0]["expr"] == 'last_over_time(atlas:availability:ratio_365d{scope="atlas"}[24h])' + assert panel["title"] == "Atlas Serving Availability" + availability_expr = panel["targets"][0]["expr"] + assert ( + 'last_over_time(atlas:availability:ratio_365d{scope="atlas",definition="serving-v2"}[24h])' + in availability_expr + ) + assert ">= bool 2" in availability_expr + assert "> bool 0" in availability_expr + assert "[1h:5m]" in availability_expr assert panel["targets"][0]["instant"] is True - assert "precomputed" in panel["description"] - assert "last successful rollup for up to 24h" in panel["description"] + assert "at least one Traefik replica serving" in panel["description"] + assert "partial-capacity samples remain available" in panel["description"] + assert "monitoring gaps are excluded" in panel["description"] + assert "falls back to the live binary serving state" in panel["description"] def test_overview_uses_readable_quality_power_and_gitops_panels(): @@ -148,6 +159,7 @@ def test_overview_uses_readable_quality_power_and_gitops_panels(): assert panels_by_title["Test Category Health"]["options"]["showValue"] == "auto" assert panels_by_title["Test Category Health"]["options"]["rowHeight"] == 0.9 assert panels_by_title["Test Category Health"]["targets"][0]["legendFormat"] == "{{category}}" + assert 'label_set(vector(0), "category", "none")' in panels_by_title["Test Category Health"]["targets"][0]["expr"] assert not any(variable["name"] == "overview_suite" for variable in dashboard["templating"]["list"]) pvc_backup_expr = panels_by_title["PVC Backup Health / Age"]["targets"][0]["expr"] @@ -281,7 +293,7 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability(): suite_freshness_expr = panels_by_title["Suite Freshness (24h)"]["targets"][0]["expr"] assert "platform_quality:suite_runs:increase_24h" in suite_freshness_expr assert "max_over_time(platform_quality_gate_runs_total" not in suite_freshness_expr - assert "[30d:15m]" in panels_by_title["CI Run Success Rate (30d)"]["targets"][0]["expr"] + assert "[7d:1h]" in panels_by_title["CI Run Success Rate (7d)"]["targets"][0]["expr"] assert panels_by_title["Latest Gate Health by Suite"]["gridPos"]["w"] == 6 assert panels_by_title["CI Run Success by Suite (24h)"]["gridPos"]["w"] == 6 assert panels_by_title["Coverage by Suite (Latest, gate 95)"]["gridPos"] == {"h": 7, "w": 6, "x": 12, "y": 4} @@ -300,7 +312,7 @@ def test_jobs_dashboard_separates_current_gate_health_from_reliability(): rolling_panel = panels_by_title["CI Run Success by Suite (7d rolling)"] assert rolling_panel["type"] == "state-timeline" - assert "[7d:1m]" in rolling_panel["targets"][0]["expr"] + assert "[7d:1h]" in rolling_panel["targets"][0]["expr"] category_panel = panels_by_title["Test Category Health History"] assert category_panel["type"] == "state-timeline" assert "category" in category_panel["targets"][0]["expr"] @@ -384,7 +396,7 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): for child in row.get("panels", []) } - assert len(panels) == 18 + assert len(panels) == 19 assert len(visible_query_panels) == 12 assert sum(len(panel.get("targets", [])) for panel in visible_query_panels) == 12 assert all( @@ -398,15 +410,18 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): "Test Drilldowns And Problem Tests", "Telemetry Completeness And Branches", "SonarQube Project Health", + "Public ZAP Baseline", ] assert all(row["collapsed"] for row in rows) assert "Coverage Failure Rate" in nested_panels_by_title + assert "Semgrep Failure Rate" in nested_panels_by_title assert "Supply Chain Healthy Rate" in nested_panels_by_title assert "Test Category Health History" in nested_panels_by_title assert "Selected Test Pass Rate History" in nested_panels_by_title assert "Coverage Metrics Present by Suite" in nested_panels_by_title assert "SonarQube API Up" in nested_panels_by_title + assert "ZAP Target Health" in nested_panels_by_title failure_rate_panel = nested_panels_by_title["Coverage Failure Rate"] assert failure_rate_panel["type"] == "state-timeline" @@ -457,13 +472,22 @@ def test_jobs_dashboard_collapses_heavy_drilldowns_for_light_first_paint(): assert sonar_health_panel["type"] == "state-timeline" assert "platform_quality:sonar_gate_health_percent:latest_1h" in sonar_health_panel["targets"][0]["expr"] assert "sonarqube_project_quality_gate_pass" not in sonar_health_panel["targets"][0]["expr"] + semgrep_panel = nested_panels_by_title["Semgrep Healthy Rate"] + assert semgrep_panel["type"] == "state-timeline" + assert 'check=~"semgrep|sast"' in semgrep_panel["targets"][0]["expr"] - branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (30d)"] - recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (30d)"] + zap_health_panel = nested_panels_by_title["ZAP Target Health"] + assert zap_health_panel["type"] == "state-timeline" + assert "platform_zap_baseline_target_health_percent" in zap_health_panel["targets"][0]["expr"] + zap_alerts_panel = nested_panels_by_title["ZAP Alerts by Host"] + assert "platform_zap_baseline_alerts_total" in zap_alerts_panel["targets"][0]["expr"] + + branch_panel = nested_panels_by_title["Primary Branch Clean by Suite (7d)"] + recent_branch_panel = nested_panels_by_title["Recent Branch Evidence by Suite (7d)"] assert branch_panel["gridPos"]["x"] == 12 assert recent_branch_panel["gridPos"]["x"] == 18 - assert "[30d:15m]" in recent_branch_panel["targets"][0]["expr"] - assert "[30d:15m]" in branch_panel["targets"][0]["expr"] + assert "[7d:1h]" in recent_branch_panel["targets"][0]["expr"] + assert "[7d:1h]" in branch_panel["targets"][0]["expr"] assert branch_panel["fieldConfig"]["defaults"]["unit"] == "percent" assert "unless on(suite)" in branch_panel["targets"][0]["expr"] assert "> bool 0" in branch_panel["targets"][0]["expr"] diff --git a/scripts/tests/test_monitoring_query_capacity.py b/scripts/tests/test_monitoring_query_capacity.py new file mode 100644 index 000000000..714679910 --- /dev/null +++ b/scripts/tests/test_monitoring_query_capacity.py @@ -0,0 +1,106 @@ +"""Protect the monitoring backend from known query-starvation regressions.""" + +from pathlib import Path + +import yaml + + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _documents(path: Path) -> list[dict]: + """Load every non-empty YAML document from a repository manifest.""" + return [document for document in yaml.safe_load_all(path.read_text()) if document] + + +def test_victoria_metrics_has_dashboard_burst_headroom() -> None: + """Keep search capacity and pod resources above the proven failure floor.""" + manifests = _documents(REPO_ROOT / "services/monitoring/helmrelease.yaml") + release = next( + manifest + for manifest in manifests + if manifest.get("kind") == "HelmRelease" + and manifest.get("metadata", {}).get("name") == "victoria-metrics-single" + ) + server = release["spec"]["values"]["server"] + + assert int(server["extraArgs"]["search.maxConcurrentRequests"]) >= 4 + assert server["extraArgs"]["search.maxQueryDuration"] == "1m" + assert server["extraArgs"]["search.maxQueueDuration"] == "30s" + assert server["resources"]["requests"]["memory"] == "2Gi" + assert server["resources"]["limits"]["cpu"] == "2" + assert server["resources"]["limits"]["memory"] == "4Gi" + + required_terms = server["affinity"]["nodeAffinity"][ + "requiredDuringSchedulingIgnoredDuringExecution" + ]["nodeSelectorTerms"] + hostname_rule = next( + expression + for term in required_terms + for expression in term["matchExpressions"] + if expression["key"] == "kubernetes.io/hostname" + ) + assert hostname_rule["operator"] == "NotIn" + assert {"titan-14", "titan-18"} <= set(hostname_rule["values"]) + + +def test_yearly_availability_reuses_the_hourly_rollup() -> None: + """Prevent the yearly rule from rescanning raw cluster metrics for 365 days.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] + yearly = next( + rule + for rule in rules["rules"] + if rule["record"] == "atlas:availability:ratio_365d" + ) + + assert "atlas:availability:ratio_1h" in yearly["expr"] + assert 'definition="serving-v2"' in yearly["expr"] + assert 'definition=""' in yearly["expr"] + assert "share_gt_over_time" in yearly["expr"] + assert "[365d:15m]" in yearly["expr"] + assert "kube_node_status_condition" not in yearly["expr"] + assert "[365d:1h]" not in yearly["expr"] + + +def test_availability_counts_serving_state_instead_of_replica_capacity() -> None: + """Treat quorum and any serving ingress replica as available binary states.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + rules = yaml.safe_load(manifest["data"]["atlas-availability.yaml"])["groups"][0] + hourly = next( + rule + for rule in rules["rules"] + if rule["record"] == "atlas:availability:ratio_1h" + ) + + assert ">= bool 2" in hourly["expr"] + assert "> bool 0" in hourly["expr"] + assert "/ 3" not in hourly["expr"] + assert hourly["labels"]["definition"] == "serving-v2" + + +def test_quality_rollups_do_not_run_every_minute() -> None: + """Keep high-cardinality quality rollups below the backend saturation cadence.""" + manifest = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + )[0] + quality = yaml.safe_load(manifest["data"]["platform-quality.yaml"])["groups"][0] + + assert quality["interval"] == "5m" + + +def test_vmalert_reloads_updated_rule_files() -> None: + """Make Flux ConfigMap updates take effect without manual pod revision bumps.""" + manifests = _documents( + REPO_ROOT / "services/monitoring/vmalert-atlas-availability.yaml" + ) + deployment = next( + manifest for manifest in manifests if manifest.get("kind") == "Deployment" + ) + args = deployment["spec"]["template"]["spec"]["containers"][0]["args"] + + assert "-configCheckInterval=30s" in args diff --git a/services/bstein-dev-home/kustomization.yaml b/services/bstein-dev-home/kustomization.yaml index 21d6b79e3..a765a5fff 100644 --- a/services/bstein-dev-home/kustomization.yaml +++ b/services/bstein-dev-home/kustomization.yaml @@ -16,13 +16,13 @@ resources: - backend-deployment.yaml - backend-service.yaml - vaultwarden-cred-sync-cronjob.yaml - - oneoffs/portal-onboarding-e2e-test-job.yaml + - validation-jobs/portal-onboarding-e2e-test-job.yaml - ingress.yaml images: - name: registry.bstein.dev/bstein/bstein-dev-home-frontend - newTag: 0.1.1-413 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"} + newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-frontend:tag"} - name: registry.bstein.dev/bstein/bstein-dev-home-backend - newTag: 0.1.1-413 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"} + newTag: 0.1.1-414 # {"$imagepolicy": "bstein-dev-home:bstein-dev-home-backend:tag"} configMapGenerator: - name: chat-ai-gateway namespace: bstein-dev-home diff --git a/services/bstein-dev-home/oneoffs/migrations/kustomization.yaml b/services/bstein-dev-home/migration-jobs/kustomization.yaml similarity index 66% rename from services/bstein-dev-home/oneoffs/migrations/kustomization.yaml rename to services/bstein-dev-home/migration-jobs/kustomization.yaml index 1d1dfc827..541bd0782 100644 --- a/services/bstein-dev-home/oneoffs/migrations/kustomization.yaml +++ b/services/bstein-dev-home/migration-jobs/kustomization.yaml @@ -1,4 +1,4 @@ -# services/bstein-dev-home/oneoffs/migrations/kustomization.yaml +# services/bstein-dev-home/migration-jobs/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: bstein-dev-home diff --git a/services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml b/services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml similarity index 96% rename from services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml rename to services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml index 1f7e092be..d36fbc0c0 100644 --- a/services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml +++ b/services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml @@ -1,4 +1,4 @@ -# services/bstein-dev-home/oneoffs/migrations/portal-migrate-job.yaml +# services/bstein-dev-home/migration-jobs/portal-migrate-job.yaml # One-off job for bstein-dev-home/bstein-dev-home-portal-migrate-36. # Purpose: bstein dev home portal migrate 36 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml b/services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml similarity index 97% rename from services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml rename to services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml index 5c800385b..63731c92c 100644 --- a/services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml +++ b/services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml @@ -1,4 +1,4 @@ -# services/bstein-dev-home/oneoffs/portal-onboarding-e2e-test-job.yaml +# services/bstein-dev-home/validation-jobs/portal-onboarding-e2e-test-job.yaml # One-off job for bstein-dev-home/portal-onboarding-e2e-test-27. # Purpose: portal onboarding e2e test 27 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/comms-secrets-ensure-job.yaml b/services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml similarity index 98% rename from services/comms/oneoffs/comms-secrets-ensure-job.yaml rename to services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml index 35ca73c52..dedac2563 100644 --- a/services/comms/oneoffs/comms-secrets-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/comms-secrets-ensure-job.yaml +# services/comms/bootstrap-jobs/comms-secrets-ensure-job.yaml # One-off job for comms/comms-secrets-ensure-7. # Purpose: comms secrets ensure 7 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml b/services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml similarity index 98% rename from services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml rename to services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml index 7c7d0c87a..3be852f1c 100644 --- a/services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/mas-admin-client-secret-ensure-job.yaml +# services/comms/bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml # One-off job for comms/mas-admin-client-secret-writer. # Purpose: mas admin client secret writer (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/mas-db-ensure-job.yaml b/services/comms/bootstrap-jobs/mas-db-ensure-job.yaml similarity index 98% rename from services/comms/oneoffs/mas-db-ensure-job.yaml rename to services/comms/bootstrap-jobs/mas-db-ensure-job.yaml index 44137da8f..e6b5eaa83 100644 --- a/services/comms/oneoffs/mas-db-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/mas-db-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/mas-db-ensure-job.yaml +# services/comms/bootstrap-jobs/mas-db-ensure-job.yaml # One-off job for comms/mas-db-ensure-22. # Purpose: mas db ensure 22 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/mas-local-users-ensure-job.yaml b/services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml similarity index 99% rename from services/comms/oneoffs/mas-local-users-ensure-job.yaml rename to services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml index 7b5107278..a40f4183f 100644 --- a/services/comms/oneoffs/mas-local-users-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/mas-local-users-ensure-job.yaml +# services/comms/bootstrap-jobs/mas-local-users-ensure-job.yaml # One-off job for comms/mas-local-users-ensure-18. # Purpose: mas local users ensure 18 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/synapse-admin-ensure-job.yaml b/services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml similarity index 99% rename from services/comms/oneoffs/synapse-admin-ensure-job.yaml rename to services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml index 95bc9f2ab..1c30463f4 100644 --- a/services/comms/oneoffs/synapse-admin-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/synapse-admin-ensure-job.yaml +# services/comms/bootstrap-jobs/synapse-admin-ensure-job.yaml # One-off job for comms/synapse-admin-ensure-3. # Purpose: synapse admin ensure 3 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml b/services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml similarity index 98% rename from services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml rename to services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml index 29c6ec52c..9405744c5 100644 --- a/services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/synapse-seeder-admin-ensure-job.yaml +# services/comms/bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml # One-off job for comms/synapse-seeder-admin-ensure-9. # Purpose: synapse seeder admin ensure 9 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/synapse-signingkey-ensure-job.yaml b/services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml similarity index 97% rename from services/comms/oneoffs/synapse-signingkey-ensure-job.yaml rename to services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml index d97c59874..480dc94ad 100644 --- a/services/comms/oneoffs/synapse-signingkey-ensure-job.yaml +++ b/services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/synapse-signingkey-ensure-job.yaml +# services/comms/bootstrap-jobs/synapse-signingkey-ensure-job.yaml # One-off job for comms/othrys-synapse-signingkey-ensure-7. # Purpose: othrys synapse signingkey ensure 7 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/synapse-user-seed-job.yaml b/services/comms/bootstrap-jobs/synapse-user-seed-job.yaml similarity index 99% rename from services/comms/oneoffs/synapse-user-seed-job.yaml rename to services/comms/bootstrap-jobs/synapse-user-seed-job.yaml index 714658632..bda97f830 100644 --- a/services/comms/oneoffs/synapse-user-seed-job.yaml +++ b/services/comms/bootstrap-jobs/synapse-user-seed-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/synapse-user-seed-job.yaml +# services/comms/bootstrap-jobs/synapse-user-seed-job.yaml # One-off job for comms/synapse-user-seed-9. # Purpose: synapse user seed 9 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/knowledge/INDEX.md b/services/comms/knowledge/INDEX.md index fac915330..28c9b4ebb 100644 --- a/services/comms/knowledge/INDEX.md +++ b/services/comms/knowledge/INDEX.md @@ -12,7 +12,7 @@ Layout Regeneration - Update manifests/docs, then regenerate generated artifacts: - - `python scripts/knowledge_render_atlas.py --write` + - `python scripts/render/knowledge_render_atlas.py --write` Authoring rules - Never include secret values. Prefer `secretRef` names or Vault paths like `kv/atlas/...`. diff --git a/services/comms/knowledge/catalog/atlas-summary.json b/services/comms/knowledge/catalog/atlas-summary.json index ea825ce74..f56760663 100644 --- a/services/comms/knowledge/catalog/atlas-summary.json +++ b/services/comms/knowledge/catalog/atlas-summary.json @@ -1,8 +1,8 @@ { "counts": { - "helmrelease_host_hints": 19, - "http_endpoints": 45, - "services": 47, - "workloads": 74 + "helmrelease_host_hints": 22, + "http_endpoints": 54, + "services": 70, + "workloads": 100 } } diff --git a/services/comms/knowledge/catalog/atlas.json b/services/comms/knowledge/catalog/atlas.json index 951c80796..daae83d58 100644 --- a/services/comms/knowledge/catalog/atlas.json +++ b/services/comms/knowledge/catalog/atlas.json @@ -13,7 +13,7 @@ }, { "name": "bstein-dev-home-migrations", - "path": "services/bstein-dev-home/migrations", + "path": "services/bstein-dev-home/migration-jobs", "targetNamespace": "bstein-dev-home" }, { @@ -41,6 +41,11 @@ "path": "services/crypto", "targetNamespace": "crypto" }, + { + "name": "descheduler", + "path": "infrastructure/descheduler", + "targetNamespace": "kube-system" + }, { "name": "finance", "path": "services/finance", @@ -51,6 +56,11 @@ "path": "clusters/atlas/flux-system", "targetNamespace": null }, + { + "name": "game-stream", + "path": "services/game-stream", + "targetNamespace": "game-stream" + }, { "name": "gitea", "path": "services/gitea", @@ -76,6 +86,11 @@ "path": "infrastructure/sources/helm", "targetNamespace": "flux-system" }, + { + "name": "hermes", + "path": "services/hermes", + "targetNamespace": "hermes" + }, { "name": "jellyfin", "path": "services/jellyfin", @@ -151,6 +166,11 @@ "path": "services/oauth2-proxy", "targetNamespace": "sso" }, + { + "name": "openclaw", + "path": "services/openclaw", + "targetNamespace": "openclaw" + }, { "name": "openldap", "path": "services/openldap", @@ -176,6 +196,16 @@ "path": "infrastructure/postgres", "targetNamespace": "postgres" }, + { + "name": "quality", + "path": "services/quality", + "targetNamespace": "quality" + }, + { + "name": "resource-guardrails", + "path": "infrastructure/resource-guardrails", + "targetNamespace": null + }, { "name": "sui-metrics", "path": "services/sui-metrics/overlays/atlas", @@ -186,6 +216,11 @@ "path": "infrastructure/traefik", "targetNamespace": "traefik" }, + { + "name": "typhon", + "path": "services/typhon", + "targetNamespace": "climate" + }, { "name": "vault", "path": "services/vault", @@ -206,6 +241,11 @@ "path": "services/vaultwarden", "targetNamespace": "vaultwarden" }, + { + "name": "veles", + "path": "services/veles", + "targetNamespace": "veles" + }, { "name": "wallet-monero-temp", "path": "services/crypto/wallet-monero-temp", @@ -244,7 +284,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-157" + "registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-414" ] }, { @@ -260,7 +300,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-157" + "registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-414" ] }, { @@ -292,6 +332,35 @@ "python:3.11-slim" ] }, + { + "kind": "Deployment", + "namespace": "climate", + "name": "typhon", + "labels": { + "app": "typhon" + }, + "serviceAccountName": "typhon", + "nodeSelector": { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "registry.bstein.dev/bstein/typhon:main" + ] + }, + { + "kind": "Deployment", + "namespace": "climate", + "name": "typhon-vault-sync", + "labels": { + "app": "typhon-vault-sync" + }, + "serviceAccountName": "typhon", + "nodeSelector": {}, + "images": [ + "alpine:3.20" + ] + }, { "kind": "Deployment", "namespace": "comms", @@ -301,7 +370,8 @@ }, "serviceAccountName": "atlasbot", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "python:3.11-slim" @@ -329,7 +399,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "ghcr.io/coturn/coturn:4.6.2" @@ -344,7 +415,8 @@ }, "serviceAccountName": null, "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "ghcr.io/element-hq/element-call@sha256:e6897c7818331714eae19d83ef8ea94a8b41115f0d8d3f62c2fed2d02c65c9bc" @@ -359,7 +431,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "livekit/livekit-server:v1.9.0" @@ -374,7 +447,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "registry.bstein.dev/tools/lk-jwt-service-vault:0.3.0" @@ -389,7 +463,8 @@ }, "serviceAccountName": "comms-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "ghcr.io/element-hq/matrix-authentication-service:1.8.0" @@ -416,7 +491,9 @@ "app": "matrix-wellknown" }, "serviceAccountName": null, - "nodeSelector": {}, + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, "images": [ "nginx:1.27-alpine" ] @@ -426,14 +503,16 @@ "namespace": "crypto", "name": "monero-xmrig", "labels": { - "app": "monero-xmrig" + "app": "monero-xmrig", + "atlas.bstein.dev/workload-profile": "scavenger" }, "serviceAccountName": null, "nodeSelector": { + "atlas.bstein.dev/crypto-mining-enabled": "true", "node-role.kubernetes.io/worker": "true" }, "images": [ - "ghcr.io/tari-project/xmrig@sha256:80defbfd0b640d604c91cb5101d3642db7928e1e68ee3c6b011289b3565a39d9" + "ghcr.io/tari-project/xmrig@sha256:d590a41613fea974f155280920095ea10c3710f55ecf16fc38fd3a1c18718129" ] }, { @@ -454,7 +533,8 @@ "namespace": "crypto", "name": "monero-p2pool", "labels": { - "app": "monero-p2pool" + "app": "monero-p2pool", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": "crypto-vault-sync", "nodeSelector": { @@ -469,13 +549,15 @@ "namespace": "crypto", "name": "monerod", "labels": { - "app": "monerod" + "app": "monerod", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": null, "nodeSelector": { "node-role.kubernetes.io/worker": "true" }, "images": [ + "python:3.11-alpine", "registry.bstein.dev/crypto/monerod:0.18.4.1" ] }, @@ -640,12 +722,60 @@ "ghcr.io/fluxcd/source-controller:v1.7.4" ] }, + { + "kind": "DaemonSet", + "namespace": "game-stream", + "name": "wolf-gatekeeper", + "labels": { + "app": "wolf-gatekeeper" + }, + "serviceAccountName": null, + "nodeSelector": { + "kubernetes.io/hostname": "titan-24" + }, + "images": [ + "ghcr.io/games-on-whales/wolf:stable" + ] + }, + { + "kind": "Deployment", + "namespace": "game-stream", + "name": "oauth2-proxy-wolf", + "labels": { + "app": "oauth2-proxy-wolf" + }, + "serviceAccountName": "game-stream-vault", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "StatefulSet", + "namespace": "game-stream", + "name": "wolf", + "labels": { + "app": "wolf", + "atlas.bstein.dev/game-mode": "wolf" + }, + "serviceAccountName": null, + "nodeSelector": { + "kubernetes.io/hostname": "titan-24" + }, + "images": [ + "ghcr.io/games-on-whales/wolf:stable", + "ghcr.io/games-on-whales/wolfmanager/wolfmanager:latest" + ] + }, { "kind": "Deployment", "namespace": "gitea", "name": "gitea", "labels": { - "app": "gitea" + "app": "gitea", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": "gitea-vault", "nodeSelector": { @@ -685,12 +815,39 @@ "wger/server@sha256:710588b78af4e0aa0b4d8a8061e4563e16eae80eeaccfe7f9e0d9cbdd7f0cbc5" ] }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes", + "labels": { + "app": "hermes" + }, + "serviceAccountName": "hermes-triage", + "nodeSelector": {}, + "images": [ + "nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973" + ] + }, + { + "kind": "Deployment", + "namespace": "hermes", + "name": "hermes-ollama", + "labels": { + "app": "hermes-ollama" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d" + ] + }, { "kind": "Deployment", "namespace": "jellyfin", "name": "jellyfin", "labels": { - "app": "jellyfin" + "app": "jellyfin", + "atlas.bstein.dev/workload-profile": "heavy" }, "serviceAccountName": "pegasus-vault-sync", "nodeSelector": {}, @@ -723,7 +880,9 @@ "app": "pegasus-vault-sync" }, "serviceAccountName": "pegasus-vault-sync", - "nodeSelector": {}, + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, "images": [ "alpine:3.20" ] @@ -851,7 +1010,7 @@ "kubernetes.io/os": "linux" }, "images": [ - "registry.bstein.dev/infra/coredns:1.12.1" + "registry.k8s.io/coredns/coredns:v1.12.1" ] }, { @@ -893,7 +1052,7 @@ }, "serviceAccountName": "node-log-rotation", "nodeSelector": { - "hardware": "rpi5" + "node-role.kubernetes.io/worker": "true" }, "images": [ "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" @@ -1016,6 +1175,38 @@ "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" ] }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "metis-sentinel-amd64", + "labels": { + "app": "metis-sentinel-amd64" + }, + "serviceAccountName": "metis", + "nodeSelector": { + "kubernetes.io/arch": "amd64", + "kubernetes.io/os": "linux" + }, + "images": [ + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64" + ] + }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "metis-sentinel-arm64", + "labels": { + "app": "metis-sentinel-arm64" + }, + "serviceAccountName": "metis", + "nodeSelector": { + "kubernetes.io/arch": "arm64", + "kubernetes.io/os": "linux" + }, + "images": [ + "registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64" + ] + }, { "kind": "DaemonSet", "namespace": "maintenance", @@ -1044,6 +1235,51 @@ "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" ] }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "rpi-resource-reservation", + "labels": { + "app": "rpi-resource-reservation" + }, + "serviceAccountName": "node-nofile", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" + ] + }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "titan-22-link-keeper", + "labels": { + "app": "titan-22-link-keeper" + }, + "serviceAccountName": null, + "nodeSelector": { + "kubernetes.io/hostname": "titan-22" + }, + "images": [ + "bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131" + ] + }, + { + "kind": "DaemonSet", + "namespace": "maintenance", + "name": "titan-24-docker", + "labels": { + "app": "titan-24-docker" + }, + "serviceAccountName": "titan-24-docker", + "nodeSelector": { + "kubernetes.io/hostname": "titan-24" + }, + "images": [ + "debian:13-slim" + ] + }, { "kind": "Deployment", "namespace": "maintenance", @@ -1057,7 +1293,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "registry.bstein.dev/bstein/ariadne:0.1.0-49" + "registry.bstein.dev/bstein/ariadne:0.1.0-356" ] }, { @@ -1068,11 +1304,72 @@ "app": "maintenance-vault-sync" }, "serviceAccountName": "maintenance-vault-sync", - "nodeSelector": {}, + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, "images": [ "alpine:3.20" ] }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "metis", + "labels": { + "app": "metis" + }, + "serviceAccountName": "metis", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/bstein/metis:0.1.0-243-arm64" + ] + }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "oauth2-proxy-metis", + "labels": { + "app": "oauth2-proxy-metis" + }, + "serviceAccountName": "maintenance-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "oauth2-proxy-soteria", + "labels": { + "app": "oauth2-proxy-soteria" + }, + "serviceAccountName": "maintenance-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "Deployment", + "namespace": "maintenance", + "name": "soteria", + "labels": { + "app": "soteria" + }, + "serviceAccountName": "soteria", + "nodeSelector": { + "kubernetes.io/arch": "arm64", + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "registry.bstein.dev/bstein/soteria:0.1.0-120" + ] + }, { "kind": "DaemonSet", "namespace": "monitoring", @@ -1101,6 +1398,19 @@ "python:3.10-slim" ] }, + { + "kind": "DaemonSet", + "namespace": "monitoring", + "name": "nvidia-process-exporter", + "labels": { + "app": "nvidia-process-exporter" + }, + "serviceAccountName": "nvidia-process-exporter", + "nodeSelector": {}, + "images": [ + "python:3.12-slim" + ] + }, { "kind": "Deployment", "namespace": "monitoring", @@ -1114,6 +1424,19 @@ "alpine:3.20" ] }, + { + "kind": "Deployment", + "namespace": "monitoring", + "name": "platform-quality-gateway", + "labels": { + "app": "platform-quality-gateway" + }, + "serviceAccountName": null, + "nodeSelector": {}, + "images": [ + "prom/pushgateway:v1.11.2" + ] + }, { "kind": "Deployment", "namespace": "monitoring", @@ -1127,6 +1450,19 @@ "python:3.12-alpine" ] }, + { + "kind": "Deployment", + "namespace": "monitoring", + "name": "vmalert-atlas-availability", + "labels": { + "app": "vmalert-atlas-availability" + }, + "serviceAccountName": "vmalert-atlas-availability", + "nodeSelector": {}, + "images": [ + "victoriametrics/vmalert:v1.113.0" + ] + }, { "kind": "Deployment", "namespace": "nextcloud", @@ -1136,7 +1472,7 @@ }, "serviceAccountName": null, "nodeSelector": { - "hardware": "rpi5" + "kubernetes.io/arch": "amd64" }, "images": [ "collabora/code@sha256:3c58d0e9bae75e4647467d0c7d91cb66f261d3e814709aed590b5c334a04db26" @@ -1151,7 +1487,8 @@ }, "serviceAccountName": "nextcloud-vault", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "nextcloud:29-apache" @@ -1211,6 +1548,7 @@ }, "serviceAccountName": "postgres-vault", "nodeSelector": { + "hardware": "rpi5", "node-role.kubernetes.io/worker": "true" }, "images": [ @@ -1218,6 +1556,51 @@ "quay.io/prometheuscommunity/postgres-exporter:v0.15.0" ] }, + { + "kind": "Deployment", + "namespace": "quality", + "name": "oauth2-proxy-sonarqube", + "labels": { + "app": "oauth2-proxy-sonarqube" + }, + "serviceAccountName": "quality-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "quay.io/oauth2-proxy/oauth2-proxy:v7.6.0" + ] + }, + { + "kind": "Deployment", + "namespace": "quality", + "name": "sonarqube", + "labels": { + "app": "sonarqube" + }, + "serviceAccountName": "sonarqube", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "sonarqube:lts-community" + ] + }, + { + "kind": "Deployment", + "namespace": "quality", + "name": "sonarqube-exporter", + "labels": { + "app": "sonarqube-exporter" + }, + "serviceAccountName": "quality-vault-sync", + "nodeSelector": { + "node-role.kubernetes.io/worker": "true" + }, + "images": [ + "registry.bstein.dev/bstein/python:3.12-slim" + ] + }, { "kind": "Deployment", "namespace": "sso", @@ -1287,7 +1670,8 @@ }, "serviceAccountName": "sui-metrics", "nodeSelector": { - "hardware": "rpi5" + "hardware": "rpi5", + "node-role.kubernetes.io/worker": "true" }, "images": [ "victoriametrics/vmagent:v1.103.0" @@ -1302,7 +1686,7 @@ "app.kubernetes.io/instance": "traefik-kube-system", "app.kubernetes.io/name": "traefik" }, - "serviceAccountName": "traefik-ingress-controller", + "serviceAccountName": "atlas-traefik-ingress-controller", "nodeSelector": { "node-role.kubernetes.io/worker": "true" }, @@ -1323,7 +1707,7 @@ "node-role.kubernetes.io/worker": "true" }, "images": [ - "hashicorp/vault:1.17.6" + "hashicorp/vault:1.21.4" ] }, { @@ -1335,12 +1719,70 @@ }, "serviceAccountName": "vaultwarden-vault", "nodeSelector": { + "hardware": "rpi5", "kubernetes.io/arch": "arm64", "node-role.kubernetes.io/worker": "true" }, "images": [ "vaultwarden/server:1.35.2" ] + }, + { + "kind": "Deployment", + "namespace": "veles", + "name": "veles-backend", + "labels": { + "app": "veles-backend" + }, + "serviceAccountName": "veles-backend", + "nodeSelector": { + "kubernetes.io/arch": "amd64", + "veles.bstein.dev/node-pool": "oceanus" + }, + "images": [ + "registry.bstein.dev/veles/veles-backend:0.7.56" + ] + }, + { + "kind": "Deployment", + "namespace": "veles", + "name": "veles-frontend", + "labels": { + "app": "veles-frontend" + }, + "serviceAccountName": "veles-frontend", + "nodeSelector": {}, + "images": [ + "registry.bstein.dev/veles/veles-frontend:0.7.56" + ] + }, + { + "kind": "Deployment", + "namespace": "veles", + "name": "veles-vault-sync", + "labels": { + "app": "veles-vault-sync" + }, + "serviceAccountName": "veles-vault-sync", + "nodeSelector": {}, + "images": [ + "alpine:3.20" + ] + }, + { + "kind": "StatefulSet", + "namespace": "veles", + "name": "veles-postgres", + "labels": { + "app": "veles-postgres" + }, + "serviceAccountName": "veles-postgres", + "nodeSelector": { + "veles.bstein.dev/node-pool": "oceanus" + }, + "images": [ + "postgres:15" + ] } ], "services": [ @@ -1387,7 +1829,7 @@ { "name": "http", "port": 80, - "targetPort": 80, + "targetPort": 8080, "protocol": "TCP" } ] @@ -1408,6 +1850,44 @@ } ] }, + { + "namespace": "climate", + "name": "typhon", + "type": "ClusterIP", + "selector": { + "app": "typhon" + }, + "ports": [ + { + "name": "metrics", + "port": 9108, + "targetPort": "metrics", + "protocol": "TCP" + }, + { + "name": "control", + "port": 9110, + "targetPort": "control", + "protocol": "TCP" + } + ] + }, + { + "namespace": "comms", + "name": "atlasbot", + "type": "ClusterIP", + "selector": { + "app": "atlasbot" + }, + "ports": [ + { + "name": "http", + "port": 8090, + "targetPort": 8090, + "protocol": "TCP" + } + ] + }, { "namespace": "comms", "name": "coturn", @@ -1876,6 +2356,12 @@ "targetPort": 18081, "protocol": "TCP" }, + { + "name": "status", + "port": 18084, + "targetPort": 18084, + "protocol": "TCP" + }, { "name": "p2p", "port": 18080, @@ -2002,6 +2488,132 @@ } ] }, + { + "namespace": "game-stream", + "name": "oauth2-proxy-wolf", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-wolf" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf", + "type": "ClusterIP", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "placeholder", + "port": 47989, + "targetPort": 47989, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf-api", + "type": "ClusterIP", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "http", + "port": 8088, + "targetPort": 8088, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf-gatekeeper", + "type": "ClusterIP", + "selector": { + "app": "wolf-gatekeeper" + }, + "ports": [ + { + "name": "http", + "port": 8087, + "targetPort": 8087, + "protocol": "TCP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolf-moonlight", + "type": "LoadBalancer", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "tcp-https", + "port": 47984, + "targetPort": 47984, + "protocol": "TCP" + }, + { + "name": "tcp-http", + "port": 47989, + "targetPort": 47989, + "protocol": "TCP" + }, + { + "name": "tcp-rtsp", + "port": 48010, + "targetPort": 48010, + "protocol": "TCP" + }, + { + "name": "udp-control", + "port": 47999, + "targetPort": 47999, + "protocol": "UDP" + }, + { + "name": "udp-video", + "port": 48100, + "targetPort": 48100, + "protocol": "UDP" + }, + { + "name": "udp-audio", + "port": 48200, + "targetPort": 48200, + "protocol": "UDP" + } + ] + }, + { + "namespace": "game-stream", + "name": "wolfmanager", + "type": "ClusterIP", + "selector": { + "app": "wolf" + }, + "ports": [ + { + "name": "http", + "port": 8080, + "targetPort": 3000, + "protocol": "TCP" + } + ] + }, { "namespace": "gitea", "name": "gitea", @@ -2050,6 +2662,44 @@ } ] }, + { + "namespace": "hermes", + "name": "hermes", + "type": "ClusterIP", + "selector": { + "app": "hermes" + }, + "ports": [ + { + "name": "api", + "port": 8642, + "targetPort": "api", + "protocol": "TCP" + }, + { + "name": "dashboard", + "port": 9119, + "targetPort": "dashboard", + "protocol": "TCP" + } + ] + }, + { + "namespace": "hermes", + "name": "hermes-ollama", + "type": "ClusterIP", + "selector": { + "app": "hermes-ollama" + }, + "ports": [ + { + "name": "http", + "port": 11434, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "jellyfin", "name": "jellyfin", @@ -2200,6 +2850,70 @@ } ] }, + { + "namespace": "maintenance", + "name": "metis", + "type": "ClusterIP", + "selector": { + "app": "metis" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, + { + "namespace": "maintenance", + "name": "oauth2-proxy-metis", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-metis" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "maintenance", + "name": "oauth2-proxy-soteria", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-soteria" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "maintenance", + "name": "soteria", + "type": "ClusterIP", + "selector": { + "app": "soteria" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "monitoring", "name": "dcgm-exporter", @@ -2232,6 +2946,38 @@ } ] }, + { + "namespace": "monitoring", + "name": "nvidia-process-exporter", + "type": "ClusterIP", + "selector": { + "app": "nvidia-process-exporter" + }, + "ports": [ + { + "name": "metrics", + "port": 9401, + "targetPort": "metrics", + "protocol": "TCP" + } + ] + }, + { + "namespace": "monitoring", + "name": "platform-quality-gateway", + "type": "ClusterIP", + "selector": { + "app": "platform-quality-gateway" + }, + "ports": [ + { + "name": "http", + "port": 9091, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "monitoring", "name": "postmark-exporter", @@ -2248,6 +2994,22 @@ } ] }, + { + "namespace": "monitoring", + "name": "vmalert-atlas-availability", + "type": "ClusterIP", + "selector": { + "app": "vmalert-atlas-availability" + }, + "ports": [ + { + "name": "http", + "port": 8880, + "targetPort": "http", + "protocol": "TCP" + } + ] + }, { "namespace": "nextcloud", "name": "collabora", @@ -2350,6 +3112,54 @@ } ] }, + { + "namespace": "quality", + "name": "oauth2-proxy-sonarqube", + "type": "ClusterIP", + "selector": { + "app": "oauth2-proxy-sonarqube" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 4180, + "protocol": "TCP" + } + ] + }, + { + "namespace": "quality", + "name": "sonarqube", + "type": "ClusterIP", + "selector": { + "app": "sonarqube" + }, + "ports": [ + { + "name": "http", + "port": 9000, + "targetPort": 9000, + "protocol": "TCP" + } + ] + }, + { + "namespace": "quality", + "name": "sonarqube-exporter", + "type": "ClusterIP", + "selector": { + "app": "sonarqube-exporter" + }, + "ports": [ + { + "name": "metrics", + "port": 9798, + "targetPort": "metrics", + "protocol": "TCP" + } + ] + }, { "namespace": "sso", "name": "keycloak", @@ -2517,9 +3327,77 @@ "protocol": "TCP" } ] + }, + { + "namespace": "veles", + "name": "veles-backend", + "type": "ClusterIP", + "selector": { + "app": "veles-backend" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 8796, + "protocol": "TCP" + } + ] + }, + { + "namespace": "veles", + "name": "veles-frontend", + "type": "ClusterIP", + "selector": { + "app": "veles-frontend" + }, + "ports": [ + { + "name": "http", + "port": 80, + "targetPort": 8080, + "protocol": "TCP" + } + ] + }, + { + "namespace": "veles", + "name": "veles-postgres", + "type": "ClusterIP", + "selector": { + "app": "veles-postgres" + }, + "ports": [ + { + "name": "postgres", + "port": 5432, + "targetPort": 5432, + "protocol": "TCP" + } + ] } ], "http_endpoints": [ + { + "host": "agent.bstein.dev", + "path": "/", + "backend": { + "namespace": "hermes", + "service": "hermes", + "port": "dashboard", + "workloads": [ + { + "kind": "Deployment", + "name": "hermes" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "agent", + "source": "hermes" + } + }, { "host": "auth.bstein.dev", "path": "/", @@ -2540,6 +3418,26 @@ "source": "oauth2-proxy" } }, + { + "host": "backup.bstein.dev", + "path": "/", + "backend": { + "namespace": "maintenance", + "service": "oauth2-proxy-soteria", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-soteria" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "soteria", + "source": "maintenance" + } + }, { "host": "bstein.dev", "path": "/", @@ -3280,6 +4178,46 @@ "source": "pegasus" } }, + { + "host": "quality.bstein.dev", + "path": "/", + "backend": { + "namespace": "quality", + "service": "oauth2-proxy-sonarqube", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-sonarqube" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "sonarqube", + "source": "quality" + } + }, + { + "host": "recovery.bstein.dev", + "path": "/", + "backend": { + "namespace": "maintenance", + "service": "oauth2-proxy-metis", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-metis" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "metis", + "source": "maintenance" + } + }, { "host": "scm.bstein.dev", "path": "/", @@ -3399,9 +4337,112 @@ "name": "vaultwarden-ingress", "source": "vaultwarden" } + }, + { + "host": "veles.bstein.dev", + "path": "/", + "backend": { + "namespace": "veles", + "service": "veles-frontend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-frontend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "veles.bstein.dev", + "path": "/api", + "backend": { + "namespace": "veles", + "service": "veles-backend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-backend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "veles.bstein.dev", + "path": "/events", + "backend": { + "namespace": "veles", + "service": "veles-backend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-backend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "veles.bstein.dev", + "path": "/ws", + "backend": { + "namespace": "veles", + "service": "veles-backend", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "veles-backend" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "veles", + "source": "veles" + } + }, + { + "host": "wolf.bstein.dev", + "path": "/", + "backend": { + "namespace": "game-stream", + "service": "oauth2-proxy-wolf", + "port": 80, + "workloads": [ + { + "kind": "Deployment", + "name": "oauth2-proxy-wolf" + } + ] + }, + "via": { + "kind": "Ingress", + "name": "wolf", + "source": "game-stream" + } } ], "helmrelease_host_hints": { + "cert-manager:cert-manager/cert-manager": [ + "atlas.bstein.dev" + ], "comms:comms/othrys-element": [ "call.live.bstein.dev", "live.bstein.dev", @@ -3423,7 +4464,8 @@ "registry.bstein.dev" ], "longhorn:longhorn-system/longhorn": [ - "registry.bstein.dev" + "registry.bstein.dev", + "veles.bstein.dev" ], "mailu:mailu-mailserver/mailu": [ "bstein.dev", @@ -3433,13 +4475,16 @@ "alerts.bstein.dev" ], "monitoring:monitoring/grafana": [ + "atlas.bstein.dev", "bstein.dev", - "mail.bstein.dev", "metrics.bstein.dev", "sso.bstein.dev" ], "monitoring:monitoring/kube-state-metrics": [ "atlas.bstein.dev" + ], + "vault-injector:vault/vault-injector": [ + "atlas.bstein.dev" ] } } diff --git a/services/comms/knowledge/catalog/atlas.yaml b/services/comms/knowledge/catalog/atlas.yaml index 637b5f970..0592c2036 100644 --- a/services/comms/knowledge/catalog/atlas.yaml +++ b/services/comms/knowledge/catalog/atlas.yaml @@ -1,5 +1,5 @@ # knowledge/catalog/atlas.yaml -# Generated by scripts/knowledge_render_atlas.py (do not edit by hand) +# Generated by scripts/render/knowledge_render_atlas.py (do not edit by hand) cluster: atlas sources: - name: ai-llm @@ -9,7 +9,7 @@ sources: path: services/bstein-dev-home targetNamespace: bstein-dev-home - name: bstein-dev-home-migrations - path: services/bstein-dev-home/migrations + path: services/bstein-dev-home/migration-jobs targetNamespace: bstein-dev-home - name: cert-manager path: infrastructure/cert-manager @@ -26,12 +26,18 @@ sources: - name: crypto path: services/crypto targetNamespace: crypto +- name: descheduler + path: infrastructure/descheduler + targetNamespace: kube-system - name: finance path: services/finance targetNamespace: finance - name: flux-system path: clusters/atlas/flux-system targetNamespace: null +- name: game-stream + path: services/game-stream + targetNamespace: game-stream - name: gitea path: services/gitea targetNamespace: gitea @@ -47,6 +53,9 @@ sources: - name: helm path: infrastructure/sources/helm targetNamespace: flux-system +- name: hermes + path: services/hermes + targetNamespace: hermes - name: jellyfin path: services/jellyfin targetNamespace: jellyfin @@ -92,6 +101,9 @@ sources: - name: oauth2-proxy path: services/oauth2-proxy targetNamespace: sso +- name: openclaw + path: services/openclaw + targetNamespace: openclaw - name: openldap path: services/openldap targetNamespace: sso @@ -107,12 +119,21 @@ sources: - name: postgres path: infrastructure/postgres targetNamespace: postgres +- name: quality + path: services/quality + targetNamespace: quality +- name: resource-guardrails + path: infrastructure/resource-guardrails + targetNamespace: null - name: sui-metrics path: services/sui-metrics/overlays/atlas targetNamespace: sui-metrics - name: traefik path: infrastructure/traefik targetNamespace: traefik +- name: typhon + path: services/typhon + targetNamespace: climate - name: vault path: services/vault targetNamespace: vault @@ -125,6 +146,9 @@ sources: - name: vaultwarden path: services/vaultwarden targetNamespace: vaultwarden +- name: veles + path: services/veles + targetNamespace: veles - name: wallet-monero-temp path: services/crypto/wallet-monero-temp targetNamespace: crypto @@ -151,7 +175,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-157 + - registry.bstein.dev/bstein/bstein-dev-home-backend:0.1.1-414 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-frontend @@ -162,7 +186,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-157 + - registry.bstein.dev/bstein/bstein-dev-home-frontend:0.1.1-414 - kind: Deployment namespace: bstein-dev-home name: bstein-dev-home-vault-sync @@ -183,6 +207,26 @@ workloads: node-role.kubernetes.io/worker: 'true' images: - python:3.11-slim +- kind: Deployment + namespace: climate + name: typhon + labels: + app: typhon + serviceAccountName: typhon + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: 'true' + images: + - registry.bstein.dev/bstein/typhon:main +- kind: Deployment + namespace: climate + name: typhon-vault-sync + labels: + app: typhon-vault-sync + serviceAccountName: typhon + nodeSelector: {} + images: + - alpine:3.20 - kind: Deployment namespace: comms name: atlasbot @@ -191,6 +235,7 @@ workloads: serviceAccountName: atlasbot nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - python:3.11-slim - kind: Deployment @@ -210,6 +255,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - ghcr.io/coturn/coturn:4.6.2 - kind: Deployment @@ -220,6 +266,7 @@ workloads: serviceAccountName: null nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - ghcr.io/element-hq/element-call@sha256:e6897c7818331714eae19d83ef8ea94a8b41115f0d8d3f62c2fed2d02c65c9bc - kind: Deployment @@ -230,6 +277,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - livekit/livekit-server:v1.9.0 - kind: Deployment @@ -240,6 +288,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - registry.bstein.dev/tools/lk-jwt-service-vault:0.3.0 - kind: Deployment @@ -250,6 +299,7 @@ workloads: serviceAccountName: comms-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - ghcr.io/element-hq/matrix-authentication-service:1.8.0 - kind: Deployment @@ -267,7 +317,8 @@ workloads: labels: app: matrix-wellknown serviceAccountName: null - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/worker: 'true' images: - nginx:1.27-alpine - kind: DaemonSet @@ -275,11 +326,13 @@ workloads: name: monero-xmrig labels: app: monero-xmrig + atlas.bstein.dev/workload-profile: scavenger serviceAccountName: null nodeSelector: + atlas.bstein.dev/crypto-mining-enabled: 'true' node-role.kubernetes.io/worker: 'true' images: - - ghcr.io/tari-project/xmrig@sha256:80defbfd0b640d604c91cb5101d3642db7928e1e68ee3c6b011289b3565a39d9 + - ghcr.io/tari-project/xmrig@sha256:d590a41613fea974f155280920095ea10c3710f55ecf16fc38fd3a1c18718129 - kind: Deployment namespace: crypto name: crypto-vault-sync @@ -294,6 +347,7 @@ workloads: name: monero-p2pool labels: app: monero-p2pool + atlas.bstein.dev/workload-profile: heavy serviceAccountName: crypto-vault-sync nodeSelector: node-role.kubernetes.io/worker: 'true' @@ -304,10 +358,12 @@ workloads: name: monerod labels: app: monerod + atlas.bstein.dev/workload-profile: heavy serviceAccountName: null nodeSelector: node-role.kubernetes.io/worker: 'true' images: + - python:3.11-alpine - registry.bstein.dev/crypto/monerod:0.18.4.1 - kind: Deployment namespace: crypto @@ -425,11 +481,44 @@ workloads: kubernetes.io/os: linux images: - ghcr.io/fluxcd/source-controller:v1.7.4 +- kind: DaemonSet + namespace: game-stream + name: wolf-gatekeeper + labels: + app: wolf-gatekeeper + serviceAccountName: null + nodeSelector: + kubernetes.io/hostname: titan-24 + images: + - ghcr.io/games-on-whales/wolf:stable +- kind: Deployment + namespace: game-stream + name: oauth2-proxy-wolf + labels: + app: oauth2-proxy-wolf + serviceAccountName: game-stream-vault + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: StatefulSet + namespace: game-stream + name: wolf + labels: + app: wolf + atlas.bstein.dev/game-mode: wolf + serviceAccountName: null + nodeSelector: + kubernetes.io/hostname: titan-24 + images: + - ghcr.io/games-on-whales/wolf:stable + - ghcr.io/games-on-whales/wolfmanager/wolfmanager:latest - kind: Deployment namespace: gitea name: gitea labels: app: gitea + atlas.bstein.dev/workload-profile: heavy serviceAccountName: gitea-vault nodeSelector: node-role.kubernetes.io/worker: 'true' @@ -456,11 +545,30 @@ workloads: images: - nginx:1.27.5-alpine@sha256:65645c7bb6a0661892a8b03b89d0743208a18dd2f3f17a54ef4b76fb8e2f2a10 - wger/server@sha256:710588b78af4e0aa0b4d8a8061e4563e16eae80eeaccfe7f9e0d9cbdd7f0cbc5 +- kind: Deployment + namespace: hermes + name: hermes + labels: + app: hermes + serviceAccountName: hermes-triage + nodeSelector: {} + images: + - nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 +- kind: Deployment + namespace: hermes + name: hermes-ollama + labels: + app: hermes-ollama + serviceAccountName: null + nodeSelector: {} + images: + - ollama/ollama@sha256:2c9595c555fd70a28363489ac03bd5bf9e7c5bdf2890373c3a830ffd7252ce6d - kind: Deployment namespace: jellyfin name: jellyfin labels: app: jellyfin + atlas.bstein.dev/workload-profile: heavy serviceAccountName: pegasus-vault-sync nodeSelector: {} images: @@ -483,7 +591,8 @@ workloads: labels: app: pegasus-vault-sync serviceAccountName: pegasus-vault-sync - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/worker: 'true' images: - alpine:3.20 - kind: Deployment @@ -572,7 +681,7 @@ workloads: nodeSelector: kubernetes.io/os: linux images: - - registry.bstein.dev/infra/coredns:1.12.1 + - registry.k8s.io/coredns/coredns:v1.12.1 - kind: DaemonSet namespace: logging name: node-image-gc-rpi4 @@ -600,7 +709,7 @@ workloads: app: node-log-rotation serviceAccountName: node-log-rotation nodeSelector: - hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 - kind: Deployment @@ -682,6 +791,28 @@ workloads: node-role.kubernetes.io/worker: 'true' images: - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: metis-sentinel-amd64 + labels: + app: metis-sentinel-amd64 + serviceAccountName: metis + nodeSelector: + kubernetes.io/arch: amd64 + kubernetes.io/os: linux + images: + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-amd64 +- kind: DaemonSet + namespace: maintenance + name: metis-sentinel-arm64 + labels: + app: metis-sentinel-arm64 + serviceAccountName: metis + nodeSelector: + kubernetes.io/arch: arm64 + kubernetes.io/os: linux + images: + - registry.bstein.dev/bstein/metis-sentinel:0.1.0-243-arm64 - kind: DaemonSet namespace: maintenance name: node-image-sweeper @@ -701,6 +832,36 @@ workloads: nodeSelector: {} images: - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: rpi-resource-reservation + labels: + app: rpi-resource-reservation + serviceAccountName: node-nofile + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: titan-22-link-keeper + labels: + app: titan-22-link-keeper + serviceAccountName: null + nodeSelector: + kubernetes.io/hostname: titan-22 + images: + - bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 +- kind: DaemonSet + namespace: maintenance + name: titan-24-docker + labels: + app: titan-24-docker + serviceAccountName: titan-24-docker + nodeSelector: + kubernetes.io/hostname: titan-24 + images: + - debian:13-slim - kind: Deployment namespace: maintenance name: ariadne @@ -711,16 +872,57 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - registry.bstein.dev/bstein/ariadne:0.1.0-49 + - registry.bstein.dev/bstein/ariadne:0.1.0-356 - kind: Deployment namespace: maintenance name: maintenance-vault-sync labels: app: maintenance-vault-sync serviceAccountName: maintenance-vault-sync - nodeSelector: {} + nodeSelector: + node-role.kubernetes.io/worker: 'true' images: - alpine:3.20 +- kind: Deployment + namespace: maintenance + name: metis + labels: + app: metis + serviceAccountName: metis + nodeSelector: {} + images: + - registry.bstein.dev/bstein/metis:0.1.0-243-arm64 +- kind: Deployment + namespace: maintenance + name: oauth2-proxy-metis + labels: + app: oauth2-proxy-metis + serviceAccountName: maintenance-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: Deployment + namespace: maintenance + name: oauth2-proxy-soteria + labels: + app: oauth2-proxy-soteria + serviceAccountName: maintenance-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: Deployment + namespace: maintenance + name: soteria + labels: + app: soteria + serviceAccountName: soteria + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: 'true' + images: + - registry.bstein.dev/bstein/soteria:0.1.0-120 - kind: DaemonSet namespace: monitoring name: dcgm-exporter @@ -740,6 +942,15 @@ workloads: jetson: 'true' images: - python:3.10-slim +- kind: DaemonSet + namespace: monitoring + name: nvidia-process-exporter + labels: + app: nvidia-process-exporter + serviceAccountName: nvidia-process-exporter + nodeSelector: {} + images: + - python:3.12-slim - kind: Deployment namespace: monitoring name: monitoring-vault-sync @@ -749,6 +960,15 @@ workloads: nodeSelector: {} images: - alpine:3.20 +- kind: Deployment + namespace: monitoring + name: platform-quality-gateway + labels: + app: platform-quality-gateway + serviceAccountName: null + nodeSelector: {} + images: + - prom/pushgateway:v1.11.2 - kind: Deployment namespace: monitoring name: postmark-exporter @@ -758,6 +978,15 @@ workloads: nodeSelector: {} images: - python:3.12-alpine +- kind: Deployment + namespace: monitoring + name: vmalert-atlas-availability + labels: + app: vmalert-atlas-availability + serviceAccountName: vmalert-atlas-availability + nodeSelector: {} + images: + - victoriametrics/vmalert:v1.113.0 - kind: Deployment namespace: nextcloud name: collabora @@ -765,7 +994,7 @@ workloads: app: collabora serviceAccountName: null nodeSelector: - hardware: rpi5 + kubernetes.io/arch: amd64 images: - collabora/code@sha256:3c58d0e9bae75e4647467d0c7d91cb66f261d3e814709aed590b5c334a04db26 - kind: Deployment @@ -776,6 +1005,7 @@ workloads: serviceAccountName: nextcloud-vault nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - nextcloud:29-apache - kind: Deployment @@ -815,10 +1045,41 @@ workloads: app: postgres serviceAccountName: postgres-vault nodeSelector: + hardware: rpi5 node-role.kubernetes.io/worker: 'true' images: - postgres:15 - quay.io/prometheuscommunity/postgres-exporter:v0.15.0 +- kind: Deployment + namespace: quality + name: oauth2-proxy-sonarqube + labels: + app: oauth2-proxy-sonarqube + serviceAccountName: quality-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 +- kind: Deployment + namespace: quality + name: sonarqube + labels: + app: sonarqube + serviceAccountName: sonarqube + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - sonarqube:lts-community +- kind: Deployment + namespace: quality + name: sonarqube-exporter + labels: + app: sonarqube-exporter + serviceAccountName: quality-vault-sync + nodeSelector: + node-role.kubernetes.io/worker: 'true' + images: + - registry.bstein.dev/bstein/python:3.12-slim - kind: Deployment namespace: sso name: keycloak @@ -868,6 +1129,7 @@ workloads: serviceAccountName: sui-metrics nodeSelector: hardware: rpi5 + node-role.kubernetes.io/worker: 'true' images: - victoriametrics/vmagent:v1.103.0 - kind: Deployment @@ -877,7 +1139,7 @@ workloads: app: traefik app.kubernetes.io/instance: traefik-kube-system app.kubernetes.io/name: traefik - serviceAccountName: traefik-ingress-controller + serviceAccountName: atlas-traefik-ingress-controller nodeSelector: node-role.kubernetes.io/worker: 'true' images: @@ -892,7 +1154,7 @@ workloads: kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - - hashicorp/vault:1.17.6 + - hashicorp/vault:1.21.4 - kind: Deployment namespace: vaultwarden name: vaultwarden @@ -900,10 +1162,50 @@ workloads: app: vaultwarden serviceAccountName: vaultwarden-vault nodeSelector: + hardware: rpi5 kubernetes.io/arch: arm64 node-role.kubernetes.io/worker: 'true' images: - vaultwarden/server:1.35.2 +- kind: Deployment + namespace: veles + name: veles-backend + labels: + app: veles-backend + serviceAccountName: veles-backend + nodeSelector: + kubernetes.io/arch: amd64 + veles.bstein.dev/node-pool: oceanus + images: + - registry.bstein.dev/veles/veles-backend:0.7.56 +- kind: Deployment + namespace: veles + name: veles-frontend + labels: + app: veles-frontend + serviceAccountName: veles-frontend + nodeSelector: {} + images: + - registry.bstein.dev/veles/veles-frontend:0.7.56 +- kind: Deployment + namespace: veles + name: veles-vault-sync + labels: + app: veles-vault-sync + serviceAccountName: veles-vault-sync + nodeSelector: {} + images: + - alpine:3.20 +- kind: StatefulSet + namespace: veles + name: veles-postgres + labels: + app: veles-postgres + serviceAccountName: veles-postgres + nodeSelector: + veles.bstein.dev/node-pool: oceanus + images: + - postgres:15 services: - namespace: ai name: ollama @@ -933,7 +1235,7 @@ services: ports: - name: http port: 80 - targetPort: 80 + targetPort: 8080 protocol: TCP - namespace: bstein-dev-home name: chat-ai-gateway @@ -945,6 +1247,30 @@ services: port: 80 targetPort: 8080 protocol: TCP +- namespace: climate + name: typhon + type: ClusterIP + selector: + app: typhon + ports: + - name: metrics + port: 9108 + targetPort: metrics + protocol: TCP + - name: control + port: 9110 + targetPort: control + protocol: TCP +- namespace: comms + name: atlasbot + type: ClusterIP + selector: + app: atlasbot + ports: + - name: http + port: 8090 + targetPort: 8090 + protocol: TCP - namespace: comms name: coturn type: LoadBalancer @@ -1253,6 +1579,10 @@ services: port: 18081 targetPort: 18081 protocol: TCP + - name: status + port: 18084 + targetPort: 18084 + protocol: TCP - name: p2p port: 18080 targetPort: 18080 @@ -1331,6 +1661,86 @@ services: port: 80 targetPort: http-webhook protocol: TCP +- namespace: game-stream + name: oauth2-proxy-wolf + type: ClusterIP + selector: + app: oauth2-proxy-wolf + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: game-stream + name: wolf + type: ClusterIP + selector: + app: wolf + ports: + - name: placeholder + port: 47989 + targetPort: 47989 + protocol: TCP +- namespace: game-stream + name: wolf-api + type: ClusterIP + selector: + app: wolf + ports: + - name: http + port: 8088 + targetPort: 8088 + protocol: TCP +- namespace: game-stream + name: wolf-gatekeeper + type: ClusterIP + selector: + app: wolf-gatekeeper + ports: + - name: http + port: 8087 + targetPort: 8087 + protocol: TCP +- namespace: game-stream + name: wolf-moonlight + type: LoadBalancer + selector: + app: wolf + ports: + - name: tcp-https + port: 47984 + targetPort: 47984 + protocol: TCP + - name: tcp-http + port: 47989 + targetPort: 47989 + protocol: TCP + - name: tcp-rtsp + port: 48010 + targetPort: 48010 + protocol: TCP + - name: udp-control + port: 47999 + targetPort: 47999 + protocol: UDP + - name: udp-video + port: 48100 + targetPort: 48100 + protocol: UDP + - name: udp-audio + port: 48200 + targetPort: 48200 + protocol: UDP +- namespace: game-stream + name: wolfmanager + type: ClusterIP + selector: + app: wolf + ports: + - name: http + port: 8080 + targetPort: 3000 + protocol: TCP - namespace: gitea name: gitea type: ClusterIP @@ -1361,6 +1771,30 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: hermes + name: hermes + type: ClusterIP + selector: + app: hermes + ports: + - name: api + port: 8642 + targetPort: api + protocol: TCP + - name: dashboard + port: 9119 + targetPort: dashboard + protocol: TCP +- namespace: hermes + name: hermes-ollama + type: ClusterIP + selector: + app: hermes-ollama + ports: + - name: http + port: 11434 + targetPort: http + protocol: TCP - namespace: jellyfin name: jellyfin type: ClusterIP @@ -1457,6 +1891,46 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: maintenance + name: metis + type: ClusterIP + selector: + app: metis + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP +- namespace: maintenance + name: oauth2-proxy-metis + type: ClusterIP + selector: + app: oauth2-proxy-metis + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: maintenance + name: oauth2-proxy-soteria + type: ClusterIP + selector: + app: oauth2-proxy-soteria + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: maintenance + name: soteria + type: ClusterIP + selector: + app: soteria + ports: + - name: http + port: 80 + targetPort: http + protocol: TCP - namespace: monitoring name: dcgm-exporter type: ClusterIP @@ -1477,6 +1951,26 @@ services: port: 9100 targetPort: metrics protocol: TCP +- namespace: monitoring + name: nvidia-process-exporter + type: ClusterIP + selector: + app: nvidia-process-exporter + ports: + - name: metrics + port: 9401 + targetPort: metrics + protocol: TCP +- namespace: monitoring + name: platform-quality-gateway + type: ClusterIP + selector: + app: platform-quality-gateway + ports: + - name: http + port: 9091 + targetPort: http + protocol: TCP - namespace: monitoring name: postmark-exporter type: ClusterIP @@ -1487,6 +1981,16 @@ services: port: 8000 targetPort: http protocol: TCP +- namespace: monitoring + name: vmalert-atlas-availability + type: ClusterIP + selector: + app: vmalert-atlas-availability + ports: + - name: http + port: 8880 + targetPort: http + protocol: TCP - namespace: nextcloud name: collabora type: ClusterIP @@ -1551,6 +2055,36 @@ services: port: 9187 targetPort: 9187 protocol: TCP +- namespace: quality + name: oauth2-proxy-sonarqube + type: ClusterIP + selector: + app: oauth2-proxy-sonarqube + ports: + - name: http + port: 80 + targetPort: 4180 + protocol: TCP +- namespace: quality + name: sonarqube + type: ClusterIP + selector: + app: sonarqube + ports: + - name: http + port: 9000 + targetPort: 9000 + protocol: TCP +- namespace: quality + name: sonarqube-exporter + type: ClusterIP + selector: + app: sonarqube-exporter + ports: + - name: metrics + port: 9798 + targetPort: metrics + protocol: TCP - namespace: sso name: keycloak type: ClusterIP @@ -1657,7 +2191,50 @@ services: port: 80 targetPort: http protocol: TCP +- namespace: veles + name: veles-backend + type: ClusterIP + selector: + app: veles-backend + ports: + - name: http + port: 80 + targetPort: 8796 + protocol: TCP +- namespace: veles + name: veles-frontend + type: ClusterIP + selector: + app: veles-frontend + ports: + - name: http + port: 80 + targetPort: 8080 + protocol: TCP +- namespace: veles + name: veles-postgres + type: ClusterIP + selector: + app: veles-postgres + ports: + - name: postgres + port: 5432 + targetPort: 5432 + protocol: TCP http_endpoints: +- host: agent.bstein.dev + path: / + backend: + namespace: hermes + service: hermes + port: dashboard + workloads: + - kind: Deployment + name: hermes + via: + kind: Ingress + name: agent + source: hermes - host: auth.bstein.dev path: / backend: @@ -1671,6 +2248,19 @@ http_endpoints: kind: Ingress name: oauth2-proxy source: oauth2-proxy +- host: backup.bstein.dev + path: / + backend: + namespace: maintenance + service: oauth2-proxy-soteria + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-soteria + via: + kind: Ingress + name: soteria + source: maintenance - host: bstein.dev path: / backend: @@ -2129,6 +2719,32 @@ http_endpoints: kind: Ingress name: pegasus source: pegasus +- host: quality.bstein.dev + path: / + backend: + namespace: quality + service: oauth2-proxy-sonarqube + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-sonarqube + via: + kind: Ingress + name: sonarqube + source: quality +- host: recovery.bstein.dev + path: / + backend: + namespace: maintenance + service: oauth2-proxy-metis + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-metis + via: + kind: Ingress + name: metis + source: maintenance - host: scm.bstein.dev path: / backend: @@ -2207,7 +2823,70 @@ http_endpoints: kind: Ingress name: vaultwarden-ingress source: vaultwarden +- host: veles.bstein.dev + path: / + backend: + namespace: veles + service: veles-frontend + port: 80 + workloads: + - kind: Deployment + name: veles-frontend + via: + kind: Ingress + name: veles + source: veles +- host: veles.bstein.dev + path: /api + backend: + namespace: veles + service: veles-backend + port: 80 + workloads: &id004 + - kind: Deployment + name: veles-backend + via: + kind: Ingress + name: veles + source: veles +- host: veles.bstein.dev + path: /events + backend: + namespace: veles + service: veles-backend + port: 80 + workloads: *id004 + via: + kind: Ingress + name: veles + source: veles +- host: veles.bstein.dev + path: /ws + backend: + namespace: veles + service: veles-backend + port: 80 + workloads: *id004 + via: + kind: Ingress + name: veles + source: veles +- host: wolf.bstein.dev + path: / + backend: + namespace: game-stream + service: oauth2-proxy-wolf + port: 80 + workloads: + - kind: Deployment + name: oauth2-proxy-wolf + via: + kind: Ingress + name: wolf + source: game-stream helmrelease_host_hints: + cert-manager:cert-manager/cert-manager: + - atlas.bstein.dev comms:comms/othrys-element: - call.live.bstein.dev - live.bstein.dev @@ -2225,15 +2904,18 @@ helmrelease_host_hints: - registry.bstein.dev longhorn:longhorn-system/longhorn: - registry.bstein.dev + - veles.bstein.dev mailu:mailu-mailserver/mailu: - bstein.dev - mail.bstein.dev monitoring:monitoring/alertmanager: - alerts.bstein.dev monitoring:monitoring/grafana: + - atlas.bstein.dev - bstein.dev - - mail.bstein.dev - metrics.bstein.dev - sso.bstein.dev monitoring:monitoring/kube-state-metrics: - atlas.bstein.dev + vault-injector:vault/vault-injector: + - atlas.bstein.dev diff --git a/services/comms/knowledge/catalog/metrics.json b/services/comms/knowledge/catalog/metrics.json index e929db582..adb4eea38 100644 --- a/services/comms/knowledge/catalog/metrics.json +++ b/services/comms/knowledge/catalog/metrics.json @@ -1,26 +1,181 @@ [ { - "dashboard": "Atlas GPU", - "panel_title": "Namespace GPU Share", + "dashboard": "Atlas GitOps", + "panel_title": "Flux Source", "panel_id": 1, - "panel_type": "piechart", - "description": "Shares are normalized within the selected filter. Switching scope changes the denominator.", + "panel_type": "stat", + "description": "Branch and revision currently reported by Flux's GitRepository source.", "tags": [ "atlas", - "gpu" + "gitops", + "flux" ], "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(100 * (sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) / clamp_min((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)) == 0))" + "max by (branch, revision) (ananke_gitops_flux_source_info{job=\"ananke-power\",namespace=\"flux-system\",name=\"flux-system\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Kustomizations Ready", + "panel_id": 2, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})), 1)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Kustomizations Suspended", + "panel_id": 3, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(max by (namespace, name) (ananke_gitops_kustomization_suspended{job=\"ananke-power\"})) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "HelmReleases Ready", + "panel_id": 4, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * sum(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})), 1)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "HelmReleases Suspended", + "panel_id": 5, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(max by (namespace, name) (ananke_gitops_helmrelease_suspended{job=\"ananke-power\"})) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "GitOps Exporter", + "panel_id": 6, + "panel_type": "stat", + "description": "", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "min(ananke_gitops_scrape_success{job=\"ananke-power\"}) or on() vector(0)", + "(time() - max(ananke_gitops_last_scrape_timestamp_seconds{job=\"ananke-power\"})) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Readiness History", + "panel_id": 7, + "panel_type": "state-timeline", + "description": "Ready percentage over time for Flux Kustomizations and HelmReleases.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(100 * sum(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})), 1), \"kind\", \"Kustomizations\", \"__name__\", \".*\") or label_replace(100 * sum(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})), 1), \"kind\", \"HelmReleases\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Flux Sources", + "panel_id": 8, + "panel_type": "table", + "description": "A Value of 1 means Ready; 0 means not Ready.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (namespace, name, url, branch, revision, ready, reason) (ananke_gitops_flux_source_info{job=\"ananke-power\"}) * on(namespace, name) group_left() max by (namespace, name) (ananke_gitops_flux_source_ready{job=\"ananke-power\"})" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "Kustomizations", + "panel_id": 9, + "panel_type": "table", + "description": "A Value of 1 means Ready; 0 means not Ready. The ready/reason labels come from Flux status.conditions.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (namespace, name, path, source_namespace, source_name, revision, ready, reason) (ananke_gitops_kustomization_info{job=\"ananke-power\"}) * on(namespace, name) group_left() max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})" + ] + }, + { + "dashboard": "Atlas GitOps", + "panel_title": "HelmReleases", + "panel_id": 10, + "panel_type": "table", + "description": "A Value of 1 means Ready; 0 means not Ready. Chart/version/app_version are included when Flux reports them.", + "tags": [ + "atlas", + "gitops", + "flux" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (namespace, name, chart, version, app_version, revision, ready, reason) (ananke_gitops_helmrelease_info{job=\"ananke-power\"}) * on(namespace, name) group_left() max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})" ] }, { "dashboard": "Atlas GPU", - "panel_title": "GPU Util by Namespace", - "panel_id": 2, - "panel_type": "timeseries", - "description": "", + "panel_title": "Namespace GPU Utilization", + "panel_id": 1, + "panel_type": "piechart", + "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.", "tags": [ "atlas", "gpu" @@ -28,7 +183,23 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)" + "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))" + ] + }, + { + "dashboard": "Atlas GPU", + "panel_title": "GPU Process Util by Namespace", + "panel_id": 2, + "panel_type": "timeseries", + "description": "NVML process-level SM utilization by namespace. Host covers GPU work outside Kubernetes pods.", + "tags": [ + "atlas", + "gpu" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)" ] }, { @@ -44,15 +215,15 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum by (Hostname) (DCGM_FI_DEV_GPU_UTIL{pod!=\"\"})" + "label_replace(avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"}), \"Hostname\", \"$1\", \"node\", \"(.*)\")" ] }, { "dashboard": "Atlas GPU", - "panel_title": "Top Pods by GPU Util", + "panel_title": "GPU Pods Reporting Device Util", "panel_id": 4, "panel_type": "table", - "description": "", + "description": "DCGM labels the device utilization sample with GPU-consuming pods; multiple pods on one device can report the same value.", "tags": [ "atlas", "gpu" @@ -63,313 +234,6 @@ "topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=\"\"}) by (namespace,pod,Hostname))" ] }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (range)", - "panel_id": 1, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (task) (increase(ariadne_task_runs_total{status=\"error\"}[$__range])))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Attempts / Failures", - "panel_id": 2, - "panel_type": "timeseries", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total[$__interval]))", - "sum(increase(ariadne_task_runs_total{status=\"error\"}[$__interval]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "One-off Job Pods (age hours)", - "panel_id": 3, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(((time() - kube_pod_start_time{pod!=\"\"}) / 3600) * on(namespace,pod) group_left(owner_name) (kube_pod_owner{owner_kind=\"Job\"} unless on(namespace, owner_name) label_replace(kube_job_owner{owner_kind=\"CronJob\"}, \"owner_name\", \"$1\", \"job_name\", \"(.*)\")) * on(namespace,pod) group_left(phase) max by (namespace,pod,phase) (kube_pod_status_phase{phase=~\"Running|Succeeded\"}))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Stale (>36h)", - "panel_id": 4, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "(sum((((time() - (kube_cronjob_status_last_successful_time and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"})) > bool 129600) unless on(namespace,cronjob) (kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1)) + count(((kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"} unless on(namespace,cronjob) kube_cronjob_status_last_successful_time) unless on(namespace,cronjob) (kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1))) or on() vector(0)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Missing Success", - "panel_id": 5, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "count(((kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"} unless on(namespace,cronjob) kube_cronjob_status_last_successful_time) unless on(namespace,cronjob) (kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1)) or on() vector(0)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Suspended", - "panel_id": 6, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum((kube_cronjob_spec_suspend and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"}) == 1) or on() vector(0)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (1h)", - "panel_id": 7, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total{status=\"error\"}[1h]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (24h)", - "panel_id": 8, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total{status=\"error\"}[24h]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Runs (1h)", - "panel_id": 9, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total[1h]))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Schedule Last Error (hours ago)", - "panel_id": 10, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time(ariadne_schedule_last_error_timestamp_seconds[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Schedule Last Success (hours ago)", - "panel_id": 11, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time(ariadne_schedule_last_success_timestamp_seconds[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Last Success (hours ago)", - "panel_id": 12, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time((kube_cronjob_status_last_successful_time and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"})[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Glue Jobs Last Schedule (hours ago)", - "panel_id": 13, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc((time() - max_over_time((kube_cronjob_status_last_schedule_time and on(namespace,cronjob) kube_cronjob_labels{label_atlas_bstein_dev_glue=\"true\"})[$__range])) / 3600)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (1h)", - "panel_id": 14, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (task) (increase(ariadne_task_runs_total{status=\"error\"}[1h])))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Task Errors (30d)", - "panel_id": 15, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (task) (increase(ariadne_task_runs_total{status=\"error\"}[30d])))" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne Access Requests", - "panel_id": 16, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(ariadne_access_requests_total)" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne CI Coverage (%)", - "panel_id": 17, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "ariadne_ci_coverage_percent{repo=\"ariadne\"}" - ] - }, - { - "dashboard": "Atlas Jobs", - "panel_title": "Ariadne CI Tests (latest)", - "panel_id": 18, - "panel_type": "table", - "description": "", - "tags": [ - "atlas", - "jobs", - "glue" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "ariadne_ci_tests_total{repo=\"ariadne\"}" - ] - }, { "dashboard": "Atlas Mail", "panel_title": "Sent (1d)", @@ -688,7 +552,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((sum(rate(node_network_transmit_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0) + sum(rate(node_network_receive_bytes_total{device!~\"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*\"}[5m])) or on() vector(0)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -768,7 +632,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"})" + "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})" ] }, { @@ -864,7 +728,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -880,7 +744,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -896,7 +760,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -912,7 +776,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -928,7 +792,23 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + ] + }, + { + "dashboard": "Atlas Nodes", + "panel_title": "Astraios Usage", + "panel_id": 9, + "panel_type": "timeseries", + "description": "", + "tags": [ + "atlas", + "nodes" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -936,7 +816,7 @@ "panel_title": "Control Plane Ready", "panel_id": 2, "panel_type": "gauge", - "description": "", + "description": "Control-plane nodes currently Ready; full count is good, lower means Kubernetes core capacity is missing.", "tags": [ "atlas", "overview" @@ -952,7 +832,7 @@ "panel_title": "Control Plane Workloads", "panel_id": 3, "panel_type": "stat", - "description": "", + "description": "Non-core pods running on control-plane nodes; zero is good because control nodes should stay focused.", "tags": [ "atlas", "overview" @@ -968,7 +848,7 @@ "panel_title": "Stuck Terminating", "panel_id": 5, "panel_type": "stat", - "description": "", + "description": "Pods that Kubernetes cannot finish deleting; zero is good, growth means cleanup or storage may be stuck.", "tags": [ "atlas", "overview" @@ -981,10 +861,10 @@ }, { "dashboard": "Atlas Overview", - "panel_title": "Atlas Availability", + "panel_title": "Atlas Availability (365d)", "panel_id": 27, "panel_type": "stat", - "description": "", + "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data.", "tags": [ "atlas", "overview" @@ -992,7 +872,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) / 3)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) / clamp_min(sum(kube_deployment_spec_replicas{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}), 1)))))[365d:1h])" + "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\"}[24h])" ] }, { @@ -1000,7 +880,7 @@ "panel_title": "Problem Pods", "panel_id": 4, "panel_type": "stat", - "description": "", + "description": "Pods in unhealthy phases; zero is good, any count means a workload needs attention.", "tags": [ "atlas", "overview" @@ -1016,7 +896,7 @@ "panel_title": "CrashLoop / ImagePull", "panel_id": 6, "panel_type": "stat", - "description": "", + "description": "Pods restarting or unable to pull images; zero is good, any count usually blocks a service.", "tags": [ "atlas", "overview" @@ -1032,7 +912,7 @@ "panel_title": "Workers Ready", "panel_id": 1, "panel_type": "gauge", - "description": "", + "description": "Worker nodes currently Ready; full count is good, lower means less place to run services.", "tags": [ "atlas", "overview" @@ -1040,7 +920,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"})" + "sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"})" ] }, { @@ -1048,7 +928,7 @@ "panel_title": "Hottest node: CPU", "panel_id": 7, "panel_type": "stat", - "description": "", + "description": "Highest worker CPU load right now; lower is calmer, hot nodes may need pods moved.", "tags": [ "atlas", "overview" @@ -1056,7 +936,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" ] }, { @@ -1064,7 +944,7 @@ "panel_title": "Hottest node: RAM", "panel_id": 8, "panel_type": "stat", - "description": "", + "description": "Highest worker memory use right now; lower is safer, high values risk evictions.", "tags": [ "atlas", "overview" @@ -1072,7 +952,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" ] }, { @@ -1080,7 +960,7 @@ "panel_title": "Hottest node: NET (rx+tx)", "panel_id": 9, "panel_type": "stat", - "description": "", + "description": "Busiest node network rate; spikes can reveal traffic concentration or noisy services.", "tags": [ "atlas", "overview" @@ -1088,7 +968,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_network_receive_bytes_total{device!~\"lo\"}[5m]) + rate(node_network_transmit_bytes_total{device!~\"lo\"}[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" ] }, { @@ -1096,7 +976,7 @@ "panel_title": "Hottest node: I/O (r+w)", "panel_id": 10, "panel_type": "stat", - "description": "", + "description": "Busiest node disk I/O rate; high values can explain slow storage-backed apps.", "tags": [ "atlas", "overview" @@ -1104,7 +984,401 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + "label_replace(topk(1, avg by (node) ((sum by (instance) (rate(node_disk_read_bytes_total[5m]) + rate(node_disk_written_bytes_total[5m]))) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))), \"__name__\", \"$1\", \"node\", \"(.*)\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Astreae Usage", + "panel_id": 23, + "panel_type": "stat", + "description": "Percent of Astreae used; lower is safer, high values reduce storage headroom.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) * 100)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Asteria Usage", + "panel_id": 24, + "panel_type": "stat", + "description": "Percent of Asteria used; lower is safer, high values reduce storage headroom.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) * 100)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Astreae Free", + "panel_id": 25, + "panel_type": "stat", + "description": "Free space on Astreae; higher is better for backups and workload growth.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Asteria Free", + "panel_id": 26, + "panel_type": "stat", + "description": "Free space on Asteria; higher is better for backups and workload growth.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Pyrphoros UPS Current", + "panel_id": 40, + "panel_type": "stat", + "description": "Live Pyrphoros UPS draw and runtime; stable runtime means the lab can ride out short outages.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100) or on() vector(0), \"metric\", \"Draw\", \"__name__\", \".*\") or label_replace(max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Pyrphoros\"}) or on() vector(0), \"metric\", \"Runtime\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Statera UPS Current", + "panel_id": 144, + "panel_type": "stat", + "description": "Live Statera UPS draw and runtime; stable runtime means the lab can ride out short outages.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100) or on() vector(0), \"metric\", \"Draw\", \"__name__\", \".*\") or label_replace(max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Statera\"}) or on() vector(0), \"metric\", \"Runtime\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "UPS History (Power Draw)", + "panel_id": 41, + "panel_type": "timeseries", + "description": "UPS power draw over time; steady draw is normal, spikes show sudden load changes.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100)", + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Current Enclosure Temperature", + "panel_id": 42, + "panel_type": "stat", + "description": "Current tent temperature in C and F; moderate values protect hardware and plants.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)) or on() vector(0), \"metric\", \"\u00b0C\", \"__name__\", \".*\") or label_replace(max((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)) * 9 / 5 + 32) or on() vector(0), \"metric\", \"\u00b0F\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Current Enclosure Climate", + "panel_id": 143, + "panel_type": "stat", + "description": "Current humidity and VPD; in-range values mean the enclosure climate is stable.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)) or on() vector(0), \"metric\", \"%RH\", \"__name__\", \".*\") or label_replace(max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)) or on() vector(0), \"metric\", \"kPa\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Enclosure Climate History", + "panel_id": 43, + "panel_type": "timeseries", + "description": "Temperature on left axis, humidity and pressure on right axis with dynamic bound series so small swings remain visible.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)", + "(min_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)[$__range]) - 0.08)", + "(max_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)[$__range]) + 0.08)", + "clamp_min((min_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)[$__range]) - 0.35), 0)", + "clamp_max((max_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)[$__range]) + 0.35), 100)", + "clamp_min((min_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)[$__range]) - 0.03), 0)", + "(max_over_time(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)[$__range]) + 0.03)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Fan Intensity History", + "panel_id": 141, + "panel_type": "state-timeline", + "description": "Fan intensity lanes on the 0-10 controller scale. Cooler colors are quiet/low intensity; warmer colors mean the enclosure is pushing harder.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"1\"}), \"fan\", \"Outlet\", \"__name__\", \".*\") or label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"2\"}), \"fan\", \"Inlet - Inside\", \"__name__\", \".*\") or label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"3\"}), \"fan\", \"Inlet - Outside\", \"__name__\", \".*\") or label_replace(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{port=\"4\"}), \"fan\", \"Tent Interior\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Flux Source", + "panel_id": 140, + "panel_type": "stat", + "description": "Flux GitRepository branch reported by Ananke. Revision and object detail live in Atlas GitOps.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (branch, revision) (ananke_gitops_flux_source_info{job=\"ananke-power\",namespace=\"flux-system\",name=\"flux-system\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Current Gate Health", + "panel_id": 151, + "panel_type": "stat", + "description": "Current gate-check health across suites; skipped or not-applicable checks count as healthy, failures lower it.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((min by (suite) (((100 * sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch=~\"main|master|origin/main|origin/master\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch=~\"main|master|origin/main|origin/master\",check=\"supply_chain\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1) unless on(suite, check) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch=~\"main|master|origin/main|origin/master\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch=~\"main|master|origin/main|origin/master\",check=\"supply_chain\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1))) / clamp_min(sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch=~\"main|master|origin/main|origin/master\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!=\"\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch=~\"main|master|origin/main|origin/master\",check=\"supply_chain\",status!=\"\"})))) > 0), 1)), 1))) or (min by (suite) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"}))))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "CI Run Success (24h)", + "panel_id": 152, + "panel_type": "stat", + "description": "Percent of published quality-gate CI runs that completed successfully in 24h; this is automation health, not raw test pass rate.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * ((sum(platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|pegasus-health|pegasus_health|soteria|titan_iac|titan-iac|bstein_home|bstein-home|data_prepper|data-prepper|lesavka\",status=~\"ok|passed|success\"}) or on() vector(0))) / clamp_min(((sum(platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|pegasus-health|pegasus_health|soteria|titan_iac|titan-iac|bstein_home|bstein-home|data_prepper|data-prepper|lesavka\"}) or on() vector(0))), 1)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Failed Runs (24h)", + "panel_id": 153, + "panel_type": "stat", + "description": "Published quality-gate runs that failed in 24h; zero is good, any value needs a look.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|pegasus-health|pegasus_health|soteria|titan_iac|titan-iac|bstein_home|bstein-home|data_prepper|data-prepper|lesavka\",status!~\"ok|passed|success\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Suites With Runs (24h)", + "panel_id": 154, + "panel_type": "stat", + "description": "Configured suites with at least one published quality-gate run in 24h; full count means the dashboard is fresh.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\"}) > bool 0)) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Avg Coverage", + "panel_id": 155, + "panel_type": "stat", + "description": "Average latest line coverage across suites; higher means code is better protected by tests.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\"}))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "GitOps Health", + "panel_id": 150, + "panel_type": "state-timeline", + "description": "GitOps readiness and suspension health over time. Blue means perfect; warmer colors mean a readiness or suspension problem appeared.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "label_replace(100 * sum(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})), 1), \"signal\", \"Kustomizations Ready\", \"__name__\", \".*\") or label_replace(100 * sum(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) / clamp_min(count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})), 1), \"signal\", \"HelmReleases Ready\", \"__name__\", \".*\") or label_replace(100 * (1 - (sum(max by (namespace, name) (ananke_gitops_kustomization_suspended{job=\"ananke-power\"})) or on() vector(0)) / clamp_min((count(max by (namespace, name) (ananke_gitops_kustomization_ready{job=\"ananke-power\"})) or on() vector(0)), 1)), \"signal\", \"Kustomizations Not Suspended\", \"__name__\", \".*\") or label_replace(100 * (1 - (sum(max by (namespace, name) (ananke_gitops_helmrelease_suspended{job=\"ananke-power\"})) or on() vector(0)) / clamp_min((count(max by (namespace, name) (ananke_gitops_helmrelease_ready{job=\"ananke-power\"})) or on() vector(0)), 1)), \"signal\", \"HelmReleases Not Suspended\", \"__name__\", \".*\")" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "One-off Job Pods (age hours)", + "panel_id": 44, + "panel_type": "bargauge", + "description": "Temporary job pods by age; low or empty is good, old pods usually need cleanup.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(((time() - kube_pod_start_time{pod!=\"\"}) / 3600) * on(namespace,pod) group_left(owner_name) (kube_pod_owner{owner_kind=\"Job\"} unless on(namespace, owner_name) label_replace(kube_job_owner{owner_kind=\"CronJob\"}, \"owner_name\", \"$1\", \"job_name\", \"(.*)\")) * on(namespace,pod) group_left(phase) max by (namespace,pod,phase) (kube_pod_status_phase{phase=~\"Running|Succeeded\"}))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Ariadne Run Volume", + "panel_id": 45, + "panel_type": "timeseries", + "description": "Ariadne automation attempts and failures; attempts show activity, failures show work to investigate.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(increase(ariadne_task_runs_total[5m])) or on() vector(0)", + "sum(increase(ariadne_task_runs_total{status=\"error\"}[5m])) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Test Category Health", + "panel_id": 46, + "panel_type": "state-timeline", + "description": "Health by major test category across all suites over the last 24 hours. Skipped tests are healthy; failures and errors lower the lane.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|performance|regression|reliability|security|smoke|system|ui\"})" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Jenkins Last Success (h, newest first)", + "panel_id": 142, + "panel_type": "stat", + "description": "Top 6 most recent Jenkins successes by age (newest first). Green means last run succeeded; red means last run did not succeed. Use Atlas Jobs for the full list.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "Jenkins Last Failure (h, newest first)", + "panel_id": 243, + "panel_type": "stat", + "description": "Top 6 most recent Jenkins failures by age (newest first). Green means last run succeeded; red means last run did not succeed. Use Atlas Jobs for the full list.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) == 1), \"run_state\", \"ok\", \"exported_job\", \".*\")) or (label_replace((sort(bottomk(6, min by (exported_job,job_url,weather_icon) ((time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600)))) and on(exported_job,job_url,weather_icon) (max by (exported_job,job_url,weather_icon) (ariadne_jenkins_build_weather_job_last_status) != 1), \"run_state\", \"bad\", \"exported_job\", \".*\")))" + ] + }, + { + "dashboard": "Atlas Overview", + "panel_title": "PVC Backup Health / Age", + "panel_id": 47, + "panel_type": "bargauge", + "description": "Backup age in hours computed from last-success timestamps for restic-managed PVCs (nightly target: <=20h green, <40h yellow, <50h orange, >=50h red). PVCs that have backup history but currently no successful backup (missing/no_completed/error) are pinned to 999h for visibility.", + "tags": [ + "atlas", + "overview" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(max by (namespace, pvc) (((time() - pvc_backup_last_success_timestamp_seconds{driver=\"restic\"}) / 3600) or on(namespace,pvc,volume,driver) ((((pvc_backup_health_reason{driver=\"restic\",reason=~\"missing|no_completed|lookup_failed|unknown_timestamp\"} > 0) * (pvc_backup_count{driver=\"restic\"} > bool 0)) * 999))) or on() ((label_replace(label_replace(vector(999), \"namespace\", \"maintenance\", \"__name__\", \".*\"), \"pvc\", \"backup-telemetry-missing\", \"__name__\", \".*\")) unless on() ((count({__name__=~\"pvc_backup_(count|last_success_timestamp_seconds|health_reason)\",driver=\"restic\"})) > 0)))" ] }, { @@ -1112,7 +1386,7 @@ "panel_title": "Mail Sent (1d)", "panel_id": 30, "panel_type": "stat", - "description": "", + "description": "Outbound mail sent in the last day; useful context for mail health and bounce rates.", "tags": [ "atlas", "overview" @@ -1128,7 +1402,7 @@ "panel_title": "Mail Bounces (1d)", "panel_id": 31, "panel_type": "stat", - "description": "", + "description": "Outbound mail bounce rate and count; zero is best, high values risk delivery reputation.", "tags": [ "atlas", "overview" @@ -1145,7 +1419,7 @@ "panel_title": "Mail Success Rate (1d)", "panel_id": 32, "panel_type": "stat", - "description": "", + "description": "Outbound mail success rate; higher is better for user notifications.", "tags": [ "atlas", "overview" @@ -1161,7 +1435,7 @@ "panel_title": "Mail Limit Used (30d)", "panel_id": 33, "panel_type": "stat", - "description": "", + "description": "Postmark monthly send limit used; lower leaves more quota headroom.", "tags": [ "atlas", "overview" @@ -1177,7 +1451,7 @@ "panel_title": "Postgres Connections Used", "panel_id": 34, "panel_type": "stat", - "description": "", + "description": "Current Postgres connections; lower leaves room for apps during spikes.", "tags": [ "atlas", "overview" @@ -1193,7 +1467,7 @@ "panel_title": "Postgres Hottest Connections", "panel_id": 35, "panel_type": "stat", - "description": "", + "description": "Database with the most active connections; high values identify the pressure source.", "tags": [ "atlas", "overview" @@ -1204,135 +1478,6 @@ "topk(1, sum by (datname) (pg_stat_activity_count))" ] }, - { - "dashboard": "Atlas Overview", - "panel_title": "Astreae Usage", - "panel_id": 23, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"}) * 100)" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Asteria Usage", - "panel_id": 24, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "100 - (sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) / sum(node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"}) * 100)" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Astreae Free", - "panel_id": 25, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Asteria Free", - "panel_id": 26, - "panel_type": "stat", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "One-off Job Pods (age hours)", - "panel_id": 40, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(((time() - kube_pod_start_time{pod!=\"\"}) / 3600) * on(namespace,pod) group_left(owner_name) (kube_pod_owner{owner_kind=\"Job\"} unless on(namespace, owner_name) label_replace(kube_job_owner{owner_kind=\"CronJob\"}, \"owner_name\", \"$1\", \"job_name\", \"(.*)\")) * on(namespace,pod) group_left(phase) max by (namespace,pod,phase) (kube_pod_status_phase{phase=~\"Running|Succeeded\"}))" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Ariadne Attempts / Failures", - "panel_id": 41, - "panel_type": "timeseries", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sum(increase(ariadne_task_runs_total[$__interval]))", - "sum(increase(ariadne_task_runs_total{status=\"error\"}[$__interval]))" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Ariadne Test Success Rate", - "panel_id": 42, - "panel_type": "timeseries", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "100 * sum(max_over_time(ariadne_ci_tests_total{repo=\"ariadne\",result=\"passed\"}[1h])) / clamp_min(sum(max_over_time(ariadne_ci_tests_total{repo=\"ariadne\",result=~\"passed|failed|error\"}[1h])), 1)" - ] - }, - { - "dashboard": "Atlas Overview", - "panel_title": "Tests with Failures (24h)", - "panel_id": 43, - "panel_type": "bargauge", - "description": "", - "tags": [ - "atlas", - "overview" - ], - "datasource_uid": "atlas-vm", - "datasource_type": "prometheus", - "exprs": [ - "sort_desc(sum by (result) (max_over_time(ariadne_ci_tests_total{repo=\"ariadne\",result=~\"failed|error\"}[24h])))" - ] - }, { "dashboard": "Atlas Overview", "panel_title": "Namespace CPU Share", @@ -1351,10 +1496,10 @@ }, { "dashboard": "Atlas Overview", - "panel_title": "Namespace GPU Share", + "panel_title": "Namespace GPU Utilization", "panel_id": 12, "panel_type": "piechart", - "description": "Shares are normalized within the selected filter. Switching scope changes the denominator.", + "description": "Instant share of observed GPU compute activity by namespace. Host covers GPU work outside Kubernetes pods; idle appears only when observed GPU activity is zero.", "tags": [ "atlas", "overview" @@ -1362,7 +1507,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(100 * (sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) / clamp_min((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum(sum(DCGM_FI_DEV_GPU_UTIL{namespace!=\"\",pod!=\"\",$namespace_scope_gpu}) by (namespace)) or on() vector(0)) == 0))" + "(100 * ((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) / clamp_min((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)), 1)) or (label_replace(vector(100), \"namespace\", \"idle\", \"\", \"\") and on() ((sum((((sum by (namespace) (nvidia_namespace_gpu_sm_util_percent{namespace!=\"\",pod!=\"\",$namespace_scope_gpu})) > 0)) or (label_replace(((sum(((avg by (node) (label_replace(label_replace(DCGM_FI_DEV_GPU_UTIL, \"pod\", \"$1\", \"Hostname\", \"(.*)\"), \"namespace\", \"monitoring\", \"\", \"\") * on(namespace,pod) group_left(node) kube_pod_info{namespace=\"monitoring\"}) or max by (node) (jetson_gr3d_freq_percent{node!=\"\"})) unless on(node) (max by (node,gpu,uuid,model) (last_over_time(nvidia_gpu_device_utilization_percent[5m]))))) or on() vector(0)) > 0), \"namespace\", \"unattributed\", \"\", \"\"))) or on() vector(0)) == 0))" ] }, { @@ -1386,7 +1531,7 @@ "panel_title": "Worker Node CPU", "panel_id": 14, "panel_type": "timeseries", - "description": "", + "description": "Worker CPU over time; lower is calmer, sustained high load may need rescheduling.", "tags": [ "atlas", "overview" @@ -1394,7 +1539,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1402,7 +1547,7 @@ "panel_title": "Worker Node RAM", "panel_id": 15, "panel_type": "timeseries", - "description": "", + "description": "Worker memory over time; lower is safer, sustained high use risks evictions.", "tags": [ "atlas", "overview" @@ -1410,7 +1555,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1418,7 +1563,7 @@ "panel_title": "Control plane CPU", "panel_id": 16, "panel_type": "timeseries", - "description": "", + "description": "Control-plane CPU over time; low steady usage means Kubernetes has control headroom.", "tags": [ "atlas", "overview" @@ -1426,7 +1571,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) (((1 - avg by (instance) (rate(node_cpu_seconds_total{mode=\"idle\"}[5m]))) * 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1434,7 +1579,7 @@ "panel_title": "Control plane RAM", "panel_id": 17, "panel_type": "timeseries", - "description": "", + "description": "Control-plane memory over time; low steady usage means Kubernetes has control headroom.", "tags": [ "atlas", "overview" @@ -1442,7 +1587,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1450,7 +1595,7 @@ "panel_title": "Node Pod Share", "panel_id": 28, "panel_type": "piechart", - "description": "", + "description": "Share of pods per node; uneven share can reveal overloaded workers.", "tags": [ "atlas", "overview" @@ -1466,7 +1611,7 @@ "panel_title": "Top Nodes by Pod Count", "panel_id": 29, "panel_type": "bargauge", - "description": "", + "description": "Nodes with the most pods; lower and balanced is easier to operate.", "tags": [ "atlas", "overview" @@ -1482,7 +1627,7 @@ "panel_title": "Cluster Ingress Throughput", "panel_id": 18, "panel_type": "timeseries", - "description": "", + "description": "Traffic entering the cluster; spikes should line up with expected usage.", "tags": [ "atlas", "overview" @@ -1498,7 +1643,7 @@ "panel_title": "Cluster Egress Throughput", "panel_id": 19, "panel_type": "timeseries", - "description": "", + "description": "Traffic leaving the cluster; spikes should line up with expected usage.", "tags": [ "atlas", "overview" @@ -1514,7 +1659,7 @@ "panel_title": "Intra-Cluster Throughput", "panel_id": 20, "panel_type": "timeseries", - "description": "", + "description": "Traffic inside the cluster; high values can expose chatty services.", "tags": [ "atlas", "overview" @@ -1530,7 +1675,7 @@ "panel_title": "Root Filesystem Usage", "panel_id": 21, "panel_type": "timeseries", - "description": "", + "description": "Node root disk usage; lower is safer, high values can break kubelet.", "tags": [ "atlas", "overview" @@ -1538,15 +1683,15 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { "dashboard": "Atlas Overview", - "panel_title": "Nodes Closest to Full Root Disks", + "panel_title": "Nodes Closest to Full Astraios Disks", "panel_id": 22, - "panel_type": "bargauge", - "description": "", + "panel_type": "timeseries", + "description": "Astraios disk fullness by node; lower is safer for storage reliability.", "tags": [ "atlas", "overview" @@ -1554,7 +1699,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "sort_desc(topk(12, avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))))" + "avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astraios\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))" ] }, { @@ -1714,7 +1859,127 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.025)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.025)))))" + "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-jh\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-20\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-21\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.022) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.023) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.024) or (sum by (node) (kube_node_info{node=\"titan-23\"}) * 0 + 0.025) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.026000000000000002)))))" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "UPS Current Load", + "panel_id": 1, + "panel_type": "stat", + "description": "Per-UPS live snapshot: current draw in watts, estimated battery runtime if discharge started now, and charging/discharging status.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100) or on() vector(0)", + "max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Pyrphoros\"}) or on() vector(0)", + "max(ananke_ups_on_battery{job=\"ananke-power\",source=\"Pyrphoros\"}) or on() vector(0)", + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100) or on() vector(0)", + "max(ananke_ups_runtime_seconds{job=\"ananke-power\",source=\"Statera\"}) or on() vector(0)", + "max(ananke_ups_on_battery{job=\"ananke-power\",source=\"Statera\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "UPS History (Power Draw)", + "panel_id": 2, + "panel_type": "timeseries", + "description": "Historical UPS power consumption in watts for titan-db and tethys.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Pyrphoros\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Pyrphoros\"}) / 100)", + "max((ananke_ups_load_percent{job=\"ananke-power\",source=\"Statera\"} * ananke_ups_power_nominal_watts{job=\"ananke-power\",source=\"Statera\"}) / 100)" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Current Climate", + "panel_id": 3, + "panel_type": "stat", + "description": "Current tent temperature, humidity, VPD, and dew point. These render once Typhon climate telemetry is online.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)) or on() vector(0)", + "max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)) or on() vector(0)", + "max(max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)) or on() vector(0)", + "max((243.12 * (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))))) / (17.62 - (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)))))) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Climate History", + "panel_id": 4, + "panel_type": "timeseries", + "description": "Two-axis chart: tent temperature/humidity/dew point (left axis) and tent VPD in kPa (right axis).", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_vpd_kpa != 0)", + "(243.12 * (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))))) / (17.62 - (ln(clamp_min((max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_relative_humidity_percent != 0)), 1) / 100) + (17.62 * (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0))) / (243.12 + (max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_temperature_celsius != 0)))))" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Fan Activity", + "panel_id": 5, + "panel_type": "stat", + "description": "Current fan activity levels (0-10): inside outlet, inside inlet, outside inlet, and interior fans.", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "round(max(typhon_fan_speed_level{fan_group=\"outlet\"}) or on() vector(0))", + "round(max(typhon_fan_speed_level{fan_group=\"inside_inlet\"}) or on() vector(0))", + "round(max(typhon_fan_speed_level{fan_group=\"outside_inlet\"}) or on() vector(0))", + "round(max(typhon_fan_speed_level{fan_group=\"interior\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Power", + "panel_title": "Fan Intensity History", + "panel_id": 6, + "panel_type": "timeseries", + "description": "Historical fan activity for all four fan groups (0-10 scale).", + "tags": [ + "atlas", + "power", + "climate" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"outlet\"})", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"inside_inlet\"})", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"outside_inlet\"})", + "max without (job,instance,pod,service,endpoint,namespace,node,controller_name,controller_id,port_name,fan_group) (typhon_fan_speed_level{fan_group=\"interior\"})" ] }, { @@ -1794,7 +2059,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[24]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/astreae\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1810,7 +2075,7 @@ "datasource_uid": "atlas-vm", "datasource_type": "prometheus", "exprs": [ - "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=\"\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[24]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" + "(avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/asteria\",fstype!~\"tmpfs|overlay\"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename=~\"titan-0a|titan-0b|titan-0c|titan-db|titan-jh|titan-04|titan-05|titan-06|titan-07|titan-08|titan-09|titan-10|titan-11|titan-20|titan-21|titan-12|titan-13|titan-14|titan-15|titan-16|titan-17|titan-18|titan-19|titan-22|titan-23|titan-24\"}, \"node\", \"$1\", \"nodename\", \"(.*)\"))) * on(node) group_left() label_replace(node_uname_info{nodename=~\"titan-1[2-9]|titan-2[2-4]\"}, \"node\", \"$1\", \"nodename\", \"(.*)\")" ] }, { @@ -1876,5 +2141,872 @@ "exprs": [ "time() - max by (cronjob) (kube_cronjob_status_last_successful_time{namespace=\"maintenance\",cronjob=\"image-sweeper\"})" ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Current Gate Health (%)", + "panel_id": 156, + "panel_type": "stat", + "description": "Average latest required gate checks passing across selected suites; this is the current quality state.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((min by (suite) (((100 * (sum by (suite) (((clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1)) unless on(suite, check) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1))))) / clamp_min((sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!=\"\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!=\"\"})))) > 0), 1))), 1))) or (min by (suite) (platform_quality:test_category_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"}))))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success Rate (24h)", + "panel_id": 2, + "panel_type": "stat", + "description": "Percent of selected quality-gate CI runs that completed successfully in 24h; this is run health, not individual test pass rate.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * ((sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status=~\"ok|passed|success\"}) or on() vector(0))) / clamp_min(((sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"}) or on() vector(0))), 1)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success Rate (7d)", + "panel_id": 3, + "panel_type": "stat", + "description": "Percent of selected quality-gate CI runs that completed successfully in 7d; higher means more stable automation.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * ((sum(increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}))[7d:1h])) or on() vector(0))) / clamp_min(((sum(increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))[7d:1h])) or on() vector(0))), 1)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Failed Runs (24h)", + "panel_id": 4, + "panel_type": "stat", + "description": "Selected quality-gate runs that failed in 24h; zero is good and anything else needs a look.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status!~\"ok|passed|success\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Runs (24h)", + "panel_id": 5, + "panel_type": "stat", + "description": "Selected quality-gate CI run count in 24h; zero means the dashboard may be stale.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Suite Freshness (24h)", + "panel_id": 157, + "panel_type": "stat", + "description": "Percent of selected suites with at least one quality-gate CI run in 24h; 100% means inputs are fresh.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "100 * (sum((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"})) > bool 0) or on() vector(0)) / clamp_min(count(((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0))), 1)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Avg Coverage (%)", + "panel_id": 6, + "panel_type": "stat", + "description": "Average latest line coverage for selected suites; higher means better test protection.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(avg((max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"}))) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Suites with LOC >500", + "panel_id": 7, + "panel_type": "stat", + "description": "Selected suites with oversized source files; zero is good for maintainability.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(((max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) > bool 0)) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Latest Gate Health by Suite", + "panel_id": 8, + "panel_type": "bargauge", + "description": "Current health by suite from required gate checks, capped by category-level test health. Skipped and not-applicable results are healthy; failures and errors lower the value.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort(((min by (suite) (((100 * (sum by (suite) (((clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status=~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1)) unless on(suite, check) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!~\"ok|passed|success|not_applicable|skipped|na|n/a\"})))) > 0), 1))))) / clamp_min((sum by (suite) (clamp_max(max by (suite, check) ((((sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|coverage|loc|style|docs_naming|gate_glue|sonarqube\",status!=\"\"})) or (sum by (suite, branch, check, status) (platform_quality:check_status:present_1h{suite=~\"${suite:regex}\",suite=~\"ariadne|atlasbot|bstein_home|data_prepper\",branch!=\"\",branch=~\"${branch:regex}\",check=\"supply_chain\",status!=\"\"})))) > 0), 1))), 1))) or (min by (suite) (platform_quality:test_category_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"})))) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success by Suite (24h)", + "panel_id": 9, + "panel_type": "bargauge", + "description": "24h CI run success rate. This is whether automation finished cleanly, so it can stay low after failed or aborted runs even when tests and latest gate checks are green.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort(((100 * (sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status=~\"ok|passed|success\"})) / clamp_min((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"})), 1)) and on(suite) ((sum by (suite) (platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\"})) > 0)) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage by Suite (Latest, gate 95)", + "panel_id": 17, + "panel_type": "bargauge", + "description": "Latest suite coverage; 95%+ is acceptable and 100% is strongest.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Files <=500 LOC by Suite (Latest)", + "panel_id": 18, + "panel_type": "bargauge", + "description": "Percent of managed LOC-gated files at or under 500 lines. Older suite payloads fall back to 100%/0% until they emit platform_quality_gate_source_files_total.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort(((100 * clamp_min((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) - (max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 0) / (max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"}))) and on(suite) ((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) > 0)) or on(suite) (100 * (1 - clamp_max((max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 1))) or on(suite) ((((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))) - 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "CI Run Success by Suite (7d rolling)", + "panel_id": 11, + "panel_type": "state-timeline", + "description": "Seven-day rolling CI run success rate per suite. Each suite gets its own lane, so failed or aborted runs lower the lane color without implying raw test failures.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(100 * sum by (suite) (increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\",status=~\"ok|passed|success\"}))[7d:1h])) / (sum by (suite) (increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))[7d:1h])))) and on(suite) ((sum by (suite) (increase((max without(instance, job) (platform_quality_gate_runs_total{suite=~\"${suite:regex}\",exported_job=\"platform-quality-ci\"}))[7d:1h]))) > 0)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Test Category Health History", + "panel_id": 153, + "panel_type": "state-timeline", + "description": "Health by test category from memoized hourly rollups. Use the Suite filter to focus one project; skipped tests are healthy, while failures and errors lower the lane.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|manual|performance|regression|reliability|security|smoke|system|ui|unit\"})" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Daily Run Volume (Selected Scope)", + "panel_id": 12, + "panel_type": "timeseries", + "description": "Twenty-four-hour rolling quality-gate run counts for the selected suite/branch scope. This is volume, not a pass-rate percentage.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status=~\"ok|passed|success\"}) or on() vector(0)", + "sum(platform_quality:suite_runs:increase_24h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",status!~\"ok|passed|success\"}) or on() vector(0)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage History by Suite", + "panel_id": 13, + "panel_type": "state-timeline", + "description": "Latest reported line coverage per suite over time. Coverage is separate from LOC compliance so one signal cannot hide the other.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "max by (suite) (platform_quality:suite_coverage_percent:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Files <=500 LOC History by Suite", + "panel_id": 14, + "panel_type": "state-timeline", + "description": "Percent of LOC-gated source files at or under the 500-line limit. This uses the existing file-count telemetry; longest-file history needs a new publisher metric.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(100 * clamp_min((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) - (max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 0) / (max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"}))) and on(suite) ((max by (suite) (platform_quality:suite_source_files_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})) > 0) or on(suite) (100 * (1 - clamp_max((max by (suite) (platform_quality:suite_source_lines_over_500_total:latest_1h{suite=~\"${suite:regex}\",branch=~\"${branch:regex}\"})), 1)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Tests Failure Rate", + "panel_id": 130, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage Failure Rate", + "panel_id": 131, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "LOC Failure Rate", + "panel_id": 132, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Style Failure Rate", + "panel_id": 133, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Gate Glue Failure Rate", + "panel_id": 134, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "SonarQube Failure Rate", + "panel_id": 135, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Supply Chain Failure Rate", + "panel_id": 136, + "panel_type": "state-timeline", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Tests Healthy Rate", + "panel_id": 138, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"tests|unit|build\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage Healthy Rate", + "panel_id": 139, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"coverage\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "LOC Healthy Rate", + "panel_id": 140, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"loc|smell\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Style Healthy Rate", + "panel_id": 141, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"docs|naming|hygiene|lint|docs_naming|style\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Gate Glue Healthy Rate", + "panel_id": 142, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"gate|glue|gate_glue\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "SonarQube Healthy Rate", + "panel_id": 143, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"sonarqube|sonar\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Supply Chain Healthy Rate", + "panel_id": 144, + "panel_type": "state-timeline", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"ironbank|supply_chain|image_compliance|artifact_security\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Problematic Tests Over Time (Top failures)", + "panel_id": 145, + "panel_type": "state-timeline", + "description": "Current outlier tests by rolling 24h failure count. A test needs at least two recent failures to appear, then falls off once it quiets down.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum by (suite, test) (sum_over_time(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",status=\"failed\"}[24h:1h]))) and on (suite, test) topk(12, (sum by (suite, test) (sum_over_time(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",status=\"failed\"}[24h:1h] @ end()))) >= 2)" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Most Problematic Test by Suite (7d)", + "panel_id": 147, + "panel_type": "bargauge", + "description": "Worst test per suite summed across 7d. This catches repeat offenders while keeping dashboard loads bounded; current hourly top list is quiet.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(topk by (suite) (1, (sum by (suite, test) (sum_over_time(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test!=\"__no_test_cases__\",status=\"failed\"}[7d:1h])))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Selected Test Pass/Fail History", + "panel_id": 146, + "panel_type": "timeseries", + "description": "Stacked hourly outcome volume for the selected suite/branch/test scope. This uses vmalert rollups only, avoiding expensive raw long-range per-test scans.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(sum(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\",status=\"passed\"}) or on() vector(0))", + "(sum(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\",status=\"failed\"}) or on() vector(0))", + "(sum(platform_quality:test_case_status:count_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\",status=\"skipped\"}) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Selected Test Pass Rate History", + "panel_id": 152, + "panel_type": "state-timeline", + "description": "Average pass rate per suite for the selected test filter, using memoized hourly test-case pass-rate rollups instead of raw historical scans.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "avg by (suite) (platform_quality:test_case_pass_rate:percent_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",test!=\"\",test=~\"${test:regex}\",test!=\"__no_test_cases__\"})" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Tests Metrics Present by Suite", + "panel_id": 27, + "panel_type": "bargauge", + "description": "Whether suite-level test counts are present; 100% means the suite is reporting.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) ({__name__=~\".*_quality_gate_tests_total\",exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Checks Metrics Present by Suite", + "panel_id": 28, + "panel_type": "bargauge", + "description": "Whether gate check metrics are present; 100% means health panels have inputs.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) ({__name__=~\".*_quality_gate_checks_total\",exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Coverage Metrics Present by Suite", + "panel_id": 29, + "panel_type": "bargauge", + "description": "Whether coverage metrics are present; 100% means coverage panels are reliable.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_workspace_line_coverage_percent{exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "LOC Compliance Metrics Present by Suite", + "panel_id": 30, + "panel_type": "bargauge", + "description": "Whether LOC metrics are present; 100% means size panels are reliable.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_source_lines_over_500_total{exported_job=\"platform-quality-ci\"}) and on(suite) count by (suite) (platform_quality_gate_source_files_total{exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Test-Case Metrics Present by Suite", + "panel_id": 148, + "panel_type": "bargauge", + "description": "Whether per-test metrics are present; 100% enables drilldowns.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_test_case_result{exported_job=\"platform-quality-ci\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Real Test Cases Present by Suite", + "panel_id": 151, + "panel_type": "bargauge", + "description": "Whether real test names are present; 100% means not just placeholder telemetry.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\")) and on(suite) count by (suite) (platform_quality_gate_test_case_result{exported_job=\"platform-quality-ci\",test!=\"__no_test_cases__\"})))) or on(suite) (0 * (label_replace(vector(1), \"suite\", \"ariadne\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"metis\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"ananke\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"atlasbot\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"pegasus\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"soteria\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"titan_iac\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"bstein_home\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"data_prepper\", \"__name__\", \".*\") or label_replace(vector(1), \"suite\", \"lesavka\", \"__name__\", \".*\"))))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Primary Branch Clean by Suite (7d)", + "panel_id": 150, + "panel_type": "bargauge", + "description": "Percent clean of non-primary branch evidence; 100% means only main/master is reporting.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort((100 * (((count by (suite) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}[7d:1h]))) > bool 0) unless on(suite) ((count by (suite) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\",branch!~\"main|master|origin/main|origin/master|unknown\"}[7d:1h]))) > bool 0))) or on(suite) (0 * ((count by (suite) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}[7d:1h]))) > bool 0)))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Recent Branch Evidence by Suite (7d)", + "panel_id": 149, + "panel_type": "bargauge", + "description": "Branches with recent CI evidence; unexpected branches can mean drift or stale work.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "sort_desc(count by (suite, branch) (max_over_time(platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}[7d:1h])))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "SonarQube API Up", + "panel_id": 31, + "panel_type": "stat", + "description": "Whether the SonarQube exporter can reach SonarQube; 1 is good.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(max(sonarqube_up) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Projects (Selected)", + "panel_id": 32, + "panel_type": "stat", + "description": "Selected SonarQube project count; zero means Sonar is not tracking that suite.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(count(max by (project_key) (sonarqube_project_quality_gate_pass{project_key=~\"${suite:regex}\"})) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Gate Fetch Errors", + "panel_id": 33, + "panel_type": "stat", + "description": "Sonar exporter fetch errors; zero is good because stale Sonar data misleads.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "(max(sonarqube_quality_gate_fetch_errors_total) or on() vector(0))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Gate Status Mix (Selected)", + "panel_id": 34, + "panel_type": "piechart", + "description": "Mix of Sonar gate states; OK is good and non-OK needs cleanup.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "count by (status) (max by (project_key, status) (sonarqube_project_quality_gate_pass{project_key=~\"${suite:regex}\"}))" + ] + }, + { + "dashboard": "Atlas Testing", + "panel_title": "Sonar Gate Health by Project", + "panel_id": 35, + "panel_type": "state-timeline", + "description": "SonarQube gate status over time by project. OK projects render as full healthy lanes; non-OK projects drop to red without disappearing.", + "tags": [ + "atlas", + "testing", + "quality-gate", + "ci" + ], + "datasource_uid": "atlas-vm", + "datasource_type": "prometheus", + "exprs": [ + "platform_quality:sonar_gate_health_percent:latest_1h{project_key=~\"${suite:regex}\"}" + ] } ] diff --git a/services/comms/knowledge/catalog/runbooks.json b/services/comms/knowledge/catalog/runbooks.json index 960510d21..3a34ac883 100644 --- a/services/comms/knowledge/catalog/runbooks.json +++ b/services/comms/knowledge/catalog/runbooks.json @@ -15,10 +15,18 @@ "source_paths": [ "services/gitea", "services/jenkins", - "scripts/jenkins_cred_sync.sh", - "scripts/gitea_cred_sync.sh" + "scripts/sync/jenkins_cred_sync.sh", + "scripts/sync/gitea_cred_sync.sh" ], - "body": "# CI: Gitea \u2192 Jenkins pipeline\n\n## What this is\nAtlas uses Gitea for source control and Jenkins for CI. Authentication is via Keycloak (SSO).\n\n## Where it is configured\n- Gitea manifests: `services/gitea/`\n- Jenkins manifests: `services/jenkins/`\n- Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh`\n\n## What users do (typical flow)\n- Create a repo in Gitea.\n- Create/update a Jenkins job/pipeline that can fetch the repo.\n- Configure a webhook (or SCM polling) so pushes trigger builds.\n\n## Troubleshooting (common)\n- \u201cWebhook not firing\u201d: confirm ingress host, webhook URL, and Jenkins job is reachable.\n- \u201cAuth denied cloning\u201d: confirm Keycloak group membership and that Jenkins has a valid token/credential configured." + "body": "# CI: Gitea \u2192 Jenkins pipeline\n\n## What this is\nAtlas uses Gitea for source control and Jenkins for CI. Authentication is via Keycloak (SSO).\n\n## Where it is configured\n- Gitea manifests: `services/gitea/`\n- Jenkins manifests: `services/jenkins/`\n- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh`\n\n## What users do (typical flow)\n- Create a repo in Gitea.\n- Create/update a Jenkins job/pipeline that can fetch the repo.\n- Configure a webhook (or SCM polling) so pushes trigger builds.\n\n## Troubleshooting (common)\n- \u201cWebhook not firing\u201d: confirm ingress host, webhook URL, and Jenkins job is reachable.\n- \u201cAuth denied cloning\u201d: confirm Keycloak group membership and that Jenkins has a valid token/credential configured." + }, + { + "path": "runbooks/cluster-power-recovery.md", + "title": "cluster-power-recovery", + "tags": [], + "entrypoints": [], + "source_paths": [], + "body": "Atlas Cluster Power Recovery (Graceful Shutdown/Startup)\n\nPurpose\n- Provide a safe operator flow for planned power events and cold-boot recovery.\n- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path.\n- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle.\n- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery.\n\nBootstrapping risk to remember\n- Flux source is Git over SSH to `scm.bstein.dev` (Gitea).\n- Gitea itself is a Flux-managed workload and depends on storage + database.\n- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images.\n- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall.\n- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back.\n\nScript\n- `scripts/ops/cluster_power_recovery.sh`\n- `scripts/ops/cluster_power_console.sh`\n- Modes:\n - `prepare`\n - `shutdown`\n - `harbor-seed`\n - `startup`\n - `status`\n- Default is dry-run. Add `--execute` to actually perform actions.\n\nDry-run examples\n- Shutdown preview:\n - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain`\n- Startup preview:\n - `scripts/ops/cluster_power_recovery.sh startup`\n- Harbor seed preview:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed`\n\nExecute examples\n- Prepare helper image on every node:\n - `scripts/ops/cluster_power_recovery.sh prepare --execute`\n- Seed Harbor runtime images onto `titan-05` from the control-host bundle:\n - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute`\n- Planned shutdown:\n - `scripts/ops/cluster_power_recovery.sh shutdown --execute`\n- Planned startup (canonical branch):\n - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main`\n\nManual remote console examples\n- Canonical operator hosts:\n - `titan-db`\n - `tethys` (`titan-24`)\n- Both hosts now have:\n - `~/ananke-tools/cluster_power_recovery.sh`\n - `~/ananke-tools/cluster_power_console.sh`\n - `~/ananke-tools/bootstrap/recovery-config.env`\n - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt`\n - `~/ananke-tools/kubeconfig`\n - `~/ananke-cluster-power`\n - `~/bin/ananke-cluster-power`\n - `~/ananke-repo/{infrastructure,services,scripts}`\n- Both hosts also keep the Harbor bootstrap bundle at:\n - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n- Remote usage:\n - `ssh titan-db`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n - `ssh tethys`\n - `~/ananke-cluster-power status`\n - `~/ananke-cluster-power prepare --execute`\n - `~/ananke-cluster-power shutdown --execute`\n - `~/ananke-cluster-power startup --execute --force-flux-branch main`\n\nUseful options\n- `--shutdown-mode host-poweroff|cluster-only`\n- `--expected-flux-branch main`\n- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git`\n- `--force-flux-branch main`\n- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only)\n- `--skip-local-bootstrap` (not recommended for cold-start recovery)\n- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred)\n- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node)\n- `--skip-helper-prewarm`\n- `--min-startup-battery 35`\n- `--ups-host pyrphoros@localhost`\n- `--require-ups-battery`\n- `--drain-timeout 180`\n- `--emergency-drain-timeout 45`\n- `--flux-ready-timeout 1200`\n- `--startup-checklist-timeout 900`\n- `--startup-stability-window 180`\n- `--startup-stability-timeout 900`\n- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state`\n- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`\n\nControlled drill checklist (recommended)\n- Operator host: use `titan-db` as canonical control host for the drill.\n- On-site coordination:\n - Have on-site operator ready before shutdown starts.\n - Confirm they will manually power cluster nodes back on after shutdown completes.\n - Confirm who will announce \"all nodes powered on\" to resume startup.\n- Preflight on `titan-db`:\n - `mkdir -p ~/ananke-logs`\n - `~/ananke-cluster-power status` and verify:\n - `ups_host=pyrphoros@localhost`\n - `ups_battery` is numeric\n - `flux_source_ready=True`\n- Warm helper image just before shutdown:\n - `~/ananke-cluster-power prepare --execute`\n- Run in a persistent shell and capture logs:\n - `tmux new -s ananke-drill`\n - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log`\n- Execute controlled shutdown with telemetry enforcement:\n - `~/ananke-cluster-power shutdown --execute --require-ups-battery`\n- After on-site power-on confirmation, execute startup:\n - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery`\n- Post-check:\n - `~/ananke-cluster-power status`\n - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures.\n\nOperational notes\n- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn.\n- Shutdown behavior is explicit:\n - `host-poweroff` schedules host poweroff after service stop.\n - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off.\n- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted.\n- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`).\n- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo.\n- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`.\n- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer.\n- Harbor is reconciled after the first critical stateful services.\n- Harbor bootstrap is now designed around a control-host bundle:\n - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`.\n - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`.\n - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`.\n- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations.\n- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls.\n- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap.\n- Startup completion is strict now:\n - all non-optional Flux kustomizations must be `Ready=True`\n - external service checklist must pass (defaults include Gitea, Grafana, Harbor)\n - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`)\n - stability soak must pass with no crashloop/pull-failure churn\n- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile.\n- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster.\n- Dry-run mode no longer mutates outage recovery state.\n- `harbor-seed --execute` was validated by:\n - prewarming the helper image across all nodes\n - streaming the Harbor bootstrap bundle to `titan-05`\n - importing Harbor runtime images into host `containerd`\n - successfully running a Harbor-backed canary pod (`harbor-canary-ok`)\n- After bootstrap, Flux resources are resumed and reconciled.\n- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`." }, { "path": "runbooks/comms-verify.md", @@ -47,7 +55,7 @@ "entrypoints": [], "source_paths": [ "knowledge/runbooks", - "scripts/knowledge_render_atlas.py" + "scripts/render/knowledge_render_atlas.py" ], "body": "# KB authoring: what to write (and what not to)\n\n## The goal\nGive Atlas assistants enough grounded, Atlas-specific context to answer \u201chow do I\u2026?\u201d questions without guessing.\n\n## What to capture (high value)\n- User workflows: \u201cclick here, set X, expected result\u201d\n- Operator workflows: \u201cedit these files, reconcile this kustomization, verify with these commands\u201d\n- Wiring: \u201cthis host routes to this service; this service depends on Postgres/Vault/etc\u201d\n- Failure modes: exact error messages + the 2\u20135 checks that usually resolve them\n- Permissions: Keycloak groups/roles and what they unlock\n\n## What to avoid (low value / fluff)\n- Generic Kubernetes explanations (link to upstream docs instead)\n- Copy-pasting large manifests (prefer file paths + small snippets)\n- Anything that will drift quickly (render it from GitOps instead)\n- Any secret values (reference Secret/Vault locations by name only)\n\n## Document pattern (recommended)\nEach runbook should answer:\n- \u201cWhat is this?\u201d\n- \u201cWhat do users do?\u201d\n- \u201cWhat do operators change (where in Git)?\u201d\n- \u201cHow do we verify it works?\u201d\n- \u201cWhat breaks and how to debug it?\u201d" }, @@ -92,6 +100,6 @@ "tags": [], "entrypoints": [], "source_paths": [], - "body": "# Metis (node recovery)\n\n## Node classes (current map)\n- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent)\n- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)\n- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)\n- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)\n- amd64 agents: titan-22/24 (Debian 13, k3s agent)\n- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers.\n\n### Jetson nodes (titan-20/21)\n- Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.\n- Storage: NVMe 232G at / (ext4); onboard mmc partitions present but root on NVMe; 1.9T sda present (unused).\n- k3s agent with drop-in 99-nofile.conf.\n\n## Longhorn disk UUIDs (critical nodes)\n- titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)\n- titan-15: /mnt/astreae UUID=f3362f14-5822-449f-944b-ac570b5cd615 (ext4); /mnt/asteria UUID=9c5316e6-f847-4884-b502-11f2d0d15d6f (ext4)\n- titan-17: /mnt/astreae UUID=1fecdade-08b0-49cb-9ae3-be6c188b0a96 (ext4); /mnt/asteria UUID=2fe9f613-d372-47ca-b84f-82084e4edda0 (ext4)\n- titan-19: /mnt/astreae UUID=4890abb9-dda2-4f4f-9c0f-081ee82849cf (ext4); /mnt/asteria UUID=2b4ea28d-b0e6-4fa3-841b-cd7067ae9153 (ext4)\n\n## Metis repo (~/Development/metis)\n- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).\n- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).\n- `AGENTS.md` in repo is untracked and holds raw notes.\n\n## Next implementation steps\n- Add per-class golden image refs and checksums (Harbor or file://) when ready.\n- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.\n- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.\n- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited.\n\n## Node OS/Kernel/CRI snapshot (Jan 2026)\n- titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-05: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-06: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-07: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-08: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-09: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0a: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0b: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0c: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-10: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-11: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-12: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-13: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-14: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-15: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-17: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-18: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-19: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-20: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-21: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-22: Debian 13 (trixie), kernel 6.12.41+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n- titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n\n\n### External hosts\n- titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.\n- titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).\n- titan-23/oceanus: TODO audit (future).\n\n\n### Control plane Pis (titan-0a/0b/0c)\n- Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, containerd 2.0.5-k3s2.\n- Storage: 477G SSD root (/dev/sda2 ext4), /boot/firmware vfat (/dev/sda1). fstab uses LABEL=writable and LABEL=system-boot.\n- k3s server (control-plane taint expected); etcd snapshots not yet cataloged (TODO).\n\n\n## k3s versions\n- rpi5 workers/control-plane: k3s v1.33.3+k3s1 (crictl v1.31.0-k3s2)\n- rpi4 nodes: k3s v1.31.5+k3s1 (crictl v1.31.0-k3s2)\n- Jetson titan-20/21: k3s v1.33.3+k3s1 (per node info), crictl v1.31.0-k3s2" + "body": "# Metis (node recovery)\n\n## Node classes (current map)\n- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent)\n- rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint)\n- rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks)\n- rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent)\n- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent)\n- amd64 agents: titan-22/24 (Debian 13, k3s agent)\n- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels)\n- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers.\n\n### Jetson nodes (titan-20/21)\n- Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64.\n- Storage: NVMe 232G at / (ext4); onboard mmc partitions present but root on NVMe; 1.9T sda present (unused).\n- k3s agent with drop-in 99-nofile.conf.\n\n## Longhorn disk UUIDs (critical nodes)\n- titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4)\n- titan-15: /mnt/astreae UUID=f3362f14-5822-449f-944b-ac570b5cd615 (ext4); /mnt/asteria UUID=9c5316e6-f847-4884-b502-11f2d0d15d6f (ext4)\n- titan-17: /mnt/astreae UUID=1fecdade-08b0-49cb-9ae3-be6c188b0a96 (ext4); /mnt/asteria UUID=2fe9f613-d372-47ca-b84f-82084e4edda0 (ext4)\n- titan-19: /mnt/astreae UUID=4890abb9-dda2-4f4f-9c0f-081ee82849cf (ext4); /mnt/asteria UUID=2b4ea28d-b0e6-4fa3-841b-cd7067ae9153 (ext4)\n\n## Metis repo (~/Development/metis)\n- CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`).\n- `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints).\n\n## Next implementation steps\n- Add per-class golden image refs and checksums (Harbor or file://) when ready.\n- Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths.\n- Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection.\n- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited.\n\n## Node OS/Kernel/CRI snapshot (Jan 2026)\n- titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-05: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-06: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-07: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-08: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-09: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0a: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0b: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-0c: Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-10: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-11: Ubuntu 24.04.3 LTS, kernel 6.8.0-1039-raspi, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-12: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-13: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-14: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-15: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-17: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-18: Armbian 24.11.1 noble, kernel 6.6.60-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-19: Armbian 25.2.1 noble, kernel 6.6.63-current-bcm2711, CRI containerd://1.7.23-k3s2, arch arm64\n- titan-20: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-21: Ubuntu 20.04.6 LTS, kernel 5.10.104-tegra, CRI containerd://2.0.5-k3s2, arch arm64\n- titan-22: Debian 13 (trixie), kernel 6.12.41+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n- titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64\n\n\n### Dedicated and special-purpose hosts\n- titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled.\n- titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q).\n- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement.\n\n\n### Control plane Pis (titan-0a/0b/0c)\n- Ubuntu 24.04.1 LTS, kernel 6.8.0-1038-raspi, containerd 2.0.5-k3s2.\n- Storage: 477G SSD root (/dev/sda2 ext4), /boot/firmware vfat (/dev/sda1). fstab uses LABEL=writable and LABEL=system-boot.\n- k3s server (control-plane taint expected); etcd snapshots not yet cataloged (TODO).\n\n\n## k3s versions\n- rpi5 workers/control-plane: k3s v1.33.3+k3s1 (crictl v1.31.0-k3s2)\n- rpi4 nodes: k3s v1.31.5+k3s1 (crictl v1.31.0-k3s2)\n- Jetson titan-20/21: k3s v1.33.3+k3s1 (per node info), crictl v1.31.0-k3s2" } ] diff --git a/services/comms/knowledge/diagrams/atlas-http.mmd b/services/comms/knowledge/diagrams/atlas-http.mmd index 1aa7ac804..4c4c704cc 100644 --- a/services/comms/knowledge/diagrams/atlas-http.mmd +++ b/services/comms/knowledge/diagrams/atlas-http.mmd @@ -1,9 +1,19 @@ flowchart LR + host_agent_bstein_dev["agent.bstein.dev"] + svc_hermes_hermes["hermes/hermes (Service)"] + host_agent_bstein_dev --> svc_hermes_hermes + wl_hermes_hermes["hermes/hermes (Deployment)"] + svc_hermes_hermes --> wl_hermes_hermes host_auth_bstein_dev["auth.bstein.dev"] svc_sso_oauth2_proxy["sso/oauth2-proxy (Service)"] host_auth_bstein_dev --> svc_sso_oauth2_proxy wl_sso_oauth2_proxy["sso/oauth2-proxy (Deployment)"] svc_sso_oauth2_proxy --> wl_sso_oauth2_proxy + host_backup_bstein_dev["backup.bstein.dev"] + svc_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Service)"] + host_backup_bstein_dev --> svc_maintenance_oauth2_proxy_soteria + wl_maintenance_oauth2_proxy_soteria["maintenance/oauth2-proxy-soteria (Deployment)"] + svc_maintenance_oauth2_proxy_soteria --> wl_maintenance_oauth2_proxy_soteria host_bstein_dev["bstein.dev"] svc_bstein_dev_home_bstein_dev_home_frontend["bstein-dev-home/bstein-dev-home-frontend (Service)"] host_bstein_dev --> svc_bstein_dev_home_bstein_dev_home_frontend @@ -111,6 +121,16 @@ flowchart LR host_pegasus_bstein_dev --> svc_jellyfin_pegasus wl_jellyfin_pegasus["jellyfin/pegasus (Deployment)"] svc_jellyfin_pegasus --> wl_jellyfin_pegasus + host_quality_bstein_dev["quality.bstein.dev"] + svc_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Service)"] + host_quality_bstein_dev --> svc_quality_oauth2_proxy_sonarqube + wl_quality_oauth2_proxy_sonarqube["quality/oauth2-proxy-sonarqube (Deployment)"] + svc_quality_oauth2_proxy_sonarqube --> wl_quality_oauth2_proxy_sonarqube + host_recovery_bstein_dev["recovery.bstein.dev"] + svc_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Service)"] + host_recovery_bstein_dev --> svc_maintenance_oauth2_proxy_metis + wl_maintenance_oauth2_proxy_metis["maintenance/oauth2-proxy-metis (Deployment)"] + svc_maintenance_oauth2_proxy_metis --> wl_maintenance_oauth2_proxy_metis host_scm_bstein_dev["scm.bstein.dev"] svc_gitea_gitea["gitea/gitea (Service)"] host_scm_bstein_dev --> svc_gitea_gitea @@ -141,6 +161,20 @@ flowchart LR host_vault_bstein_dev --> svc_vaultwarden_vaultwarden_service wl_vaultwarden_vaultwarden["vaultwarden/vaultwarden (Deployment)"] svc_vaultwarden_vaultwarden_service --> wl_vaultwarden_vaultwarden + host_veles_bstein_dev["veles.bstein.dev"] + svc_veles_veles_frontend["veles/veles-frontend (Service)"] + host_veles_bstein_dev --> svc_veles_veles_frontend + wl_veles_veles_frontend["veles/veles-frontend (Deployment)"] + svc_veles_veles_frontend --> wl_veles_veles_frontend + svc_veles_veles_backend["veles/veles-backend (Service)"] + host_veles_bstein_dev --> svc_veles_veles_backend + wl_veles_veles_backend["veles/veles-backend (Deployment)"] + svc_veles_veles_backend --> wl_veles_veles_backend + host_wolf_bstein_dev["wolf.bstein.dev"] + svc_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Service)"] + host_wolf_bstein_dev --> svc_game_stream_oauth2_proxy_wolf + wl_game_stream_oauth2_proxy_wolf["game-stream/oauth2-proxy-wolf (Deployment)"] + svc_game_stream_oauth2_proxy_wolf --> wl_game_stream_oauth2_proxy_wolf subgraph bstein_dev_home[bstein-dev-home] svc_bstein_dev_home_bstein_dev_home_frontend @@ -175,6 +209,10 @@ flowchart LR svc_finance_firefly wl_finance_firefly end + subgraph game_stream[game-stream] + svc_game_stream_oauth2_proxy_wolf + wl_game_stream_oauth2_proxy_wolf + end subgraph gitea[gitea] svc_gitea_gitea wl_gitea_gitea @@ -183,6 +221,10 @@ flowchart LR svc_health_wger wl_health_wger end + subgraph hermes[hermes] + svc_hermes_hermes + wl_hermes_hermes + end subgraph jellyfin[jellyfin] svc_jellyfin_pegasus wl_jellyfin_pegasus @@ -204,6 +246,12 @@ flowchart LR subgraph mailu_mailserver[mailu-mailserver] svc_mailu_mailserver_mailu_front end + subgraph maintenance[maintenance] + svc_maintenance_oauth2_proxy_soteria + wl_maintenance_oauth2_proxy_soteria + svc_maintenance_oauth2_proxy_metis + wl_maintenance_oauth2_proxy_metis + end subgraph nextcloud[nextcloud] svc_nextcloud_nextcloud wl_nextcloud_nextcloud @@ -218,6 +266,10 @@ flowchart LR svc_planka_planka wl_planka_planka end + subgraph quality[quality] + svc_quality_oauth2_proxy_sonarqube + wl_quality_oauth2_proxy_sonarqube + end subgraph sso[sso] svc_sso_oauth2_proxy wl_sso_oauth2_proxy @@ -232,3 +284,9 @@ flowchart LR svc_vaultwarden_vaultwarden_service wl_vaultwarden_vaultwarden end + subgraph veles[veles] + svc_veles_veles_frontend + wl_veles_veles_frontend + svc_veles_veles_backend + wl_veles_veles_backend + end diff --git a/services/comms/knowledge/hermes-automated-triage-24h-plan.md b/services/comms/knowledge/hermes-automated-triage-24h-plan.md new file mode 100644 index 000000000..2daf232dc --- /dev/null +++ b/services/comms/knowledge/hermes-automated-triage-24h-plan.md @@ -0,0 +1,985 @@ +# Hermes Automated Triage: 24-Hour Delivery Plan + +- Status: planning only +- Created: 2026-08-05 +- Hard delivery limit: 24 elapsed hours from execution authorization +- Priority: deadline first, then the most complete safe automation that fits + +## Executive decision + +The first delivery will automate one narrow, real, repeatable failure-to-repair path. It will not attempt general autonomous coding before that path works end to end. + +The target demonstration is: + +```text +Controlled test failure + -> Jenkins build fails + -> Ariadne detects the terminal failure automatically + -> Ariadne gathers Jenkins evidence and bounded OpenSearch logs + -> Ariadne automatically invokes the Hermes Agent + -> Hermes selects its Titan test-triage skills + -> Hermes returns a structured diagnosis and requested action + -> Ariadne validates the request against an exact allowlist + -> Ariadne creates one predefined repair Job + -> Ariadne triggers one Jenkins rebuild + -> the rebuild passes and the incident resolves +``` + +A second path will demonstrate safe escalation: + +```text +Unsupported, ambiguous or unsafe failure + -> Hermes returns human_required + -> Ariadne publishes a triage metric + -> VictoriaMetrics and vmalert evaluate it + -> Grafana and Alertmanager provide the human-facing signal +``` + +At H+24 work stops. Lower-priority functionality is removed or deferred rather than extending the deadline. + +## Definition of done + +The delivery is fully successful when all of the following are demonstrated: + +- A deliberately seeded test failure causes a Jenkins build to fail. +- Ariadne detects that exact terminal build once, preferably within two minutes. +- Ariadne assigns a stable incident ID based on the Jenkins job and build number. +- The diagnosis bundle includes the first failed stage, retained Jenkins evidence and provenance-bearing OpenSearch excerpts. +- Ariadne invokes the Hermes Agent without a human prompt. +- The invocation explicitly selects the Titan test-triage skill. +- Hermes returns schema-valid facts, inferences, classification, confidence and action request. +- Ariadne accepts only the predefined demo action and executes it once. +- A predefined repair Job corrects only the isolated demo fixture. +- Ariadne triggers one rebuild with failure seeding disabled. +- The rebuilt job passes and Ariadne records the incident as resolved. +- An unsupported classification produces a human-required metric and no mutation. +- No secret values are stored in evidence, model prompts, responses or logs. +- No path can automatically write to `main`, reconcile Flux, or execute model-supplied commands. +- A single Flux-managed setting can disable all automatic actions immediately. + +Target demo timings are: + +- Failure detection: less than two minutes. +- Evidence collection and Hermes response: less than three minutes. +- Repair launch and rebuild request: less than two minutes. +- Complete failure-to-green loop: less than ten minutes, excluding image pulls during a cold cluster start. + +## Current system context + +This section records the facts needed to execute the plan without relying on prior conversation. + +### Jenkins and the homegrown suites + +Jenkins runs the homegrown repositories on ephemeral Kubernetes agent pods. The active quality scope includes Ariadne, Metis, Ananke, Atlasbot, Pegasus, Soteria, titan-iac, bstein-dev-home, data-prepper and Lesavka, with additional build/test jobs for projects such as Arcanagon, Typhon and Veles. + +The normalized quality evidence includes, as applicable: + +```text +build + -> style/docs + -> LOC/naming + -> coverage + -> tests + -> gate glue + -> SonarQube + -> Semgrep + -> Trivy/supply-chain evidence + -> gate enforcement + -> image build and Harbor push +``` + +Jenkins retains build metadata, console output, stage results, JUnit, coverage and quality reports. These retained artifacts are the authoritative source for the exact failed test or gate. + +### Logging and OpenSearch + +OpenSearch does not collect logs by itself. Fluent Bit tails Kubernetes container logs and writes them directly into `kube-*` indices. It also writes node journal records into `journald-*` indices. + +The relevant path is: + +```text +Kubernetes pod stdout/stderr + -> Fluent Bit DaemonSet + -> OpenSearch kube-* indices +``` + +Jenkins-retained console logs and OpenSearch pod logs are separate evidence paths. Jenkins agent command output must not be assumed to exist in CRI container logs. The demo must use Jenkins artifacts for the exact assertion and must ensure that a dedicated demo test-runner or related workload writes a correlated incident message to pod stdout/stderr for OpenSearch. + +Data Prepper is not part of the ordinary log path. It receives OpenTelemetry traces, transforms them and writes trace/service-map indices into OpenSearch. It is irrelevant to the Jenkins log integration except as another separately tested workload. + +Relevant IaC sources include: + +- `services/logging/fluent-bit-helmrelease.yaml` +- `services/logging/opensearch-observability-objects.yaml` +- `services/logging/Jenkinsfile.data-prepper` + +### Metrics and human-facing status + +Jenkins publishers and runtime probes send normalized quality metrics to `platform-quality-gateway`. VictoriaMetrics scrapes Prometheus-compatible endpoints, vmalert evaluates rules, Grafana displays the Atlas Testing dashboards, and Alertmanager owns notification delivery. + +The intended human-escalation path is therefore: + +```text +Ariadne triage metric + -> VictoriaMetrics + -> vmalert + -> Alertmanager notification and Grafana status +``` + +Hermes does not send an alert "to Grafana." Grafana visualizes the metric and alert state. + +Relevant IaC sources include: + +- `services/monitoring/platform-quality-gateway-deployment.yaml` +- `services/monitoring/platform-quality-suite-probe-cronjob.yaml` +- `services/monitoring/vmalert-atlas-availability.yaml` +- `services/monitoring/dashboards/atlas-testing.json` +- `services/quality/sonarqube-exporter-deployment.yaml` + +### Ariadne today + +Ariadne is the deterministic automation and evidence-collection service in the `maintenance` namespace. Its current deployment has access to: + +- Jenkins API credentials and `JENKINS_BASE_URL`. +- VictoriaMetrics through `ARIADNE_VM_URL`. +- OpenSearch through `OPENSEARCH_URL`. +- Kubernetes through its service account. +- The local model through `ARIADNE_TESTING_TRIAGE_MODEL_URL`. + +The deployment schedules Jenkins build-weather collection every ten minutes and testing triage every fifteen minutes. Those intervals are too slow for a smooth demo and should be reduced only for the allowlisted demo path. + +Ariadne's current OpenSearch integration manages index retention. It does not currently retrieve historical log content for testing diagnosis. + +Ariadne currently points model-assisted testing triage at the Hermes Ollama service. Calling Ollama uses the same Qwen model but bypasses the Hermes Agent, its tool loop, skills, memory and approval model. This is not equivalent to invoking Hermes. + +Ariadne already has patterns for launching a platform quality probe Job and for performing Jenkins workspace maintenance. These patterns should be reused for the predefined demo repair rather than introducing a general job-execution system. + +Relevant IaC source: + +- `services/maintenance/apps/ariadne-deployment.yaml` + +Relevant application repository: + +- Gitea: `bstein/Ariadne` +- Local checkout currently at `/home/brad/Development/Ariadne` + +### Hermes today + +Hermes runs as a single read-only triage agent in the `hermes` namespace. Its current configuration provides: + +- Hermes Agent API on port `8642`. +- Hermes dashboard on port `9119`. +- Local Qwen 2.5 `7b-instruct-q4_0` inference through Hermes Ollama. +- Ariadne, Jenkins, VictoriaMetrics, Gitea and Grafana base URLs. +- `kubectl` installed as a read-only cluster inspection tool. +- A persistent home PVC containing memories and runtime-installed skills. + +Hermes does not currently receive `OPENSEARCH_URL`. In the desired design it does not need direct OpenSearch access: Ariadne will provide bounded, sanitized excerpts in the diagnosis bundle. + +Hermes's Flux-managed instructions say to start with Ariadne as the source of truth, then use Jenkins, quality metrics, Flux, Grafana and Kubernetes read-only evidence. Its RBAC allows reads of workloads, events, pod logs, services, storage objects and Flux resources. It cannot read Kubernetes Secret values. + +Hermes explicitly denies or lacks permission for actions such as: + +- `kubectl apply`, delete, patch, scale, cordon, drain or rollout restart. +- Flux suspend, resume or reconcile. +- Vault secret reads. +- Kubernetes Secret reads. +- Jenkins rebuilds. +- Git writes. + +This boundary remains in place. Hermes will request actions; Ariadne will validate and execute exact predefined actions. + +Relevant IaC sources include: + +- `services/hermes/deployment.yaml` +- `services/hermes/configmap.yaml` +- `services/hermes/rbac.yaml` + +### Hermes's current Atlas skills + +A read-only inventory of the live Hermes PVC on 2026-08-05 showed the Atlas triage entry skills: + +- `triage-titan-test-failures` +- `triage-atlas-service-health` +- `tune-atlas-alerts` +- `master-hermes-on-atlas` + +The test-triage skill pack includes: + +- `titan-triage-orchestrator` +- `jenkins-retained-evidence` +- `platform-quality-metrics` +- `kubernetes-readonly-failure-classifier` +- `flux-git-change-correlation` +- `grafana-metric-provenance` +- `triage-evidence-reporting` +- `approved-triage-actions` +- `soteria-backup-health` + +The orchestrator already defines the needed evidence order: Ariadne first, then the smallest relevant specialist, followed by a fact-versus-inference report. The demo invocation should explicitly request `$triage-titan-test-failures`; a new general triage framework is unnecessary. + +These skills currently live on the runtime PVC and are not all declared in `titan-iac`. The demo should not depend on mutating the live PVC manually. Any essential prompt or output contract must be supplied by Ariadne or added through Flux-managed Hermes configuration. + +### Current gaps + +The desired closed loop does not exist today because: + +- Ariadne does not query OpenSearch for triage log evidence. +- Ariadne calls Ollama rather than the Hermes Agent API. +- There is no shared Ariadne-to-Hermes API authentication contract. +- There is no schema for a Hermes triage response. +- There is no allowlisted Hermes-to-Ariadne action request. +- Ariadne does not automatically execute a Hermes-selected repair. +- There is no incident-level idempotency preventing duplicate repair attempts. +- There is no dedicated human-required triage metric and alert. +- The current architecture chart compresses Hermes's internal trigger, routing and decision steps into a single box. + +## Worktree and source-control safety + +The existing working directories must not be used as clean implementation bases. + +As of 2026-08-05: + +- `/home/brad/Development/titan-iac` has extensive unrelated modified, deleted and untracked user work, including an ongoing repository-layout migration and untracked Mermaid files. +- `/home/brad/Development/Ariadne` is dirty on branch `codex/ariadne-metrics-hotfix`. + +Execution must: + +1. Inspect the actual revisions deployed by Flux and Jenkins. +2. Fetch without altering either existing worktree. +3. Create dedicated clean Git worktrees for the demo branches. +4. Never reset, clean, stash, rebase or overwrite either existing dirty worktree. +5. Keep Ariadne application changes and titan-iac/Flux changes in separately reviewable commits. +6. Bring over only the intentional planning/diagram change when appropriate. + +Suggested isolated paths are: + +```text +/home/brad/Development/_worktrees/ariadne-hermes-triage-demo +/home/brad/Development/_worktrees/titan-iac-hermes-triage-demo +``` + +The implementation branch point must be the current authoritative primary/deployed revision, not an assumed local branch. + +## Deadline-driven scope + +### Required scope + +- One dedicated demo Jenkins job. +- One deterministic failure signature. +- One isolated demo fixture. +- One bounded OpenSearch search contract. +- One automatic Hermes invocation. +- One structured decision contract. +- One exact repair action. +- One rebuild attempt. +- One human-required metric and alert path. +- One complete source-level Mermaid update. + +### Explicit non-goals + +- General conversational OpenSearch access. +- Arbitrary index selection or unbounded log retrieval. +- General autonomous repository editing. +- Direct commits or merges to `main`. +- Model-generated Kubernetes manifests or commands. +- Automatic Flux reconciliation. +- Automatic SonarQube, Semgrep or Trivy waivers. +- Secret inspection. +- Multiple repair retries. +- Multiple repositories or failure signatures before the primary loop is green. +- A permanent generalized incident-management platform. + +## Demo design + +### Isolated test fixture + +Use a dedicated `hermes-triage-demo` namespace, Jenkins job and fixture. The preferred deterministic design is: + +1. A small PVC stores a single state value such as `healthy` or `unhealthy`. +2. The demo is explicitly armed through a Jenkins parameter such as `SEED_FAILURE=true`. +3. A dedicated Kubernetes test-runner Job reads the fixture. +4. When the fixture is unhealthy, it writes a structured incident line to stdout/stderr and exits nonzero. +5. Jenkins waits for the Job, records the test failure and archives a small JUnit result. +6. Fluent Bit forwards the test-runner Job output to OpenSearch. +7. The test-runner Job name and message include the Jenkins job/build incident ID. + +Example structured log: + +```json +{ + "event": "hermes_demo_test_failure", + "incident_id": "hermes-triage-demo/42", + "classification_hint": "demo_fixture_unhealthy", + "message": "expected fixture state healthy; found unhealthy" +} +``` + +The Jenkins rebuild must set `SEED_FAILURE=false`, otherwise every rebuild would deliberately recreate the failure. + +The namespace isolates the test-runner permissions and prevents a demo repair from touching production workloads. + +### Predefined repair Job + +Ariadne will contain one exact action mapping: + +```text +repair_demo_fixture + -> create hermes-demo-repair- Job + -> mount only the demo fixture PVC + -> write healthy state + -> emit a structured repair result + -> exit +``` + +Hermes cannot supply the image, command, namespace, PVC, labels or arbitrary parameters. It can request only the action ID. + +After the Job succeeds, Ariadne requests one Jenkins rebuild with failure seeding disabled. + +### Why a repair Job is the critical path + +A source-code test failure generally requires a Git change, but safe autonomous code editing requires repository credentials, checkout isolation, patch generation, tests, branch creation and PR lifecycle handling. Making that the primary demo would threaten the 24-hour limit. + +The fixture repair demonstrates a real closed loop: + +- Test failure is real. +- Jenkins and OpenSearch evidence are real. +- Hermes classification is real. +- Hermes action selection is real. +- Ariadne authorization and execution are real. +- The repair and green rebuild are real. + +Autonomous Gitea PR creation remains a stretch goal that begins only after the required loop is stable. + +## Evidence contract + +### Incident identity + +Use a stable ID: + +```text +/ +``` + +Every log line, bundle, Hermes request, action, repair Job and metric must carry that ID. + +### Jenkins evidence + +The bundle should include: + +- Job and build number. +- Branch and commit when available. +- Build URL. +- Start and end timestamps. +- Terminal result. +- First failed stage. +- Console tail. +- Failed JUnit cases. +- Selected quality reports when relevant. +- Evidence source and retrieval timestamp. + +### OpenSearch evidence + +The demo query should be intentionally fixed and bounded: + +- Indices: `kube-*` only. +- Time range: build start minus five minutes through build end plus five minutes. +- Namespace: `hermes-triage-demo`, plus `jenkins` only if it adds useful context. +- Correlation: exact incident ID first; namespace and time fallback second. +- Size: at most 50 records initially, never more than 100. +- Fields: timestamp, namespace, pod, container, log/message and selected safe labels. +- Timeout: approximately five seconds. +- Response byte cap: small enough to prevent prompt flooding. +- Pagination: none in the demo. + +Each excerpt must retain: + +- Index or source identifier. +- Timestamp. +- Namespace. +- Pod. +- Container. +- Sanitized message. + +Basic sanitization must remove or mask: + +- `Authorization` headers. +- Bearer tokens. +- Cookies and session IDs. +- Common password, token and secret assignments. +- Private key blocks. + +This is demo-grade sanitization, not a claim of comprehensive data-loss prevention. Queries remain restricted to the isolated namespace wherever possible. + +### Ariadne diagnosis bundle + +Add a `log_evidence` section without replacing the existing bundle fields: + +```json +{ + "incident_id": "hermes-triage-demo/42", + "generated_at": "...", + "jenkins": {}, + "quality_metrics": {}, + "runtime_context": {}, + "log_evidence": { + "query_window": {}, + "records": [], + "truncated": false, + "error": null + } +} +``` + +Empty or unavailable OpenSearch results must be represented explicitly. They must not block Jenkins-based diagnosis. + +## Automatic Hermes invocation + +### Required behavior + +Ariadne must call the Hermes Agent API, not only the Ollama completion endpoint. The invocation prompt should be stable and small: + +```text +Use $triage-titan-test-failures. +Analyze incident . +Treat the attached Ariadne bundle as the source of truth. +Identify the first enforced failure. +Distinguish facts from inference. +Return only the required triage response schema. +Do not perform mutations. +``` + +The bundle is attached as structured content. Ariadne records the Hermes request ID or session ID for audit and idempotency. + +### Authentication + +No API key may be committed to Git. + +The preferred path is: + +1. Generate one internal Hermes API key. +2. Store it in an approved Vault path. +3. Inject it into Hermes as `API_SERVER_KEY`. +4. Inject it into Ariadne as `HERMES_API_KEY`. +5. Restrict network access to the required namespaces where practical. + +The exact Hermes Agent API path and authentication behavior must be verified in the first hour. The current Hermes init container generates a key in its persistent `.env` if none exists; the integration must not scrape that file or copy it into Git. + +### Response schema + +Hermes must return a bounded machine-readable response: + +```json +{ + "incident_id": "hermes-triage-demo/42", + "classification": "known_demo_fixture_failure", + "confidence": 0.95, + "facts": [ + { + "statement": "...", + "source": "jenkins|opensearch|victoriametrics|kubernetes|flux|gitea", + "reference": "..." + } + ], + "inferences": [], + "first_failed_gate": "tests", + "requested_action": { + "type": "run_ariadne_job", + "id": "repair_demo_fixture" + }, + "human_required": false, + "reason": "" +} +``` + +Invalid JSON, unknown fields, mismatched incident IDs, low confidence and unknown actions all become human-required results. + +## Detection and deduplication + +For the demo, Ariadne should detect only an allowlisted job such as `hermes-triage-demo`. Reducing the global scheduler interval must not cause automatic action against unrelated failures. + +Suggested configuration: + +```text +ARIADNE_HERMES_AUTOTRIAGE_ENABLED=true +ARIADNE_HERMES_AUTOTRIAGE_JOB_ALLOWLIST=hermes-triage-demo +ARIADNE_HERMES_AUTOTRIAGE_INTERVAL_SECONDS=60 +ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false +ARIADNE_HERMES_ALLOWED_ACTIONS=repair_demo_fixture +ARIADNE_HERMES_MIN_CONFIDENCE=0.85 +ARIADNE_HERMES_MAX_ACTIONS_PER_INCIDENT=1 +``` + +The remediation flag remains false until the evidence-and-Hermes path passes in observe mode. + +Ariadne must persist or otherwise reliably track: + +- Incident observed. +- Hermes invocation started/completed. +- Action requested/accepted/rejected. +- Repair Job created/completed. +- Rebuild requested. +- Final build result. + +The same Jenkins build must never invoke a second repair after its action record exists. + +## Action authorization + +Ariadne may accept the demo action only when: + +- The Jenkins build is terminal and failed. +- The job is allowlisted. +- The incident ID matches the current record. +- The classification is exactly `known_demo_fixture_failure`. +- The action is exactly `repair_demo_fixture`. +- Confidence is at least the configured threshold. +- The Jenkins/JUnit or log evidence contains the expected failure signature. +- No action has already run for the incident. +- Automatic remediation is enabled. + +Hermes must not provide: + +- Shell commands. +- Kubernetes YAML. +- Container images. +- Namespaces. +- PVC names. +- Git commands. +- Retry counts. + +These remain hardcoded, tested Ariadne behavior. + +## Failure-class policy + +| Failure class | Automatic behavior in the 24-hour delivery | +|---|---| +| Known demo fixture failure | Run `hermes-demo-repair`, then rebuild once | +| Stale runtime quality telemetry | Optionally run the existing platform quality suite probe | +| Clearly transient demo Jenkins failure | At most one allowlisted rebuild if time permits | +| Ordinary source/test defect | Diagnose and recommend; human required | +| SonarQube failure | Human required; never waive automatically | +| Semgrep failure | Human required; never waive automatically | +| Trivy/supply-chain failure | Human required; never waive automatically | +| Kubernetes or Flux defect | Read-only diagnosis; human-required repo/Flux change | +| Missing or stale decisive evidence | Human required | +| Authentication, Secret or Vault issue | Human required | +| Storage-destructive or hardware issue | Human required | +| Unknown class or low confidence | Human required | + +## Triage metrics and escalation + +Ariadne should expose a small stable metric set, for example: + +```text +ariadne_hermes_triage_incident{job="...",build="...",status="detected|diagnosed|repairing|resolved|human_required"} 1 +ariadne_hermes_triage_action_total{action="...",result="accepted|rejected|success|failed"} 1 +ariadne_hermes_triage_last_success_timestamp_seconds ... +ariadne_hermes_triage_duration_seconds ... +``` + +Avoid unbounded labels such as full error messages, commit messages or model text. + +The minimum alert is: + +```text +human_required incident remains active for a short demo-safe interval + -> vmalert fires + -> Alertmanager handles notification + -> Grafana links to Jenkins and the Ariadne incident report +``` + +If dashboard changes threaten the deadline, the metric and alert rule take precedence over a polished panel. + +## 24-hour execution schedule + +### H+0 to H+1: establish authoritative state + +- Confirm the deployed Flux revision and Ariadne image/source revision. +- Confirm the Jenkins job definitions and branch behavior. +- Create clean dedicated worktrees without touching existing dirty worktrees. +- Verify the Hermes Agent API route, request format and authentication. +- Verify Ariadne-to-Hermes network reachability. +- Verify Ariadne-to-OpenSearch reachability. +- Verify the maintenance service account can create/watch the intended isolated Job or identify the minimal RBAC change. +- Freeze the exact demo job, namespace, failure signature and repair action. + +Gate at H+1: + +- No additional repositories or failure classes enter scope. +- If the Hermes Agent API path is still unknown, allocate no more than one additional hour before activating the fallback. + +### H+1 to H+4: bounded OpenSearch evidence + +- Add the fixed OpenSearch query client to Ariadne. +- Filter by the demo namespace, incident ID and build time window. +- Add response size, count and timeout limits. +- Add basic sanitization. +- Add `log_evidence` to the diagnosis bundle. +- Add unit tests for success, empty result, timeout, malformed payload, truncation and sanitization. +- Perform one live read-only query. + +Compromise at H+4: + +- Use a single fixed `kube-*` query rather than a general query builder. +- Retain raw chronological excerpts rather than semantic grouping. +- OpenSearch failure becomes explicit missing evidence and does not block Jenkins diagnosis. + +### H+4 to H+7: automatic Hermes invocation + +- Configure the shared internal API credential through Vault. +- Implement Ariadne's Hermes Agent client. +- Invoke the explicit Titan triage skill with the incident bundle. +- Require and validate structured JSON. +- Add a bounded timeout and one retry. +- Record request/session ID and duration. +- Deduplicate by incident ID. + +Compromise at H+7: + +- If the Hermes Agent API remains blocked, call the existing Hermes Ollama endpoint with the triage workflow embedded in the prompt. +- Clearly label this fallback as model-assisted Ariadne triage, not full Hermes Agent skill execution. +- Preserve the same response schema so the rest of the loop remains usable. + +### H+7 to H+10: decision and authorization contract + +- Freeze the response schema. +- Implement incident, class, confidence and action validation. +- Add idempotency and maximum-action checks. +- Add the global action kill switch. +- Reject arbitrary action content. +- Emit requested, accepted and rejected action metrics. +- Test invalid JSON, mismatched incident, low confidence, unknown action and duplicate incident. + +Gate at H+10: + +- Response schema and action list freeze. +- No additional automatic action type is added. + +### H+10 to H+14: isolated repair loop + +- Add the isolated namespace, fixture and test-runner resources. +- Add the dedicated Jenkins demo job or pipeline definition. +- Ensure the test-runner emits the incident ID to pod stdout/stderr. +- Add the predefined repair Job builder/executor to Ariadne. +- Wait for repair completion. +- Trigger one Jenkins rebuild with `SEED_FAILURE=false`. +- Record the rebuilt job and final status. +- Stop after one failed repair or rebuild. + +Compromise at H+14: + +- If repair execution is not repeatable, disable remediation. +- Ship automatic detection, evidence, Hermes diagnosis and human escalation. +- Do not consume remaining time attempting general repair logic. + +### H+14 to H+16: human-required signal + +- Add incident/action metrics. +- Add one vmalert rule for unresolved human-required incidents. +- Add Jenkins and Ariadne report links where the current dashboard model supports them. +- Confirm the signal clears on a resolved incident. + +Compromise at H+16: + +- Use the existing Atlas Testing dashboard and direct VictoriaMetrics query if a new panel is not ready. +- Preserve the metric and alert rule over dashboard polish. + +### H+16 to H+18: Flux integration and documentation + +- Add only required environment, Vault injection, network and RBAC changes. +- Keep the repair Job namespace and permissions isolated. +- Add the automatic-action kill switch to Flux-managed configuration. +- Rewrite `mermaid/TestAutomation.mmd` to show the actual trigger, evidence collection, Agent invocation, skill routing, decision schema, action gate, Ariadne execution and human escalation. +- Update this plan with final deviations and deferred hardening. +- Do not render SVG output unless separately requested. + +### H+18 to H+21: validation and first rehearsal + +- Run focused and complete relevant Ariadne tests. +- Run repository quality checks required by its Jenkins pipeline. +- Run `kustomize build` for each affected titan-iac kustomization. +- Run client-side dry-run validation without mutating the cluster. +- Deliver through Git and Flux only. +- Seed the controlled failure and observe the full loop. +- Fix only critical-path failures. + +Gate at H+21: + +- Feature freeze. +- No autonomous-code stretch work starts after this point. + +### H+21 to H+23: repeatability rehearsal + +Run these scenarios: + +1. Known fixture failure -> repair Job -> green rebuild. +2. Unsupported classification -> human-required signal and no repair. +3. Duplicate observation -> no duplicate repair. +4. OpenSearch timeout -> Jenkins-based diagnosis continues. +5. Hermes timeout or invalid response -> human-required signal. +6. Remediation kill switch disabled -> diagnosis only. + +Record timings, incident IDs, action count and final Jenkins status. + +### H+23 to H+24: hard stop + +- Make no new feature changes. +- Confirm the kill switch and rollback path. +- Capture the final working scope and known limitations. +- Ship the highest completed fallback tier. +- Stop at H+24. + +## Deadline compromise ladder + +When time is at risk, remove functionality in this order: + +1. Drop autonomous Gitea/code editing. +2. Limit remediation to `repair_demo_fixture` only. +3. Drop transient rebuild and optional suite-probe actions. +4. Drop additional failure classifications. +5. Replace dynamic OpenSearch construction with one fixed query. +6. Reuse the existing Grafana dashboard instead of adding a polished panel. +7. Fall back from Hermes Agent API to the same Qwen model with the triage skill procedure embedded in the prompt. +8. Disable automatic remediation but keep automatic diagnosis. +9. Preserve detection, evidence and human-required alerting as the minimum shippable system. + +The deadline is not extended to preserve a lower-priority feature. + +## Delivery tiers at H+24 + +### Tier A: target + +- Automatic detection. +- Jenkins and OpenSearch evidence. +- Hermes Agent skill execution. +- Structured diagnosis. +- One allowlisted repair Job. +- One rebuild. +- Human escalation for unsupported failures. + +### Tier B: acceptable compromise + +- Automatic detection. +- Jenkins and OpenSearch evidence. +- Hermes Agent skill execution. +- Structured diagnosis and recommendation. +- Human escalation. +- Automatic repair disabled. + +### Tier C: minimum shippable + +- Automatic detection. +- Jenkins and bounded OpenSearch evidence. +- Ariadne calls Qwen with the triage workflow embedded. +- Structured recommendation. +- Human-required metric/alert. + +No tier may silently claim Hermes Agent execution when only Ollama was invoked. + +## Testing strategy + +### Ariadne unit tests + +- OpenSearch query bounds and field selection. +- Correlation by incident ID and time. +- Sanitization and truncation. +- Empty, unavailable and malformed OpenSearch responses. +- Hermes request construction. +- Hermes response schema validation. +- Confidence threshold. +- Action allowlist. +- Incident mismatch rejection. +- Idempotency and maximum action count. +- Repair Job construction from hardcoded values. +- Jenkins rebuild parameter handling. +- Metrics with bounded label cardinality. + +### Integration tests + +- Mocked Jenkins failure plus mocked OpenSearch evidence. +- Mocked Hermes accepted action. +- Mocked unsupported classification. +- Repair Job success/failure. +- Jenkins rebuild request success/failure. +- Hermes timeout and malformed response. +- OpenSearch timeout while Jenkins evidence remains usable. + +### Live read-only preflight + +- Query one known `kube-*` time window. +- Confirm the expected Kubernetes field names. +- Confirm Hermes API status and authentication behavior. +- Confirm Jenkins GET access. +- Confirm VictoriaMetrics can query Ariadne metrics. + +### End-to-end demonstration + +- Arm the demo failure deliberately. +- Confirm one failed Jenkins build. +- Confirm one Ariadne incident. +- Confirm correlated OpenSearch excerpts. +- Confirm the expected Hermes skill and response. +- Confirm one repair Job. +- Confirm one rebuild. +- Confirm green result and resolved metric. +- Repeat once to prove the path is not a one-off race. + +## Flux and validation rules + +All persistent cluster changes must be represented in Git and reconciled through Flux. No manual `kubectl apply`, patch or ad hoc cluster edit is part of the delivery. + +For every affected titan-iac kustomization: + +```text +kustomize build +kubectl apply --server-side --dry-run=client -k +flux diff kustomization --path +``` + +Actual reconciliation occurs only after the intended commits are available to Flux. + +Application changes in Ariadne must run the focused tests for new services plus its relevant full test/quality suite. Exact commands should follow the authoritative branch's Jenkinsfile and test configuration rather than assumptions from the currently dirty local checkout. + +## Security and safety boundary + +- Store all shared credentials in Vault; never Git. +- Restrict automatic triage to an explicit Jenkins job allowlist. +- Restrict OpenSearch to `kube-*`, time bounds and isolated namespaces. +- Sanitize evidence before model use. +- Never send entire indices, artifacts or unrestricted logs to the model. +- Do not add Secret-read permission to Hermes. +- Do not grant Hermes Kubernetes mutation. +- Do not allow model-supplied shell commands or manifests. +- Keep action definitions in tested Ariadne code. +- Permit one action and one rebuild per incident. +- Require human intervention on low confidence or missing decisive evidence. +- Keep a default-off remediation flag until observe mode passes. + +## Rollback and kill switches + +Immediate behavioral rollback: + +```text +ARIADNE_HERMES_AUTOREMEDIATION_ENABLED=false +``` + +Further rollback steps are: + +1. Disable automatic Hermes invocation while retaining normal Ariadne schedules. +2. Restore the original Jenkins-weather and testing-triage intervals. +3. Disable the demo Jenkins job. +4. Remove or suspend only the isolated demo resources through Git/Flux. +5. Revoke the shared Hermes API key in Vault. +6. Leave ordinary Jenkins, Fluent Bit, OpenSearch, VictoriaMetrics and Hermes read-only behavior untouched. + +The repair action must be safe to run twice even though idempotency should prevent the second run. + +## Repository touchpoints + +Expected Ariadne application changes: + +- Bounded OpenSearch evidence client. +- Diagnosis-bundle enrichment. +- Hermes Agent API client. +- Response schema and validation. +- Incident deduplication/state. +- Allowlisted repair action executor. +- Jenkins rebuild request. +- Triage metrics. +- Unit and integration tests. + +Expected titan-iac changes: + +- Ariadne environment and Vault injection. +- Hermes shared API-key configuration if required. +- Isolated demo namespace/PVC/RBAC/test-runner resources. +- Predefined repair Job permissions. +- Demo Jenkins JCasC/pipeline definition. +- vmalert rule and minimal Grafana integration. +- `mermaid/TestAutomation.mmd` update. + +Do not modify unrelated workloads or fold repository-layout migrations into these changes. + +## Optional autonomous-code stretch + +This begins only if Tier A passes end to end before H+16 and at least eight hours remain. + +The only acceptable stretch is a restricted Gitea PR workflow: + +```text +Hermes proposes patch + -> Ariadne validates repository and base branch + -> dedicated service creates a demo branch/PR + -> Jenkins validates the branch + -> human reviews and merges +``` + +Restrictions: + +- One demo repository. +- Dedicated branch prefix. +- No push to `main` or `master`. +- No automatic merge. +- Separate least-privilege Gitea token in Vault. +- Patch size and file-path limits. +- Jenkins validation required. + +This stretch is dropped immediately if it threatens rehearsal or the H+24 stop. + +## Required architecture-document update + +After behavior is known, `mermaid/TestAutomation.mmd` must show these internal steps rather than compressing Hermes into one box: + +```text +Trigger + -> terminal Jenkins failure or human request + +Detection + -> Ariadne allowlisted polling and incident deduplication + +Evidence + -> Jenkins metadata/artifacts + -> bounded OpenSearch excerpts + -> VictoriaMetrics + -> Kubernetes/Flux/Gitea/Grafana context when relevant + +Agent invocation + -> Ariadne calls Hermes Agent API + -> explicit triage-titan-test-failures skill + -> local Qwen model through Hermes Ollama + +Decision + -> first enforced failure + -> facts versus inference + -> specialist skill routing + -> confidence and action schema + +Authorization + -> Ariadne allowlist, incident match, confidence, idempotency and kill switch + +Response + -> predefined repair Job and one rebuild + -> or human-required metric and alert +``` + +The chart must visually distinguish current behavior, newly implemented demo behavior and deferred capabilities. It should continue to keep Jenkins-retained logs separate from OpenSearch pod logs and should not imply direct Hermes OpenSearch access. + +## Final handoff record + +At H+24, append a short result section containing: + +- Delivered tier. +- Exact commits and deployed revisions. +- Demo job/build IDs. +- Incident ID. +- Evidence query window. +- Hermes request/session ID. +- Requested and executed action. +- Repair Job name. +- Rebuild result. +- Alert result. +- Total end-to-end duration. +- Compromises activated. +- Deferred hardening. +- Kill-switch and rollback verification. + +The handoff must state plainly whether the full Hermes Agent or the Ollama fallback performed the diagnosis. diff --git a/services/comms/knowledge/metis.md b/services/comms/knowledge/metis.md index 5b0d06be5..4f3d471a3 100644 --- a/services/comms/knowledge/metis.md +++ b/services/comms/knowledge/metis.md @@ -1,12 +1,14 @@ # Metis (node recovery) ## Node classes (current map) -- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent) +- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent) - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint) - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks) - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent) +- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent) - amd64 agents: titan-22/24 (Debian 13, k3s agent) -- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, future titan-20/21 (when added), plus any newcomers. +- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels) +- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers. ## Longhorn disk UUIDs (critical nodes) - titan-13: /mnt/astreae UUID=6031fa8b-f28c-45c3-b7bc-6133300e07c6 (ext4); /mnt/asteria UUID=cbd4989d-62b5-4741-8b2a-28fdae259cae (ext4) @@ -17,10 +19,9 @@ ## Metis repo (~/Development/metis) - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`). - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints). -- `AGENTS.md` in repo is untracked and holds raw notes. ## Next implementation steps - Add per-class golden image refs and checksums (Harbor or file://) when ready. - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths. - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection. -- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited. +- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited. diff --git a/services/comms/knowledge/runbooks/ci-gitea-jenkins.md b/services/comms/knowledge/runbooks/ci-gitea-jenkins.md index 48dc91fde..14f76d59d 100644 --- a/services/comms/knowledge/runbooks/ci-gitea-jenkins.md +++ b/services/comms/knowledge/runbooks/ci-gitea-jenkins.md @@ -3,7 +3,7 @@ title: "CI: Gitea → Jenkins pipeline" tags: ["atlas", "ci", "gitea", "jenkins"] owners: ["brad"] entrypoints: ["scm.bstein.dev", "ci.bstein.dev"] -source_paths: ["services/gitea", "services/jenkins", "scripts/jenkins_cred_sync.sh", "scripts/gitea_cred_sync.sh"] +source_paths: ["services/gitea", "services/jenkins", "scripts/sync/jenkins_cred_sync.sh", "scripts/sync/gitea_cred_sync.sh"] --- # CI: Gitea → Jenkins pipeline @@ -14,7 +14,7 @@ Atlas uses Gitea for source control and Jenkins for CI. Authentication is via Ke ## Where it is configured - Gitea manifests: `services/gitea/` - Jenkins manifests: `services/jenkins/` -- Credential sync helpers: `scripts/gitea_cred_sync.sh`, `scripts/jenkins_cred_sync.sh` +- Credential sync helpers: `scripts/sync/gitea_cred_sync.sh`, `scripts/sync/jenkins_cred_sync.sh` ## What users do (typical flow) - Create a repo in Gitea. diff --git a/services/comms/knowledge/runbooks/cluster-power-recovery.md b/services/comms/knowledge/runbooks/cluster-power-recovery.md new file mode 100644 index 000000000..86436d4c2 --- /dev/null +++ b/services/comms/knowledge/runbooks/cluster-power-recovery.md @@ -0,0 +1,152 @@ +Atlas Cluster Power Recovery (Graceful Shutdown/Startup) + +Purpose +- Provide a safe operator flow for planned power events and cold-boot recovery. +- Avoid the Flux/Gitea bootstrap deadlock by using a local bootstrap fallback path. +- Break the Harbor self-hosting deadlock by seeding Harbor runtime images from a control-host bundle. +- Refuse bootstrap when UPS charge is too low, and fall back to fast shutdown if a second outage hits mid-recovery. + +Bootstrapping risk to remember +- Flux source is Git over SSH to `scm.bstein.dev` (Gitea). +- Gitea itself is a Flux-managed workload and depends on storage + database. +- Harbor is also critical, but it is not part of the first recovery stage because Harbor serves its own runtime images. +- On cold boot, if Flux cannot fetch source before Gitea is up, reconciliation can stall. +- Recovery path: bring control plane and workers up, then locally apply minimal platform stack (`core -> helm -> longhorn -> metallb -> traefik -> vault-csi -> vault-injector -> vault -> postgres -> gitea`), then seed Harbor images onto the Harbor node from a control-host bundle, then resume/reconcile Flux. Harbor is a later recovery stage after storage, Vault, Postgres, and Gitea are back. + +Script +- `scripts/ops/cluster_power_recovery.sh` +- `scripts/ops/cluster_power_console.sh` +- Modes: + - `prepare` + - `shutdown` + - `harbor-seed` + - `startup` + - `status` +- Default is dry-run. Add `--execute` to actually perform actions. + +Dry-run examples +- Shutdown preview: + - `scripts/ops/cluster_power_recovery.sh shutdown --skip-etcd-snapshot --skip-drain` +- Startup preview: + - `scripts/ops/cluster_power_recovery.sh startup` +- Harbor seed preview: + - `scripts/ops/cluster_power_recovery.sh harbor-seed` + +Execute examples +- Prepare helper image on every node: + - `scripts/ops/cluster_power_recovery.sh prepare --execute` +- Seed Harbor runtime images onto `titan-05` from the control-host bundle: + - `scripts/ops/cluster_power_recovery.sh harbor-seed --execute` +- Planned shutdown: + - `scripts/ops/cluster_power_recovery.sh shutdown --execute` +- Planned startup (canonical branch): + - `scripts/ops/cluster_power_recovery.sh startup --execute --force-flux-branch main` + +Manual remote console examples +- Canonical operator hosts: + - `titan-db` + - `tethys` (`titan-24`) +- Both hosts now have: + - `~/ananke-tools/cluster_power_recovery.sh` + - `~/ananke-tools/cluster_power_console.sh` + - `~/ananke-tools/bootstrap/recovery-config.env` + - `~/ananke-tools/bootstrap/harbor-bootstrap-images.txt` + - `~/ananke-tools/kubeconfig` + - `~/ananke-cluster-power` + - `~/bin/ananke-cluster-power` + - `~/ananke-repo/{infrastructure,services,scripts}` +- Both hosts also keep the Harbor bootstrap bundle at: + - `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst` +- Remote usage: + - `ssh titan-db` + - `~/ananke-cluster-power status` + - `~/ananke-cluster-power prepare --execute` + - `~/ananke-cluster-power shutdown --execute` + - `~/ananke-cluster-power startup --execute --force-flux-branch main` + - `ssh tethys` + - `~/ananke-cluster-power status` + - `~/ananke-cluster-power prepare --execute` + - `~/ananke-cluster-power shutdown --execute` + - `~/ananke-cluster-power startup --execute --force-flux-branch main` + +Useful options +- `--shutdown-mode host-poweroff|cluster-only` +- `--expected-flux-branch main` +- `--expected-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` +- `--force-flux-url ssh://git@scm.bstein.dev:2242/bstein/titan-iac.git` +- `--force-flux-branch main` +- `--allow-flux-source-mutation` (required with `--force-flux-url`; breakglass only) +- `--skip-local-bootstrap` (not recommended for cold-start recovery) +- `--skip-harbor-bootstrap` (skip the Harbor recovery stage if you know Harbor should stay deferred) +- `--skip-harbor-seed` (skip bundle import if Harbor images are already cached on the target node) +- `--skip-helper-prewarm` +- `--min-startup-battery 35` +- `--ups-host pyrphoros@localhost` +- `--require-ups-battery` +- `--drain-timeout 180` +- `--emergency-drain-timeout 45` +- `--flux-ready-timeout 1200` +- `--startup-checklist-timeout 900` +- `--startup-stability-window 180` +- `--startup-stability-timeout 900` +- `--recovery-state-file ~/.local/share/ananke/cluster_power_recovery.state` +- `--harbor-bundle-file ~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst` + +Controlled drill checklist (recommended) +- Operator host: use `titan-db` as canonical control host for the drill. +- On-site coordination: + - Have on-site operator ready before shutdown starts. + - Confirm they will manually power cluster nodes back on after shutdown completes. + - Confirm who will announce "all nodes powered on" to resume startup. +- Preflight on `titan-db`: + - `mkdir -p ~/ananke-logs` + - `~/ananke-cluster-power status` and verify: + - `ups_host=pyrphoros@localhost` + - `ups_battery` is numeric + - `flux_source_ready=True` +- Warm helper image just before shutdown: + - `~/ananke-cluster-power prepare --execute` +- Run in a persistent shell and capture logs: + - `tmux new -s ananke-drill` + - `script -q -a ~/ananke-logs/ananke-drill-$(date +%Y%m%d-%H%M%S).log` +- Execute controlled shutdown with telemetry enforcement: + - `~/ananke-cluster-power shutdown --execute --require-ups-battery` +- After on-site power-on confirmation, execute startup: + - `~/ananke-cluster-power startup --execute --force-flux-branch main --require-ups-battery` +- Post-check: + - `~/ananke-cluster-power status` + - Verify critical services (`longhorn`, `vault`, `postgres`, `gitea`, `harbor`, `pegasus`) and no widespread pull/crash failures. + +Operational notes +- The flow suspends Flux Kustomizations/HelmReleases during shutdown to prevent churn. +- Shutdown behavior is explicit: + - `host-poweroff` schedules host poweroff after service stop. + - `cluster-only` stops `k3s`/`k3s-agent` without powering hosts off. +- Worker drain is no longer best-effort only. The script now escalates from normal drain, to `--force`, to `--disable-eviction` once the configured timeout is exhausted. +- Startup fails fast if Flux source URL/branch drift from expected values (unless branch override is explicitly requested with `--force-flux-branch`). +- Flux desired-state source remains `titan-iac.git`. Ananke orchestrates runtime recovery and should not be used as the normal Flux source repo. +- During startup, if Flux source is not `Ready`, local bootstrap fallback is applied first using the repo snapshot under `~/ananke-repo`. +- Longhorn is reconciled before Vault/Postgres/Gitea so storage-backed services are not racing the volume layer. +- Harbor is reconciled after the first critical stateful services. +- Harbor bootstrap is now designed around a control-host bundle: + - Build the Harbor bundle locally with `scripts/ops/build_harbor_bootstrap_bundle.sh`. + - Stage it on the operator host at `~/.local/share/ananke/bundles/harbor-bootstrap-v2.14.1-arm64.tar.zst`. + - Use `harbor-seed --execute` or a full `startup --execute` to stream/import that bundle onto `titan-05`. +- The Harbor bundle remains arm64-only because Harbor is pinned to arm64 nodes. The node-helper image is multi-arch because Ananke uses it across both arm64 and amd64 nodes during prepare/shutdown operations. +- Ananke uses a temporary privileged helper pod for host-side operations. The helper image is prewarmed with `prepare --execute` so later shutdown/startup steps do not stall on image pulls. +- The script persists outage state in `~/.local/share/ananke/cluster_power_recovery.state` by default. If startup is attempted during an outage window and power becomes unstable again, rerunning startup with insufficient UPS charge will flip into the emergency shutdown path instead of continuing to bootstrap. +- Startup completion is strict now: + - all non-optional Flux kustomizations must be `Ready=True` + - external service checklist must pass (defaults include Gitea, Grafana, Harbor) + - generated ingress reachability checks must pass (default accepted codes: `200,301,302,307,308,401,403,404`) + - stability soak must pass with no crashloop/pull-failure churn +- If Flux hits immutable one-off Job drift during reconcile, Ananke now attempts self-heal by pruning failed Flux-managed Jobs and retrying reconcile. +- In dry-run mode, the script now skips the live API wait step so preview runs do not stall on an offline cluster. +- Dry-run mode no longer mutates outage recovery state. +- `harbor-seed --execute` was validated by: + - prewarming the helper image across all nodes + - streaming the Harbor bootstrap bundle to `titan-05` + - importing Harbor runtime images into host `containerd` + - successfully running a Harbor-backed canary pod (`harbor-canary-ok`) +- After bootstrap, Flux resources are resumed and reconciled. +- Keep this runbook aligned with `clusters/atlas/flux-system/gotk-sync.yaml`. diff --git a/services/comms/knowledge/runbooks/kb-authoring.md b/services/comms/knowledge/runbooks/kb-authoring.md index 9378d1d4d..9b872354d 100644 --- a/services/comms/knowledge/runbooks/kb-authoring.md +++ b/services/comms/knowledge/runbooks/kb-authoring.md @@ -3,7 +3,7 @@ title: "KB authoring: what to write (and what not to)" tags: ["atlas", "kb", "runbooks"] owners: ["brad"] entrypoints: [] -source_paths: ["knowledge/runbooks", "scripts/knowledge_render_atlas.py"] +source_paths: ["knowledge/runbooks", "scripts/render/knowledge_render_atlas.py"] --- # KB authoring: what to write (and what not to) diff --git a/services/comms/knowledge/software/metis.md b/services/comms/knowledge/software/metis.md index 192e266eb..044a75e70 100644 --- a/services/comms/knowledge/software/metis.md +++ b/services/comms/knowledge/software/metis.md @@ -1,26 +1,14 @@ # Metis (node recovery) -## Fast path (SD/media failure) -1. Run `scripts/node_recover.sh --yes --delete-node` from `titan-iac`. -2. Reimage/reprovision the replacement host. -3. Rejoin the replacement node to k3s. -4. Reapply labels and taints from generated artifacts: - - `artifacts/node-recovery/-/restore-labels.sh` - - `artifacts/node-recovery/-/restore-taints.sh` -5. Verify workloads, then uncordon the replacement node. - -### Notes -- `node_recover.sh` snapshots node labels/taints and current pod placement before drain. -- Use `--skip-drain` for a dead/unreachable node where only artifact capture is possible. -- Use `--delete-node` after drain (or for hard-dead nodes) so replacement join is clean. - ## Node classes (current map) -- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11,20,21 (Ubuntu 24.04.3, k3s agent) +- rpi5 Ubuntu workers: titan-04,05,06,07,08,09,10,11 (Ubuntu 24.04.3, k3s agent) - rpi5 control-plane: titan-0a/0b/0c (Ubuntu 24.04.1, k3s server, control-plane taint) - rpi4 Armbian longhorn: titan-13/15/17/19 (Armbian 6.6.x, k3s agent, longhorn disks) - rpi4 Armbian standard: titan-12/14/18 (Armbian 6.6.x, k3s agent) +- Jetson workers: titan-20/21 (Ubuntu 20.04.6, k3s agent) - amd64 agents: titan-22/24 (Debian 13, k3s agent) -- External/non-cluster: tethys, titan-db, titan-jh, oceanus/titan-23, plus any newcomers. +- Veles storage/simulation worker: titan-23 (Atlas worker with `oceanus` node-pool labels) +- External/dedicated hosts: tethys, titan-db, titan-jh, plus any newcomers. ### Jetson nodes (titan-20/21) - Ubuntu 20.04.6 (Focal), kernel 5.10.104-tegra, CRI containerd 2.0.5-k3s2, arch arm64. @@ -36,13 +24,12 @@ ## Metis repo (~/Development/metis) - CLI skeleton in Go (`cmd/metis`), inventory loader (`pkg/inventory`), plan builder (`pkg/plan`). - `inventory.example.yaml` shows expected schema (classes + per-node overlay, Longhorn disks, labels, taints). -- `AGENTS.md` in repo is untracked and holds raw notes. ## Next implementation steps - Add per-class golden image refs and checksums (Harbor or file://) when ready. - Implement burn execution: download with checksum, write via dd/etcher-equivalent, mount boot/root to inject hostname/IP/k3s tokens/labels/taints, journald/GC drop-ins, and Longhorn fstab entries. Add Windows writer (diskpart + wmic) and Linux writer (dd + sgdisk) paths. - Add Keycloak/SSH bootstrap: ensure ssh user, authorized keys, and k3s token/URL injection for agents; control-plane restore path with etcd snapshot selection. -- Add per-host inventory entries for tethys, titan-db, titan-jh, oceanus/titan-23, future 20/21 once audited. +- Add per-host inventory entries for tethys, titan-db, titan-jh, and future dedicated hosts once audited. ## Node OS/Kernel/CRI snapshot (Jan 2026) - titan-04: Ubuntu 24.04.3 LTS, kernel 6.8.0-1031-raspi, CRI containerd://2.0.5-k3s2, arch arm64 @@ -69,10 +56,10 @@ - titan-24: Debian 13 (trixie), kernel 6.12.57+deb13-amd64, CRI containerd://2.0.5-k3s2, arch amd64 -### External hosts +### Dedicated and special-purpose hosts - titan-db: Ubuntu 24.10, kernel 6.11.0-1015-raspi, root on /dev/sda2 ext4 (465G), boot vfat /dev/sda1; PostgreSQL service enabled. - titan-jh: Arch Linux ARM (rolling), kernel 6.18.4-2-rpi, NVMe root ext4 238G (/), boot vfat 512M; ~495 packages installed (pacman -Q). -- titan-23/oceanus: TODO audit (future). +- titan-23: Atlas worker carrying the `oceanus` node-pool labels for Veles storage/simulation placement. ### Control plane Pis (titan-0a/0b/0c) diff --git a/services/comms/kustomization.yaml b/services/comms/kustomization.yaml index 969ca586d..f54da0c42 100644 --- a/services/comms/kustomization.yaml +++ b/services/comms/kustomization.yaml @@ -22,24 +22,24 @@ resources: - mas-db-ensure-rbac.yaml - synapse-signingkey-ensure-rbac.yaml - vault-sync-deployment.yaml - - oneoffs/mas-admin-client-secret-ensure-job.yaml - - oneoffs/mas-db-ensure-job.yaml - - oneoffs/comms-secrets-ensure-job.yaml - - oneoffs/synapse-admin-ensure-job.yaml - - oneoffs/synapse-signingkey-ensure-job.yaml - - oneoffs/synapse-seeder-admin-ensure-job.yaml - - oneoffs/synapse-user-seed-job.yaml - - oneoffs/mas-local-users-ensure-job.yaml + - bootstrap-jobs/mas-admin-client-secret-ensure-job.yaml + - bootstrap-jobs/mas-db-ensure-job.yaml + - bootstrap-jobs/comms-secrets-ensure-job.yaml + - bootstrap-jobs/synapse-admin-ensure-job.yaml + - bootstrap-jobs/synapse-signingkey-ensure-job.yaml + - bootstrap-jobs/synapse-seeder-admin-ensure-job.yaml + - bootstrap-jobs/synapse-user-seed-job.yaml + - bootstrap-jobs/mas-local-users-ensure-job.yaml - mas-deployment.yaml - livekit-token-deployment.yaml - livekit.yaml - coturn.yaml - seed-othrys-room.yaml - guest-name-job.yaml - - oneoffs/othrys-kick-numeric-job.yaml + - repair-jobs/othrys-kick-numeric-job.yaml - pin-othrys-job.yaml - reset-othrys-room-job.yaml - - oneoffs/bstein-force-leave-job.yaml + - repair-jobs/bstein-force-leave-job.yaml - livekit-ingress.yaml - livekit-middlewares.yaml - matrix-ingress.yaml diff --git a/services/comms/oneoffs/bstein-force-leave-job.yaml b/services/comms/repair-jobs/bstein-force-leave-job.yaml similarity index 99% rename from services/comms/oneoffs/bstein-force-leave-job.yaml rename to services/comms/repair-jobs/bstein-force-leave-job.yaml index 0caf6883d..4b72c73c6 100644 --- a/services/comms/oneoffs/bstein-force-leave-job.yaml +++ b/services/comms/repair-jobs/bstein-force-leave-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/bstein-force-leave-job.yaml +# services/comms/repair-jobs/bstein-force-leave-job.yaml # One-off job for comms/bstein-leave-rooms-12. # Purpose: bstein leave rooms 12 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/oneoffs/othrys-kick-numeric-job.yaml b/services/comms/repair-jobs/othrys-kick-numeric-job.yaml similarity index 99% rename from services/comms/oneoffs/othrys-kick-numeric-job.yaml rename to services/comms/repair-jobs/othrys-kick-numeric-job.yaml index da9c69ce6..2c77e34c2 100644 --- a/services/comms/oneoffs/othrys-kick-numeric-job.yaml +++ b/services/comms/repair-jobs/othrys-kick-numeric-job.yaml @@ -1,4 +1,4 @@ -# services/comms/oneoffs/othrys-kick-numeric-job.yaml +# services/comms/repair-jobs/othrys-kick-numeric-job.yaml # One-off job for comms/othrys-kick-numeric-8. # Purpose: othrys kick numeric 8 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/comms/scripts/atlasbot/bot.py b/services/comms/scripts/atlasbot/bot.py index e8619bc8b..369f28bb1 100644 --- a/services/comms/scripts/atlasbot/bot.py +++ b/services/comms/scripts/atlasbot/bot.py @@ -537,7 +537,7 @@ def send_msg(token: str, room: str, text: str): req("POST", path, token, body={"msgtype": "m.text", "body": text}) -# Atlas KB loader (no external deps; files are pre-rendered JSON via scripts/knowledge_render_atlas.py). +# Atlas KB loader (no external deps; files are pre-rendered JSON via scripts/render/knowledge_render_atlas.py). KB = {"catalog": {}, "runbooks": []} _HOST_INDEX: dict[str, list[dict]] = {} _NAME_INDEX: set[str] = set() diff --git a/services/crypto/kustomization.yaml b/services/crypto/kustomization.yaml index f31fc8a28..54249b73d 100644 --- a/services/crypto/kustomization.yaml +++ b/services/crypto/kustomization.yaml @@ -1,6 +1,7 @@ # services/crypto/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: crypto resources: - namespace.yaml - default-serviceaccount.yaml diff --git a/services/crypto/monerod/kustomization.yaml b/services/crypto/monerod/kustomization.yaml index d849663b0..b2dfdff6d 100644 --- a/services/crypto/monerod/kustomization.yaml +++ b/services/crypto/monerod/kustomization.yaml @@ -1,6 +1,7 @@ # services/crypto/monerod/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: crypto resources: - pvc.yaml - cm-release-keys.yaml diff --git a/services/crypto/wallet-monero-temp/kustomization.yaml b/services/crypto/wallet-monero-temp/kustomization.yaml index 623685895..ea3b25a9d 100644 --- a/services/crypto/wallet-monero-temp/kustomization.yaml +++ b/services/crypto/wallet-monero-temp/kustomization.yaml @@ -1,6 +1,7 @@ # services/crypto/wallet-monero-temp/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: crypto resources: - pvc.yaml - deployment.yaml diff --git a/services/crypto/xmr-miner/kustomization.yaml b/services/crypto/xmr-miner/kustomization.yaml index 2ded8dbcf..0ed27c4d7 100644 --- a/services/crypto/xmr-miner/kustomization.yaml +++ b/services/crypto/xmr-miner/kustomization.yaml @@ -1,6 +1,7 @@ # services/crypto/xmr-miner/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: crypto resources: - configmap-sources.yaml - vault-serviceaccount.yaml diff --git a/services/finance/oneoffs/finance-secrets-ensure-job.yaml b/services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml similarity index 96% rename from services/finance/oneoffs/finance-secrets-ensure-job.yaml rename to services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml index e8c8f5887..450046cde 100644 --- a/services/finance/oneoffs/finance-secrets-ensure-job.yaml +++ b/services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/finance/oneoffs/finance-secrets-ensure-job.yaml +# services/finance/bootstrap-jobs/finance-secrets-ensure-job.yaml # One-off job for finance/finance-secrets-ensure-5. # Purpose: finance secrets ensure 5 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/finance/kustomization.yaml b/services/finance/kustomization.yaml index 1559f5c85..ba6908159 100644 --- a/services/finance/kustomization.yaml +++ b/services/finance/kustomization.yaml @@ -9,7 +9,7 @@ resources: - finance-secrets-ensure-rbac.yaml - actual-budget-data-pvc.yaml - firefly-storage-pvc.yaml - - oneoffs/finance-secrets-ensure-job.yaml + - bootstrap-jobs/finance-secrets-ensure-job.yaml - actual-budget-deployment.yaml - firefly-deployment.yaml - firefly-user-sync-cronjob.yaml diff --git a/services/game-stream/kustomization.yaml b/services/game-stream/kustomization.yaml index 503cadc4f..42915b270 100644 --- a/services/game-stream/kustomization.yaml +++ b/services/game-stream/kustomization.yaml @@ -1,6 +1,7 @@ # services/game-stream/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: game-stream resources: - namespace.yaml - vault-serviceaccount.yaml diff --git a/services/gitea/kustomization.yaml b/services/gitea/kustomization.yaml index b09f5fdf1..d42793161 100644 --- a/services/gitea/kustomization.yaml +++ b/services/gitea/kustomization.yaml @@ -1,6 +1,7 @@ # services/gitea/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: gitea resources: - namespace.yaml - serviceaccount.yaml diff --git a/services/jellyfin/kustomization.yaml b/services/jellyfin/kustomization.yaml index 041b27c8f..6feeb1f49 100644 --- a/services/jellyfin/kustomization.yaml +++ b/services/jellyfin/kustomization.yaml @@ -1,6 +1,7 @@ # services/jellyfin/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: jellyfin resources: - namespace.yaml - pvc.yaml diff --git a/services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml similarity index 96% rename from services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml index d4da1f1fd..aab7ee97e 100644 --- a/services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/actual-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/actual-oidc-secret-ensure-job.yaml # One-off job for sso/actual-oidc-secret-ensure-3. # Purpose: actual oidc secret ensure 3 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml new file mode 100644 index 000000000..38039196d --- /dev/null +++ b/services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml @@ -0,0 +1,54 @@ +# services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml +# One-off job for sso/cassandra-gitea-oidc-secret-ensure-1. +# Purpose: create/update the Keycloak cassandra realm Gitea OIDC client, ensure the +# cassandra-tester group exists, and write the client secret to Vault for Gitea. +# Run by setting spec.suspend to false, reconcile, then set it back to true. +# Safe to delete the finished Job/pod; it should not run continuously. +apiVersion: batch/v1 +kind: Job +metadata: + name: cassandra-gitea-oidc-secret-ensure-1 + namespace: sso +spec: + suspend: true + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: "sso-secrets" + vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin" + vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: | + {{ with secret "kv/data/atlas/shared/keycloak-admin" }} + export KEYCLOAK_ADMIN="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" + {{ end }} + spec: + serviceAccountName: mas-secrets-ensure + restartPolicy: Never + volumes: + - name: cassandra-gitea-oidc-secret-ensure-script + configMap: + name: cassandra-gitea-oidc-secret-ensure-script + defaultMode: 0555 + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/arch + operator: In + values: ["arm64"] + - key: node-role.kubernetes.io/worker + operator: Exists + containers: + - name: apply + image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 + command: ["/scripts/cassandra_gitea_oidc_secret_ensure.sh"] + volumeMounts: + - name: cassandra-gitea-oidc-secret-ensure-script + mountPath: /scripts + readOnly: true diff --git a/services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml b/services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml new file mode 100644 index 000000000..3b3d10b38 --- /dev/null +++ b/services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml @@ -0,0 +1,401 @@ +# services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml +# One-off job for sso/cassandra-realm-ensure-1. +# Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret. +# Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once. +apiVersion: batch/v1 +kind: Job +metadata: + name: cassandra-realm-ensure-1 + namespace: sso +spec: + suspend: true + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/role: "sso-secrets" + vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin" + vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: | + {{ with secret "kv/data/atlas/shared/keycloak-admin" }} + export KEYCLOAK_ADMIN="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}" + export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}" + {{ end }} + {{ with secret "kv/data/atlas/shared/postmark-relay" }} + export KEYCLOAK_SMTP_USER="{{ index .Data.data "apikey" }}" + export KEYCLOAK_SMTP_PASSWORD="{{ index .Data.data "apikey" }}" + {{ end }} + spec: + serviceAccountName: mas-secrets-ensure + restartPolicy: Never + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/worker + operator: Exists + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/arch + operator: In + values: ["arm64"] + containers: + - name: configure + image: python:3.11-alpine + env: + - name: KEYCLOAK_SERVER + value: http://keycloak.sso.svc.cluster.local + - name: KEYCLOAK_REALM + value: cassandra + - name: KEYCLOAK_CLIENT_ID + value: cassandra-web + - name: KEYCLOAK_PUBLIC_ISSUER + value: https://sso.bstein.dev/realms/cassandra + - name: CASSANDRA_BASE_URL + value: https://cassandra.bstein.dev + - name: CASSANDRA_TESTER_GROUP + value: cassandra-tester + - name: KEYCLOAK_SMTP_HOST + value: mail.bstein.dev + - name: KEYCLOAK_SMTP_PORT + value: "587" + - name: KEYCLOAK_SMTP_FROM + value: no-reply-cassandra@bstein.dev + - name: KEYCLOAK_SMTP_FROM_NAME + value: Cassandra + - name: CASSANDRA_PUBLIC_REPLAY_USERS + value: cassandra-dev,daniel-test,viktor-test + command: ["/bin/sh", "-c"] + args: + - | + set -eu + . /vault/secrets/keycloak-admin-env.sh + python - <<'PY' + import json + import os + import time + import urllib.error + import urllib.parse + import urllib.request + + base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/") + realm = os.environ["KEYCLOAK_REALM"] + client_id = os.environ["KEYCLOAK_CLIENT_ID"] + issuer = os.environ["KEYCLOAK_PUBLIC_ISSUER"] + app_base_url = os.environ["CASSANDRA_BASE_URL"].rstrip("/") + admin_user = os.environ["KEYCLOAK_ADMIN_USER"] + admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"] + + def request(method, url, token=None, payload=None, headers=None, timeout=30): + data = None + req_headers = headers.copy() if headers else {} + if token: + req_headers["Authorization"] = f"Bearer {token}" + if payload is not None: + data = json.dumps(payload).encode() + req_headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, headers=req_headers, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read() + if not body: + return resp.status, None + return resp.status, json.loads(body.decode()) + except urllib.error.HTTPError as exc: + raw = exc.read() + if not raw: + return exc.code, None + try: + return exc.code, json.loads(raw.decode()) + except Exception: + return exc.code, {"raw": raw.decode(errors="replace")} + + token_body = None + form = urllib.parse.urlencode( + { + "grant_type": "password", + "client_id": "admin-cli", + "username": admin_user, + "password": admin_password, + } + ).encode() + for attempt in range(1, 11): + req = urllib.request.Request( + f"{base_url}/realms/master/protocol/openid-connect/token", + data=form, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + token_body = json.loads(resp.read().decode()) + break + except urllib.error.URLError as exc: + if attempt == 10: + raise SystemExit(f"Keycloak token request failed after retries: {exc}") + time.sleep(attempt * 2) + token = token_body["access_token"] + + smtp = { + "host": os.environ["KEYCLOAK_SMTP_HOST"], + "port": os.environ["KEYCLOAK_SMTP_PORT"], + "from": os.environ["KEYCLOAK_SMTP_FROM"], + "fromDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"], + "replyTo": os.environ["KEYCLOAK_SMTP_FROM"], + "replyToDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"], + "user": os.environ["KEYCLOAK_SMTP_USER"], + "password": os.environ["KEYCLOAK_SMTP_PASSWORD"], + "auth": "true", + "starttls": "true", + "ssl": "false", + } + + status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token) + if status == 404: + create_payload = { + "realm": realm, + "enabled": True, + "registrationAllowed": False, + "resetPasswordAllowed": True, + "verifyEmail": True, + "loginWithEmailAllowed": True, + "duplicateEmailsAllowed": False, + "smtpServer": smtp, + } + status, body = request("POST", f"{base_url}/admin/realms", token, create_payload) + if status not in (201, 204, 409): + raise SystemExit(f"Realm create failed: status={status} body={body}") + status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token) + if status != 200 or not isinstance(realm_rep, dict): + raise SystemExit(f"Realm fetch failed: status={status}") + + realm_rep.update( + { + "enabled": True, + "registrationAllowed": False, + "resetPasswordAllowed": True, + "verifyEmail": True, + "loginWithEmailAllowed": True, + "duplicateEmailsAllowed": False, + "smtpServer": smtp, + } + ) + status, body = request("PUT", f"{base_url}/admin/realms/{realm}", token, realm_rep) + if status not in (200, 204): + raise SystemExit(f"Realm update failed: status={status} body={body}") + + def find_group(name): + status, groups = request( + "GET", + f"{base_url}/admin/realms/{realm}/groups?search={urllib.parse.quote(name)}", + token, + ) + if status != 200: + raise SystemExit(f"Group search failed for {name}: status={status}") + for group in groups or []: + if group.get("name") == name: + return group + return None + + def ensure_group(name): + group = find_group(name) + if group: + return group + status, body = request("POST", f"{base_url}/admin/realms/{realm}/groups", token, {"name": name}) + if status not in (201, 204, 409): + raise SystemExit(f"Group create failed for {name}: status={status} body={body}") + group = find_group(name) + if not group: + raise SystemExit(f"Group missing after create for {name}") + return group + + def find_user(username): + encoded = urllib.parse.quote(username) + for query in (f"username={encoded}&exact=true", f"email={encoded}&exact=true", f"search={encoded}"): + status, users = request( + "GET", + f"{base_url}/admin/realms/{realm}/users?{query}", + token, + ) + if status != 200: + raise SystemExit(f"User search failed for {username}: status={status}") + for user in users or []: + candidates = { + str(user.get("username") or "").casefold(), + str(user.get("email") or "").casefold(), + } + if username.casefold() in candidates: + return user + return None + + def ensure_user_group(username, group): + user = find_user(username) + if not user: + print(f"WARNING: Cassandra public replay user {username!r} not found; skipping membership") + return + user_id = user.get("id") + group_id = group.get("id") + if not user_id or not group_id: + raise SystemExit(f"Missing id for user/group membership: user={username}") + status, memberships = request( + "GET", + f"{base_url}/admin/realms/{realm}/users/{user_id}/groups", + token, + ) + if status != 200: + raise SystemExit(f"User group lookup failed for {username}: status={status}") + if any(item.get("id") == group_id for item in memberships or []): + return + status, body = request( + "PUT", + f"{base_url}/admin/realms/{realm}/users/{user_id}/groups/{group_id}", + token, + ) + if status not in (200, 204): + raise SystemExit(f"User group update failed for {username}: status={status} body={body}") + + ensure_group("alpha") + ensure_group("admin") + ensure_group(os.environ["CASSANDRA_TESTER_GROUP"]) + public_replay_group = ensure_group("public-replay") + for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","): + username = username.strip() + if username: + ensure_user_group(username, public_replay_group) + + status, clients = request( + "GET", + f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}", + token, + ) + if status != 200: + raise SystemExit(f"Client lookup failed: status={status}") + client_uuid = clients[0]["id"] if clients else None + client_payload = { + "clientId": client_id, + "enabled": True, + "protocol": "openid-connect", + "publicClient": False, + "standardFlowEnabled": True, + "implicitFlowEnabled": False, + "directAccessGrantsEnabled": False, + "serviceAccountsEnabled": False, + "redirectUris": [f"{app_base_url}/*"], + "webOrigins": [app_base_url], + "rootUrl": app_base_url, + "baseUrl": "/", + "attributes": { + "pkce.code.challenge.method": "S256", + "post.logout.redirect.uris": f"{app_base_url}/*", + }, + } + if not client_uuid: + status, body = request("POST", f"{base_url}/admin/realms/{realm}/clients", token, client_payload) + if status not in (201, 204, 409): + raise SystemExit(f"Client create failed: status={status} body={body}") + status, clients = request( + "GET", + f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}", + token, + ) + client_uuid = clients[0]["id"] if clients else None + if not client_uuid: + raise SystemExit("Client cassandra-web not found after create") + status, body = request( + "PUT", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}", + token, + client_payload, + ) + if status not in (200, 204): + raise SystemExit(f"Client update failed: status={status} body={body}") + + mapper_payload = { + "name": "groups", + "protocol": "openid-connect", + "protocolMapper": "oidc-group-membership-mapper", + "consentRequired": False, + "config": { + "full.path": "false", + "id.token.claim": "true", + "access.token.claim": "true", + "userinfo.token.claim": "true", + "claim.name": "groups", + "jsonType.label": "String", + }, + } + status, mappers = request( + "GET", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models", + token, + ) + if status != 200: + raise SystemExit(f"Mapper lookup failed: status={status}") + mapper_id = next((mapper.get("id") for mapper in mappers or [] if mapper.get("name") == "groups"), None) + if mapper_id: + mapper_update_payload = dict(mapper_payload) + mapper_update_payload["id"] = mapper_id + status, body = request( + "PUT", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models/{mapper_id}", + token, + mapper_update_payload, + ) + else: + status, body = request( + "POST", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models", + token, + mapper_payload, + ) + if status not in (200, 201, 204): + raise SystemExit(f"Mapper ensure failed: status={status} body={body}") + + status, secret = request( + "GET", + f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/client-secret", + token, + ) + client_secret = (secret or {}).get("value") + if status != 200 or not client_secret: + raise SystemExit(f"Client secret fetch failed: status={status}") + + vault_addr = os.environ.get("VAULT_ADDR", "http://vault.vault.svc.cluster.local:8200") + jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip() + login_payload = json.dumps({"jwt": jwt, "role": os.environ.get("VAULT_ROLE", "sso-secrets")}).encode() + req = urllib.request.Request( + f"{vault_addr}/v1/auth/kubernetes/login", + data=login_payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=20) as resp: + vault_token = json.loads(resp.read().decode())["auth"]["client_token"] + + payload = { + "data": { + "client_id": client_id, + "client_secret": client_secret, + "issuer": issuer, + "realm": realm, + "required_groups": "alpha,admin", + } + } + req = urllib.request.Request( + f"{vault_addr}/v1/kv/data/atlas/cassandra/cassandra-oidc", + data=json.dumps(payload).encode(), + headers={"X-Vault-Token": vault_token, "Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=20) as resp: + if resp.status not in (200, 204): + raise SystemExit(f"Vault write returned {resp.status}") + + print("Cassandra Keycloak realm/client ready") + PY diff --git a/services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml similarity index 96% rename from services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml index c368241b0..668c28321 100644 --- a/services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/harbor-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml # One-off job for sso/harbor-oidc-secret-ensure-10. # Purpose: harbor oidc secret ensure 10 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/hermes-dashboard-oidc-client-job.yaml b/services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml similarity index 96% rename from services/keycloak/oneoffs/hermes-dashboard-oidc-client-job.yaml rename to services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml index bc4509f75..8a5232890 100644 --- a/services/keycloak/oneoffs/hermes-dashboard-oidc-client-job.yaml +++ b/services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/hermes-dashboard-oidc-client-job.yaml +# services/keycloak/bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml # Purpose: ensure the Hermes dashboard public OIDC client exists in Keycloak. # Bump the suffix if the immutable Job needs to be rerun after it completes. apiVersion: batch/v1 diff --git a/services/keycloak/oneoffs/ldap-federation-job.yaml b/services/keycloak/bootstrap-jobs/ldap-federation-job.yaml similarity index 99% rename from services/keycloak/oneoffs/ldap-federation-job.yaml rename to services/keycloak/bootstrap-jobs/ldap-federation-job.yaml index 5cba1e764..1787299f6 100644 --- a/services/keycloak/oneoffs/ldap-federation-job.yaml +++ b/services/keycloak/bootstrap-jobs/ldap-federation-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/ldap-federation-job.yaml +# services/keycloak/bootstrap-jobs/ldap-federation-job.yaml # One-off job for sso/keycloak-ldap-federation-12. # Purpose: keycloak ldap federation 12 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml index bce9e5b4c..6500cc6fd 100644 --- a/services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/logs-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/logs-oidc-secret-ensure-job.yaml # One-off job for sso/logs-oidc-secret-ensure-10. # Purpose: logs oidc secret ensure 10 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/mas-secrets-ensure-job.yaml b/services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/mas-secrets-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml index c3bd1be0b..06ce42ab2 100644 --- a/services/keycloak/oneoffs/mas-secrets-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/mas-secrets-ensure-job.yaml +# services/keycloak/bootstrap-jobs/mas-secrets-ensure-job.yaml # One-off job for sso/mas-secrets-ensure. # Purpose: mas secrets ensure (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml similarity index 98% rename from services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml index 745adf9b2..2c40ed3ea 100644 --- a/services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/metis-node-passwords-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml # One-off job for sso/metis-node-passwords-secret-ensure-4. # Purpose: ensure per-node Metis recovery placeholders exist in Vault. # Atlas/root values are preserved while intranet IPs are standardized per node. diff --git a/services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml index e74f629cd..9be48727d 100644 --- a/services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/metis-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/metis-oidc-secret-ensure-job.yaml # One-off job for sso/metis-oidc-secret-ensure-2. # Purpose: ensure the Metis oauth2-proxy OIDC client and Vault secret exist. # Keep this manifest around; set suspend false and bump the suffix to rerun. diff --git a/services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml similarity index 98% rename from services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml index 956a2f4a4..b56da25f4 100644 --- a/services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml # One-off job for sso/metis-ssh-keys-secret-ensure-1. # Purpose: ensure Vault path maintenance/metis-ssh-keys exists for Metis key injection. # Migration behavior: if Vault path is missing/incomplete, seed from existing maintenance/metis-ssh-keys Kubernetes Secret. diff --git a/services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml index e8a87dc69..d6177aa4d 100644 --- a/services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-admin-client-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml # One-off job for sso/keycloak-portal-admin-secret-ensure-5. # Purpose: keycloak portal admin secret ensure 5 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/portal-groups-scope-ensure-job.yaml b/services/keycloak/bootstrap-jobs/portal-groups-scope-ensure-job.yaml similarity index 98% rename from services/keycloak/oneoffs/portal-groups-scope-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/portal-groups-scope-ensure-job.yaml index 3f8ee54dd..4381431e8 100644 --- a/services/keycloak/oneoffs/portal-groups-scope-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/portal-groups-scope-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-groups-scope-ensure-job.yaml +# services/keycloak/bootstrap-jobs/portal-groups-scope-ensure-job.yaml # One-off job for sso/portal-groups-scope-ensure-1. # Purpose: allow bstein-dev-home to request the groups OIDC scope. # Keep this manifest around; set suspend false and bump the suffix to rerun. diff --git a/services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml index 3c859ecf0..c272cbd57 100644 --- a/services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/quality-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/quality-oidc-secret-ensure-job.yaml # One-off job for sso/quality-oidc-secret-ensure-1. # Purpose: ensure the SonarQube oauth2-proxy OIDC client and Vault secret exist. # Keep this manifest around; set suspend false and bump the suffix to rerun. diff --git a/services/keycloak/oneoffs/realm-settings-job.yaml b/services/keycloak/bootstrap-jobs/realm-settings-job.yaml similarity index 99% rename from services/keycloak/oneoffs/realm-settings-job.yaml rename to services/keycloak/bootstrap-jobs/realm-settings-job.yaml index 2c79eec60..6d5d47120 100644 --- a/services/keycloak/oneoffs/realm-settings-job.yaml +++ b/services/keycloak/bootstrap-jobs/realm-settings-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/realm-settings-job.yaml +# services/keycloak/bootstrap-jobs/realm-settings-job.yaml # One-off job for sso/keycloak-realm-settings-37. # Purpose: keycloak realm settings 37 (see container args/env in this file). # This revision runs once to ensure the maintenance group exists for Metis access. diff --git a/services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml index d5ee0d7cc..ce35a5cc4 100644 --- a/services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/soteria-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml # One-off job for sso/soteria-oidc-secret-ensure-1. # Purpose: ensure the Soteria oauth2-proxy OIDC client and Vault secret exist. # Keep this manifest around; set suspend false and bump the suffix to rerun. diff --git a/services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml similarity index 98% rename from services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml index 15b7a3126..5e0d80a57 100644 --- a/services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/synapse-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml # One-off job for sso/synapse-oidc-secret-ensure-10. # Purpose: synapse oidc secret ensure 10 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/user-overrides-job.yaml b/services/keycloak/bootstrap-jobs/user-overrides-job.yaml similarity index 99% rename from services/keycloak/oneoffs/user-overrides-job.yaml rename to services/keycloak/bootstrap-jobs/user-overrides-job.yaml index 2d366937e..c76b8e312 100644 --- a/services/keycloak/oneoffs/user-overrides-job.yaml +++ b/services/keycloak/bootstrap-jobs/user-overrides-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/user-overrides-job.yaml +# services/keycloak/bootstrap-jobs/user-overrides-job.yaml # One-off job for sso/keycloak-user-overrides-9. # Purpose: keycloak user overrides 9 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml similarity index 96% rename from services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml index a76c52e9d..16f0de04d 100644 --- a/services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/vault-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/vault-oidc-secret-ensure-job.yaml # One-off job for sso/vault-oidc-secret-ensure-8. # Purpose: vault oidc secret ensure 8 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/veles-gitea-oidc-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml similarity index 96% rename from services/keycloak/oneoffs/veles-gitea-oidc-secret-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml index f6b0a94b0..adaf2e0eb 100644 --- a/services/keycloak/oneoffs/veles-gitea-oidc-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/veles-gitea-oidc-secret-ensure-job.yaml +# services/keycloak/bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml # One-off job for sso/veles-gitea-oidc-secret-ensure-1. # Purpose: create/update the Keycloak veles realm Gitea OIDC client, ensure the # veles-tester group exists, and write the client secret to Vault for Gitea. diff --git a/services/keycloak/oneoffs/veles-realm-ensure-job.yaml b/services/keycloak/bootstrap-jobs/veles-realm-ensure-job.yaml similarity index 99% rename from services/keycloak/oneoffs/veles-realm-ensure-job.yaml rename to services/keycloak/bootstrap-jobs/veles-realm-ensure-job.yaml index f1fc8d478..259a514d1 100644 --- a/services/keycloak/oneoffs/veles-realm-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/veles-realm-ensure-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/veles-realm-ensure-job.yaml +# services/keycloak/bootstrap-jobs/veles-realm-ensure-job.yaml # One-off job for sso/veles-realm-ensure-4. # Purpose: create the Veles realm, groups, OIDC client, SMTP settings, and Vault client secret. # Keep suspended until Veles Vault paths/policies have reconciled, then unsuspend once. diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml index 56677b8b5..45288712d 100644 --- a/services/keycloak/kustomization.yaml +++ b/services/keycloak/kustomization.yaml @@ -10,30 +10,32 @@ resources: - secretproviderclass.yaml - vault-sync-deployment.yaml - deployment.yaml - - oneoffs/realm-settings-job.yaml - - oneoffs/portal-admin-client-secret-ensure-job.yaml - - oneoffs/portal-groups-scope-ensure-job.yaml - - oneoffs/portal-e2e-client-job.yaml - - oneoffs/portal-e2e-target-client-job.yaml - - oneoffs/portal-e2e-token-exchange-permissions-job.yaml - - oneoffs/portal-e2e-token-exchange-test-job.yaml - - oneoffs/portal-e2e-execute-actions-email-test-job.yaml - - oneoffs/ldap-federation-job.yaml - - oneoffs/user-overrides-job.yaml - - oneoffs/mas-secrets-ensure-job.yaml - - oneoffs/synapse-oidc-secret-ensure-job.yaml - - oneoffs/logs-oidc-secret-ensure-job.yaml - - oneoffs/metis-oidc-secret-ensure-job.yaml - - oneoffs/soteria-oidc-secret-ensure-job.yaml - - oneoffs/quality-oidc-secret-ensure-job.yaml - - oneoffs/hermes-dashboard-oidc-client-job.yaml - - oneoffs/veles-realm-ensure-job.yaml - - oneoffs/veles-gitea-oidc-secret-ensure-job.yaml - - oneoffs/metis-ssh-keys-secret-ensure-job.yaml - - oneoffs/metis-node-passwords-secret-ensure-job.yaml - - oneoffs/harbor-oidc-secret-ensure-job.yaml - - oneoffs/vault-oidc-secret-ensure-job.yaml - - oneoffs/actual-oidc-secret-ensure-job.yaml + - bootstrap-jobs/realm-settings-job.yaml + - bootstrap-jobs/portal-admin-client-secret-ensure-job.yaml + - bootstrap-jobs/portal-groups-scope-ensure-job.yaml + - validation-jobs/portal-e2e-client-job.yaml + - validation-jobs/portal-e2e-target-client-job.yaml + - validation-jobs/portal-e2e-token-exchange-permissions-job.yaml + - validation-jobs/portal-e2e-token-exchange-test-job.yaml + - validation-jobs/portal-e2e-execute-actions-email-test-job.yaml + - bootstrap-jobs/ldap-federation-job.yaml + - bootstrap-jobs/user-overrides-job.yaml + - bootstrap-jobs/mas-secrets-ensure-job.yaml + - bootstrap-jobs/synapse-oidc-secret-ensure-job.yaml + - bootstrap-jobs/logs-oidc-secret-ensure-job.yaml + - bootstrap-jobs/metis-oidc-secret-ensure-job.yaml + - bootstrap-jobs/soteria-oidc-secret-ensure-job.yaml + - bootstrap-jobs/quality-oidc-secret-ensure-job.yaml + - bootstrap-jobs/hermes-dashboard-oidc-client-job.yaml + - bootstrap-jobs/cassandra-realm-ensure-job.yaml + - bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml + - bootstrap-jobs/veles-realm-ensure-job.yaml + - bootstrap-jobs/veles-gitea-oidc-secret-ensure-job.yaml + - bootstrap-jobs/metis-ssh-keys-secret-ensure-job.yaml + - bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml + - bootstrap-jobs/harbor-oidc-secret-ensure-job.yaml + - bootstrap-jobs/vault-oidc-secret-ensure-job.yaml + - bootstrap-jobs/actual-oidc-secret-ensure-job.yaml - service.yaml - ingress.yaml generatorOptions: @@ -55,6 +57,9 @@ configMapGenerator: - name: hermes-dashboard-oidc-client-script files: - hermes_dashboard_oidc_client_ensure.sh=scripts/hermes_dashboard_oidc_client_ensure.sh + - name: cassandra-gitea-oidc-secret-ensure-script + files: + - cassandra_gitea_oidc_secret_ensure.sh=scripts/cassandra_gitea_oidc_secret_ensure.sh - name: veles-gitea-oidc-secret-ensure-script files: - veles_gitea_oidc_secret_ensure.sh=scripts/veles_gitea_oidc_secret_ensure.sh diff --git a/services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh b/services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh new file mode 100644 index 000000000..fc1e2ae53 --- /dev/null +++ b/services/keycloak/scripts/cassandra_gitea_oidc_secret_ensure.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env sh +set -euo pipefail + +. /vault/secrets/keycloak-admin-env.sh + +KC_URL="${KEYCLOAK_SERVER:-http://keycloak.sso.svc.cluster.local}" +REALM="${KEYCLOAK_REALM:-cassandra}" +CLIENT_ID="${KEYCLOAK_CLIENT_ID:-gitea}" +PUBLIC_BASE_URL="${GITEA_PUBLIC_BASE_URL:-https://scm.bstein.dev}" +AUTH_SOURCE_NAME="${GITEA_AUTH_SOURCE_NAME:-cassandra}" +TESTER_GROUP="${CASSANDRA_GITEA_TESTER_GROUP:-cassandra-tester}" +VAULT_SECRET_PATH="${VAULT_SECRET_PATH:-kv/data/atlas/gitea/gitea-cassandra-oidc}" + +ACCESS_TOKEN="" +for attempt in 1 2 3 4 5 6 7 8 9 10; do + if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then + break + fi + echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2 + sleep $((attempt * 2)) +done +for attempt in 1 2 3 4 5; do + TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=password" \ + -d "client_id=admin-cli" \ + -d "username=${KEYCLOAK_ADMIN}" \ + -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)" + ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)" + if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then + break + fi + echo "Keycloak token request failed (attempt ${attempt})" >&2 + sleep $((attempt * 2)) +done +if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then + echo "Failed to fetch Keycloak admin token" >&2 + exit 1 +fi + +ensure_group() { + group_name="$1" + groups="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/${REALM}/groups?search=$(printf '%s' "${group_name}" | jq -sRr @uri)" || true)" + if echo "$groups" | jq -e --arg name "$group_name" '.[]? | select(.name == $name)' >/dev/null 2>&1; then + return + fi + status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "$(jq -nc --arg name "$group_name" '{name:$name}')" \ + "${KC_URL}/admin/realms/${REALM}/groups")" + if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then + echo "Keycloak group create failed for ${group_name} (status ${status})" >&2 + exit 1 + fi +} + +ensure_group "${TESTER_GROUP}" + +CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)" +CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" + +client_payload="$(jq -nc \ + --arg client_id "${CLIENT_ID}" \ + --arg root_url "${PUBLIC_BASE_URL}" \ + --arg callback "${PUBLIC_BASE_URL}/user/oauth2/${AUTH_SOURCE_NAME}/callback" \ + '{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$callback],webOrigins:[$root_url],rootUrl:$root_url,baseUrl:"/",attributes:{"pkce.code.challenge.method":"S256","post.logout.redirect.uris":($root_url + "/*")}}')" + +if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then + status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${client_payload}" \ + "${KC_URL}/admin/realms/${REALM}/clients")" + if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then + echo "Keycloak client create failed for ${CLIENT_ID} (status ${status})" >&2 + exit 1 + fi + CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)" + CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" +fi + +if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then + echo "Keycloak client ${CLIENT_ID} not found after create" >&2 + exit 1 +fi + +status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${client_payload}" \ + "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}")" +if [ "$status" != "200" ] && [ "$status" != "204" ]; then + echo "Keycloak client update failed for ${CLIENT_ID} (status ${status})" >&2 + exit 1 +fi + +mapper_payload="$(jq -nc \ + '{name:"groups",protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper",consentRequired:false,config:{"full.path":"false","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","claim.name":"groups","jsonType.label":"String"}}')" +mappers="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models" || true)" +MAPPER_ID="$(echo "$mappers" | jq -r '.[]? | select(.name == "groups") | .id' | head -n1 || true)" +if [ -n "$MAPPER_ID" ] && [ "$MAPPER_ID" != "null" ]; then + status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${mapper_payload}" \ + "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models/${MAPPER_ID}")" +else + status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${mapper_payload}" \ + "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models")" +fi +if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then + echo "Keycloak groups mapper ensure failed (status ${status})" >&2 + exit 1 +fi + +CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/client-secret" | jq -r '.value' 2>/dev/null || true)" +if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then + echo "Keycloak client secret not found for ${CLIENT_ID}" >&2 + exit 1 +fi + +vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" +vault_role="${VAULT_ROLE:-sso-secrets}" +jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" +login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" +vault_token="$(curl -sS --request POST --data "${login_payload}" \ + "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" +if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then + echo "vault login failed" >&2 + exit 1 +fi + +payload="$(jq -nc \ + --arg client_id "${CLIENT_ID}" \ + --arg client_secret "${CLIENT_SECRET}" \ + --arg discovery_url "${PUBLIC_BASE_URL}" \ + --arg issuer "https://sso.bstein.dev/realms/${REALM}" \ + --arg auto_discovery_url "https://sso.bstein.dev/realms/${REALM}/.well-known/openid-configuration" \ + --arg auth_source_name "${AUTH_SOURCE_NAME}" \ + --arg tester_group "${TESTER_GROUP}" \ + --arg group_team_map "{\"${TESTER_GROUP}\":{\"cassandra-alpha\":[\"testers\"]}}" \ + '{data:{client_id:$client_id,client_secret:$client_secret,issuer:$issuer,openid_auto_discovery_url:$auto_discovery_url,auth_source_name:$auth_source_name,required_claim_name:"groups",required_claim_value:$tester_group,group_claim_name:"groups",restricted_group:$tester_group,group_team_map:$group_team_map,gitea_base_url:$discovery_url}}')" + +curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \ + -H 'Content-Type: application/json' \ + -d "${payload}" "${vault_addr}/v1/${VAULT_SECRET_PATH}" >/dev/null + +echo "Cassandra realm Gitea OIDC client ready" diff --git a/services/keycloak/oneoffs/portal-e2e-client-job.yaml b/services/keycloak/validation-jobs/portal-e2e-client-job.yaml similarity index 99% rename from services/keycloak/oneoffs/portal-e2e-client-job.yaml rename to services/keycloak/validation-jobs/portal-e2e-client-job.yaml index 74f55872b..287d8d43c 100644 --- a/services/keycloak/oneoffs/portal-e2e-client-job.yaml +++ b/services/keycloak/validation-jobs/portal-e2e-client-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-e2e-client-job.yaml +# services/keycloak/validation-jobs/portal-e2e-client-job.yaml # One-off job for sso/keycloak-portal-e2e-client-8. # Purpose: keycloak portal e2e client 8 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml b/services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml similarity index 97% rename from services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml rename to services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml index 836414f1f..f900c5a7e 100644 --- a/services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml +++ b/services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-e2e-execute-actions-email-test-job.yaml +# services/keycloak/validation-jobs/portal-e2e-execute-actions-email-test-job.yaml # One-off job for sso/keycloak-portal-e2e-execute-actions-email-14. # Purpose: keycloak portal e2e execute actions email 14 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/portal-e2e-target-client-job.yaml b/services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml similarity index 99% rename from services/keycloak/oneoffs/portal-e2e-target-client-job.yaml rename to services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml index 317692448..93459877a 100644 --- a/services/keycloak/oneoffs/portal-e2e-target-client-job.yaml +++ b/services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-e2e-target-client-job.yaml +# services/keycloak/validation-jobs/portal-e2e-target-client-job.yaml # One-off job for sso/keycloak-portal-e2e-target-7. # Purpose: keycloak portal e2e target 7 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml b/services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml similarity index 99% rename from services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml rename to services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml index d01f5701e..8ef05e82f 100644 --- a/services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml +++ b/services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-e2e-token-exchange-permissions-job.yaml +# services/keycloak/validation-jobs/portal-e2e-token-exchange-permissions-job.yaml # One-off job for sso/keycloak-portal-e2e-token-exchange-permissions-11. # Purpose: keycloak portal e2e token exchange permissions 11 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml b/services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml similarity index 98% rename from services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml rename to services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml index eb05e09c6..4626c117f 100644 --- a/services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml +++ b/services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml @@ -1,4 +1,4 @@ -# services/keycloak/oneoffs/portal-e2e-token-exchange-test-job.yaml +# services/keycloak/validation-jobs/portal-e2e-token-exchange-test-job.yaml # One-off job for sso/keycloak-portal-e2e-token-exchange-test-7. # Purpose: keycloak portal e2e token exchange test 7 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/logging/Jenkinsfile.data-prepper b/services/logging/Jenkinsfile.data-prepper index cffcdcbd6..c8b576870 100644 --- a/services/logging/Jenkinsfile.data-prepper +++ b/services/logging/Jenkinsfile.data-prepper @@ -57,6 +57,11 @@ spec: command: - cat tty: true + - name: semgrep + image: semgrep/semgrep:1.171.0 + command: + - cat + tty: true - name: kaniko image: gcr.io/kaniko-project/executor:v1.23.2-debug command: @@ -80,6 +85,8 @@ spec: SONARQUBE_TOKEN = credentials('sonarqube-token') QUALITY_GATE_SONARQUBE_ENFORCE = '1' QUALITY_GATE_SONARQUBE_REPORT = 'build/sonarqube-quality-gate.json' + QUALITY_GATE_SEMGREP_ENFORCE = '0' + QUALITY_GATE_SEMGREP_REPORT = 'build/semgrep-report.json' QUALITY_GATE_IRONBANK_ENFORCE = '1' QUALITY_GATE_IRONBANK_REQUIRED = '1' QUALITY_GATE_IRONBANK_REPORT = 'build/ironbank-compliance.json' @@ -103,6 +110,20 @@ spec: } stage('Collect quality evidence') { steps { + container('semgrep') { + sh '''#!/bin/sh + set -eu + mkdir -p build + set +e + semgrep scan --config auto --metrics=off --json --output build/semgrep.json \ + services/logging/Jenkinsfile.data-prepper \ + dockerfiles/Dockerfile.data-prepper + semgrep_rc=$? + set -e + printf '%s\n' "${semgrep_rc}" > build/semgrep.rc + python3 ci/scripts/semgrep_report.py --semgrep-json build/semgrep.json --exit-code "${semgrep_rc}" --output "${QUALITY_GATE_SEMGREP_REPORT}" --sonar-issues-output build/semgrep-sonar-issues.json + ''' + } container('quality-tools') { sh '''#!/usr/bin/env bash set -euo pipefail @@ -116,6 +137,7 @@ spec: "-Dsonar.inclusions=services/logging/Jenkinsfile.data-prepper,dockerfiles/Dockerfile.data-prepper" "-Dsonar.exclusions=**/.git/**,**/build/**,**/dist/**,**/node_modules/**,**/.venv/**,**/__pycache__/**" ) + [ -f build/semgrep-sonar-issues.json ] && args+=("-Dsonar.externalIssuesReportPaths=build/semgrep-sonar-issues.json") set +e sonar-scanner "${args[@]}" | tee build/sonar-scanner.log sonar_rc=${PIPESTATUS[0]} @@ -369,6 +391,18 @@ EOF esac fi + if enabled "${QUALITY_GATE_SEMGREP_ENFORCE:-0}"; then + semgrep_status="$(jq -r '.status // empty' build/semgrep-report.json 2>/dev/null | tr '[:upper:]' '[:lower:]')" + [ -n "${semgrep_status}" ] || semgrep_status="missing" + case "${semgrep_status}" in + ok|pass|passed|success) ;; + *) + echo "semgrep gate failed: ${semgrep_status}" >&2 + fail=1 + ;; + esac + fi + if enabled "${QUALITY_GATE_IRONBANK_ENFORCE:-1}"; then ironbank_required="${QUALITY_GATE_IRONBANK_REQUIRED:-1}" compliant="$(jq -r '.compliant // empty' build/ironbank-compliance.json 2>/dev/null || true)" @@ -488,6 +522,15 @@ EOF sonarqube_check="failed" fi fi + semgrep_check="not_applicable" + if [ -f build/semgrep-report.json ]; then + semgrep_status="$(jq -r '.status // empty' build/semgrep-report.json 2>/dev/null | tr '[:upper:]' '[:lower:]')" + case "${semgrep_status}" in + ok|pass|passed|success) semgrep_check="ok" ;; + "") semgrep_check="failed" ;; + *) semgrep_check="failed" ;; + esac + fi supply_chain_check="not_applicable" if [ -f build/ironbank-compliance.json ]; then compliant="$(jq -r '.compliant // empty' build/ironbank-compliance.json 2>/dev/null)" @@ -573,6 +616,7 @@ data_prepper_quality_gate_checks_total{suite="${suite}",check="loc",result="not_ data_prepper_quality_gate_checks_total{suite="${suite}",check="docs_naming",result="not_applicable"} 1 data_prepper_quality_gate_checks_total{suite="${suite}",check="gate_glue",result="${gate_glue_check}"} 1 data_prepper_quality_gate_checks_total{suite="${suite}",check="sonarqube",result="${sonarqube_check}"} 1 +data_prepper_quality_gate_checks_total{suite="${suite}",check="semgrep",result="${semgrep_check}"} 1 data_prepper_quality_gate_checks_total{suite="${suite}",check="supply_chain",result="${supply_chain_check}"} 1 # TYPE platform_quality_gate_test_case_result gauge METRICS diff --git a/services/logging/oneoffs/opensearch-dashboards-setup-job.yaml b/services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml similarity index 97% rename from services/logging/oneoffs/opensearch-dashboards-setup-job.yaml rename to services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml index 1d1a9b680..c9bc42cfd 100644 --- a/services/logging/oneoffs/opensearch-dashboards-setup-job.yaml +++ b/services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml @@ -1,4 +1,4 @@ -# services/logging/oneoffs/opensearch-dashboards-setup-job.yaml +# services/logging/bootstrap-jobs/opensearch-dashboards-setup-job.yaml # One-off job for logging/opensearch-dashboards-setup-4. # Purpose: opensearch dashboards setup 4 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/logging/oneoffs/opensearch-ism-job.yaml b/services/logging/bootstrap-jobs/opensearch-ism-job.yaml similarity index 98% rename from services/logging/oneoffs/opensearch-ism-job.yaml rename to services/logging/bootstrap-jobs/opensearch-ism-job.yaml index 476bca7a9..184c49676 100644 --- a/services/logging/oneoffs/opensearch-ism-job.yaml +++ b/services/logging/bootstrap-jobs/opensearch-ism-job.yaml @@ -1,4 +1,4 @@ -# services/logging/oneoffs/opensearch-ism-job.yaml +# services/logging/bootstrap-jobs/opensearch-ism-job.yaml # One-off job for logging/opensearch-ism-setup-5. # Purpose: opensearch ism setup 5 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/logging/oneoffs/opensearch-observability-setup-job.yaml b/services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml similarity index 95% rename from services/logging/oneoffs/opensearch-observability-setup-job.yaml rename to services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml index 6caa07654..497e06e81 100644 --- a/services/logging/oneoffs/opensearch-observability-setup-job.yaml +++ b/services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml @@ -1,4 +1,4 @@ -# services/logging/oneoffs/opensearch-observability-setup-job.yaml +# services/logging/bootstrap-jobs/opensearch-observability-setup-job.yaml # One-off job for logging/opensearch-observability-setup-2. # Purpose: opensearch observability setup 2 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/logging/kustomization.yaml b/services/logging/kustomization.yaml index 5a4c8334a..cfed11f99 100644 --- a/services/logging/kustomization.yaml +++ b/services/logging/kustomization.yaml @@ -1,6 +1,7 @@ # services/logging/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: logging resources: - namespace.yaml - opensearch-dashboards-objects.yaml @@ -15,9 +16,9 @@ resources: - opensearch-dashboards-helmrelease.yaml - data-prepper-helmrelease.yaml - otel-collector-helmrelease.yaml - - oneoffs/opensearch-ism-job.yaml - - oneoffs/opensearch-dashboards-setup-job.yaml - - oneoffs/opensearch-observability-setup-job.yaml + - bootstrap-jobs/opensearch-ism-job.yaml + - bootstrap-jobs/opensearch-dashboards-setup-job.yaml + - bootstrap-jobs/opensearch-observability-setup-job.yaml - opensearch-single-node-tune-cronjob.yaml - opensearch-prune-cronjob.yaml - fluent-bit-helmrelease.yaml diff --git a/services/logging/opensearch-dashboards-objects.yaml b/services/logging/opensearch-dashboards-objects.yaml index fafcd3388..cd3ef8a24 100644 --- a/services/logging/opensearch-dashboards-objects.yaml +++ b/services/logging/opensearch-dashboards-objects.yaml @@ -1,5 +1,5 @@ # services/logging/opensearch-dashboards-objects.yaml -# Generated by scripts/dashboards_render_logs.py --build +# Generated by scripts/render/dashboards_render_logs.py --build apiVersion: v1 kind: ConfigMap metadata: diff --git a/services/logging/opensearch-observability-objects.yaml b/services/logging/opensearch-observability-objects.yaml index 19ed1953f..be33e2cfd 100644 --- a/services/logging/opensearch-observability-objects.yaml +++ b/services/logging/opensearch-observability-objects.yaml @@ -1,5 +1,5 @@ # services/logging/opensearch-observability-objects.yaml -# Generated by scripts/logging_render_observability.py --build +# Generated by scripts/render/logging_render_observability.py --build apiVersion: v1 kind: ConfigMap metadata: diff --git a/services/maintenance/ariadne-deployment.yaml b/services/maintenance/apps/ariadne-deployment.yaml similarity index 99% rename from services/maintenance/ariadne-deployment.yaml rename to services/maintenance/apps/ariadne-deployment.yaml index f39ddba3a..42d9a8360 100644 --- a/services/maintenance/ariadne-deployment.yaml +++ b/services/maintenance/apps/ariadne-deployment.yaml @@ -1,4 +1,4 @@ -# services/maintenance/ariadne-deployment.yaml +# services/maintenance/apps/ariadne-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: diff --git a/services/maintenance/ariadne-rbac.yaml b/services/maintenance/apps/ariadne-rbac.yaml similarity index 97% rename from services/maintenance/ariadne-rbac.yaml rename to services/maintenance/apps/ariadne-rbac.yaml index 73c5e3b55..941942bce 100644 --- a/services/maintenance/ariadne-rbac.yaml +++ b/services/maintenance/apps/ariadne-rbac.yaml @@ -1,4 +1,4 @@ -# services/maintenance/ariadne-rbac.yaml +# services/maintenance/apps/ariadne-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/services/maintenance/ariadne-service.yaml b/services/maintenance/apps/ariadne-service.yaml similarity index 78% rename from services/maintenance/ariadne-service.yaml rename to services/maintenance/apps/ariadne-service.yaml index 9c93e1df0..fafa6430b 100644 --- a/services/maintenance/ariadne-service.yaml +++ b/services/maintenance/apps/ariadne-service.yaml @@ -1,4 +1,4 @@ -# services/maintenance/ariadne-service.yaml +# services/maintenance/apps/ariadne-service.yaml apiVersion: v1 kind: Service metadata: diff --git a/services/maintenance/ariadne-serviceaccount.yaml b/services/maintenance/apps/ariadne-serviceaccount.yaml similarity index 69% rename from services/maintenance/ariadne-serviceaccount.yaml rename to services/maintenance/apps/ariadne-serviceaccount.yaml index 9adcef7e1..810a4680e 100644 --- a/services/maintenance/ariadne-serviceaccount.yaml +++ b/services/maintenance/apps/ariadne-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/ariadne-serviceaccount.yaml +# services/maintenance/apps/ariadne-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/apps/kustomization.yaml b/services/maintenance/apps/kustomization.yaml new file mode 100644 index 000000000..8b07825fc --- /dev/null +++ b/services/maintenance/apps/kustomization.yaml @@ -0,0 +1,29 @@ +# services/maintenance/apps/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: maintenance +resources: + - metis-configmap.yaml + - metis-data-pvc.yaml + - soteria-configmap.yaml + - ariadne-serviceaccount.yaml + - soteria-serviceaccount.yaml + - ariadne-rbac.yaml + - soteria-rbac.yaml + - metis-serviceaccount.yaml + - metis-rbac.yaml + - metis-token-sync-serviceaccount.yaml + - ariadne-deployment.yaml + - metis-deployment.yaml + - soteria-deployment.yaml + - metis-k3s-token-sync-cronjob.yaml + - ariadne-service.yaml + - soteria-service.yaml + - metis-service.yaml +configMapGenerator: + - name: metis-inventory + namespace: maintenance + files: + - inventory.yaml=metis-inventory.yaml + options: + disableNameSuffixHash: true diff --git a/services/maintenance/metis-configmap.yaml b/services/maintenance/apps/metis-configmap.yaml similarity index 97% rename from services/maintenance/metis-configmap.yaml rename to services/maintenance/apps/metis-configmap.yaml index c6254da52..393993f10 100644 --- a/services/maintenance/metis-configmap.yaml +++ b/services/maintenance/apps/metis-configmap.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-configmap.yaml +# services/maintenance/apps/metis-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: diff --git a/services/maintenance/metis-data-pvc.yaml b/services/maintenance/apps/metis-data-pvc.yaml similarity index 82% rename from services/maintenance/metis-data-pvc.yaml rename to services/maintenance/apps/metis-data-pvc.yaml index c708c95e0..97300f3fb 100644 --- a/services/maintenance/metis-data-pvc.yaml +++ b/services/maintenance/apps/metis-data-pvc.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-data-pvc.yaml +# services/maintenance/apps/metis-data-pvc.yaml apiVersion: v1 kind: PersistentVolumeClaim metadata: diff --git a/services/maintenance/metis-deployment.yaml b/services/maintenance/apps/metis-deployment.yaml similarity index 98% rename from services/maintenance/metis-deployment.yaml rename to services/maintenance/apps/metis-deployment.yaml index 6d78c4699..bd42046c6 100644 --- a/services/maintenance/metis-deployment.yaml +++ b/services/maintenance/apps/metis-deployment.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-deployment.yaml +# services/maintenance/apps/metis-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: diff --git a/services/maintenance/metis-inventory.yaml b/services/maintenance/apps/metis-inventory.yaml similarity index 98% rename from services/maintenance/metis-inventory.yaml rename to services/maintenance/apps/metis-inventory.yaml index 3e9a3bf67..12aac5659 100644 --- a/services/maintenance/metis-inventory.yaml +++ b/services/maintenance/apps/metis-inventory.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-inventory.yaml +# services/maintenance/apps/metis-inventory.yaml classes: - name: rpi5-ubuntu-worker arch: arm64 diff --git a/services/maintenance/metis-k3s-token-sync-cronjob.yaml b/services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml similarity index 96% rename from services/maintenance/metis-k3s-token-sync-cronjob.yaml rename to services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml index 7ef490876..ff9ca09fd 100644 --- a/services/maintenance/metis-k3s-token-sync-cronjob.yaml +++ b/services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-k3s-token-sync-cronjob.yaml +# services/maintenance/apps/metis-k3s-token-sync-cronjob.yaml apiVersion: batch/v1 kind: CronJob metadata: @@ -6,6 +6,7 @@ metadata: namespace: maintenance spec: schedule: "11 */6 * * *" + suspend: true concurrencyPolicy: Forbid successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 2 diff --git a/services/maintenance/metis-rbac.yaml b/services/maintenance/apps/metis-rbac.yaml similarity index 96% rename from services/maintenance/metis-rbac.yaml rename to services/maintenance/apps/metis-rbac.yaml index 18a81f70f..3d0df5279 100644 --- a/services/maintenance/metis-rbac.yaml +++ b/services/maintenance/apps/metis-rbac.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-rbac.yaml +# services/maintenance/apps/metis-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/services/maintenance/metis-service.yaml b/services/maintenance/apps/metis-service.yaml similarity index 86% rename from services/maintenance/metis-service.yaml rename to services/maintenance/apps/metis-service.yaml index 3e08eb699..1bf3f9a05 100644 --- a/services/maintenance/metis-service.yaml +++ b/services/maintenance/apps/metis-service.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-service.yaml +# services/maintenance/apps/metis-service.yaml apiVersion: v1 kind: Service metadata: diff --git a/services/maintenance/metis-serviceaccount.yaml b/services/maintenance/apps/metis-serviceaccount.yaml similarity index 70% rename from services/maintenance/metis-serviceaccount.yaml rename to services/maintenance/apps/metis-serviceaccount.yaml index eb94b11a6..0e2f47d6a 100644 --- a/services/maintenance/metis-serviceaccount.yaml +++ b/services/maintenance/apps/metis-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-serviceaccount.yaml +# services/maintenance/apps/metis-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/metis-token-sync-serviceaccount.yaml b/services/maintenance/apps/metis-token-sync-serviceaccount.yaml similarity index 59% rename from services/maintenance/metis-token-sync-serviceaccount.yaml rename to services/maintenance/apps/metis-token-sync-serviceaccount.yaml index 55438dcc5..80f79480c 100644 --- a/services/maintenance/metis-token-sync-serviceaccount.yaml +++ b/services/maintenance/apps/metis-token-sync-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-token-sync-serviceaccount.yaml +# services/maintenance/apps/metis-token-sync-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/soteria-configmap.yaml b/services/maintenance/apps/soteria-configmap.yaml similarity index 97% rename from services/maintenance/soteria-configmap.yaml rename to services/maintenance/apps/soteria-configmap.yaml index 80c461592..a8c6bcb17 100644 --- a/services/maintenance/soteria-configmap.yaml +++ b/services/maintenance/apps/soteria-configmap.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-configmap.yaml +# services/maintenance/apps/soteria-configmap.yaml apiVersion: v1 kind: ConfigMap metadata: diff --git a/services/maintenance/soteria-deployment.yaml b/services/maintenance/apps/soteria-deployment.yaml similarity index 98% rename from services/maintenance/soteria-deployment.yaml rename to services/maintenance/apps/soteria-deployment.yaml index f6c1d1109..2d79ed18d 100644 --- a/services/maintenance/soteria-deployment.yaml +++ b/services/maintenance/apps/soteria-deployment.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-deployment.yaml +# services/maintenance/apps/soteria-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: diff --git a/services/maintenance/soteria-rbac.yaml b/services/maintenance/apps/soteria-rbac.yaml similarity index 95% rename from services/maintenance/soteria-rbac.yaml rename to services/maintenance/apps/soteria-rbac.yaml index 843028fef..a64693ef6 100644 --- a/services/maintenance/soteria-rbac.yaml +++ b/services/maintenance/apps/soteria-rbac.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-rbac.yaml +# services/maintenance/apps/soteria-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/services/maintenance/soteria-service.yaml b/services/maintenance/apps/soteria-service.yaml similarity index 87% rename from services/maintenance/soteria-service.yaml rename to services/maintenance/apps/soteria-service.yaml index bdaa5dc22..2e901ce8c 100644 --- a/services/maintenance/soteria-service.yaml +++ b/services/maintenance/apps/soteria-service.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-service.yaml +# services/maintenance/apps/soteria-service.yaml apiVersion: v1 kind: Service metadata: diff --git a/services/maintenance/soteria-serviceaccount.yaml b/services/maintenance/apps/soteria-serviceaccount.yaml similarity index 70% rename from services/maintenance/soteria-serviceaccount.yaml rename to services/maintenance/apps/soteria-serviceaccount.yaml index 5b19d8831..4060ea854 100644 --- a/services/maintenance/soteria-serviceaccount.yaml +++ b/services/maintenance/apps/soteria-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-serviceaccount.yaml +# services/maintenance/apps/soteria-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/image.yaml b/services/maintenance/bootstrap/image.yaml similarity index 98% rename from services/maintenance/image.yaml rename to services/maintenance/bootstrap/image.yaml index 09a322dca..789b04278 100644 --- a/services/maintenance/image.yaml +++ b/services/maintenance/bootstrap/image.yaml @@ -1,4 +1,4 @@ -# services/maintenance/image.yaml +# services/maintenance/bootstrap/image.yaml apiVersion: image.toolkit.fluxcd.io/v1beta2 kind: ImageRepository metadata: diff --git a/services/maintenance/bootstrap/kustomization.yaml b/services/maintenance/bootstrap/kustomization.yaml new file mode 100644 index 000000000..23d8b7a11 --- /dev/null +++ b/services/maintenance/bootstrap/kustomization.yaml @@ -0,0 +1,10 @@ +# services/maintenance/bootstrap/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: maintenance +resources: + - namespace.yaml + - image.yaml + - secretproviderclass.yaml + - vault-serviceaccount.yaml + - vault-sync-deployment.yaml diff --git a/services/maintenance/namespace.yaml b/services/maintenance/bootstrap/namespace.yaml similarity index 55% rename from services/maintenance/namespace.yaml rename to services/maintenance/bootstrap/namespace.yaml index dce28b038..7e42c2d66 100644 --- a/services/maintenance/namespace.yaml +++ b/services/maintenance/bootstrap/namespace.yaml @@ -1,4 +1,4 @@ -# services/maintenance/namespace.yaml +# services/maintenance/bootstrap/namespace.yaml apiVersion: v1 kind: Namespace metadata: diff --git a/services/maintenance/secretproviderclass.yaml b/services/maintenance/bootstrap/secretproviderclass.yaml similarity index 96% rename from services/maintenance/secretproviderclass.yaml rename to services/maintenance/bootstrap/secretproviderclass.yaml index 1070101d3..3dafec611 100644 --- a/services/maintenance/secretproviderclass.yaml +++ b/services/maintenance/bootstrap/secretproviderclass.yaml @@ -1,4 +1,4 @@ -# services/maintenance/secretproviderclass.yaml +# services/maintenance/bootstrap/secretproviderclass.yaml apiVersion: secrets-store.csi.x-k8s.io/v1 kind: SecretProviderClass metadata: diff --git a/services/maintenance/vault-serviceaccount.yaml b/services/maintenance/bootstrap/vault-serviceaccount.yaml similarity index 63% rename from services/maintenance/vault-serviceaccount.yaml rename to services/maintenance/bootstrap/vault-serviceaccount.yaml index f60b43ec9..f7a1ac4aa 100644 --- a/services/maintenance/vault-serviceaccount.yaml +++ b/services/maintenance/bootstrap/vault-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/vault-serviceaccount.yaml +# services/maintenance/bootstrap/vault-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/vault-sync-deployment.yaml b/services/maintenance/bootstrap/vault-sync-deployment.yaml similarity index 95% rename from services/maintenance/vault-sync-deployment.yaml rename to services/maintenance/bootstrap/vault-sync-deployment.yaml index 0f70888f7..3c451bf56 100644 --- a/services/maintenance/vault-sync-deployment.yaml +++ b/services/maintenance/bootstrap/vault-sync-deployment.yaml @@ -1,4 +1,4 @@ -# services/maintenance/vault-sync-deployment.yaml +# services/maintenance/bootstrap/vault-sync-deployment.yaml apiVersion: apps/v1 kind: Deployment metadata: diff --git a/services/maintenance/kustomization.yaml b/services/maintenance/kustomization.yaml index b1d090854..27a4540a3 100644 --- a/services/maintenance/kustomization.yaml +++ b/services/maintenance/kustomization.yaml @@ -1,55 +1,14 @@ # services/maintenance/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: maintenance resources: - - namespace.yaml - - image.yaml - - secretproviderclass.yaml - - metis-configmap.yaml - - metis-data-pvc.yaml - - soteria-configmap.yaml - - vault-serviceaccount.yaml - - vault-sync-deployment.yaml - - ariadne-serviceaccount.yaml - - soteria-serviceaccount.yaml - - ariadne-rbac.yaml - - soteria-rbac.yaml - - disable-k3s-traefik-serviceaccount.yaml - - disable-k3s-traefik-rbac.yaml - - k3s-traefik-cleanup-rbac.yaml - - metis-serviceaccount.yaml - - metis-rbac.yaml - - metis-token-sync-serviceaccount.yaml - - node-nofile-serviceaccount.yaml - - titan-24-docker-serviceaccount.yaml - - pod-cleaner-rbac.yaml - - ariadne-deployment.yaml - - metis-deployment.yaml - - soteria-deployment.yaml - - oneoffs/ariadne-migrate-job.yaml - - oneoffs/titan-24-rootfs-sweep-job.yaml - - ariadne-service.yaml - - soteria-service.yaml - - disable-k3s-traefik-daemonset.yaml - - oneoffs/k3s-traefik-cleanup-job.yaml - - node-nofile-daemonset.yaml - - rpi-resource-reservation-daemonset.yaml - - metis-sentinel-amd64-daemonset.yaml - - metis-sentinel-arm64-daemonset.yaml - - k3s-agent-restart-daemonset.yaml - - titan-24-docker-daemonset.yaml - - titan-22-link-keeper-daemonset.yaml - - node-image-sweeper-serviceaccount.yaml - - node-image-sweeper-daemonset.yaml - - metis-service.yaml - - soteria-networkpolicy.yaml - - oauth2-proxy-soteria-networkpolicy.yaml - - soteria-ingress.yaml - - soteria-certificate.yaml - - oauth2-proxy-soteria.yaml - - oauth2-proxy-metis.yaml - - metis-certificate.yaml - - metis-ingress.yaml + - bootstrap + - apps + - migration-jobs + - node-ops + - repair-jobs + - networking images: - name: registry.bstein.dev/bstein/ariadne newTag: 0.1.0-356 # {"$imagepolicy": "maintenance:ariadne:tag"} @@ -57,46 +16,3 @@ images: newTag: 0.1.0-243-arm64 # {"$imagepolicy": "maintenance:metis-arm64:tag"} - name: registry.bstein.dev/bstein/soteria newTag: 0.1.0-120 # {"$imagepolicy": "maintenance:soteria:tag"} -configMapGenerator: - - name: metis-inventory - namespace: maintenance - files: - - inventory.yaml=metis-inventory.yaml - options: - disableNameSuffixHash: true - - name: disable-k3s-traefik-script - namespace: maintenance - files: - - disable_k3s_traefik.sh=scripts/disable_k3s_traefik.sh - options: - disableNameSuffixHash: true - - name: k3s-traefik-cleanup-script - namespace: maintenance - files: - - k3s_traefik_cleanup.sh=scripts/k3s_traefik_cleanup.sh - options: - disableNameSuffixHash: true - - name: node-nofile-script - namespace: maintenance - files: - - node_nofile.sh=scripts/node_nofile.sh - options: - disableNameSuffixHash: true - - name: node-image-sweeper-script - namespace: maintenance - files: - - node_image_sweeper.sh=scripts/node_image_sweeper.sh - options: - disableNameSuffixHash: true - - name: titan-24-docker-script - namespace: maintenance - files: - - titan_24_docker.sh=scripts/titan_24_docker.sh - options: - disableNameSuffixHash: true - - name: rpi-resource-reservation-script - namespace: maintenance - files: - - rpi_resource_reservation.sh=scripts/rpi_resource_reservation.sh - options: - disableNameSuffixHash: true diff --git a/services/maintenance/oneoffs/ariadne-migrate-job.yaml b/services/maintenance/migration-jobs/ariadne-migrate-job.yaml similarity index 96% rename from services/maintenance/oneoffs/ariadne-migrate-job.yaml rename to services/maintenance/migration-jobs/ariadne-migrate-job.yaml index ecac68d4d..8b1cd9729 100644 --- a/services/maintenance/oneoffs/ariadne-migrate-job.yaml +++ b/services/maintenance/migration-jobs/ariadne-migrate-job.yaml @@ -1,4 +1,4 @@ -# services/maintenance/oneoffs/ariadne-migrate-job.yaml +# services/maintenance/migration-jobs/ariadne-migrate-job.yaml # One-off job for maintenance/ariadne-migrate-2. # Purpose: ariadne migrate 2 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/maintenance/migration-jobs/kustomization.yaml b/services/maintenance/migration-jobs/kustomization.yaml new file mode 100644 index 000000000..fb272af7d --- /dev/null +++ b/services/maintenance/migration-jobs/kustomization.yaml @@ -0,0 +1,6 @@ +# services/maintenance/migration-jobs/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: maintenance +resources: + - ariadne-migrate-job.yaml diff --git a/services/maintenance/networking/kustomization.yaml b/services/maintenance/networking/kustomization.yaml new file mode 100644 index 000000000..c3a47e555 --- /dev/null +++ b/services/maintenance/networking/kustomization.yaml @@ -0,0 +1,13 @@ +# services/maintenance/networking/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: maintenance +resources: + - soteria-networkpolicy.yaml + - oauth2-proxy-soteria-networkpolicy.yaml + - soteria-ingress.yaml + - soteria-certificate.yaml + - oauth2-proxy-soteria.yaml + - oauth2-proxy-metis.yaml + - metis-certificate.yaml + - metis-ingress.yaml diff --git a/services/maintenance/metis-certificate.yaml b/services/maintenance/networking/metis-certificate.yaml similarity index 80% rename from services/maintenance/metis-certificate.yaml rename to services/maintenance/networking/metis-certificate.yaml index b7cf0f812..fda18c34f 100644 --- a/services/maintenance/metis-certificate.yaml +++ b/services/maintenance/networking/metis-certificate.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-certificate.yaml +# services/maintenance/networking/metis-certificate.yaml apiVersion: cert-manager.io/v1 kind: Certificate metadata: diff --git a/services/maintenance/metis-ingress.yaml b/services/maintenance/networking/metis-ingress.yaml similarity index 92% rename from services/maintenance/metis-ingress.yaml rename to services/maintenance/networking/metis-ingress.yaml index 193b0f12c..173c2f93b 100644 --- a/services/maintenance/metis-ingress.yaml +++ b/services/maintenance/networking/metis-ingress.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-ingress.yaml +# services/maintenance/networking/metis-ingress.yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/services/maintenance/oauth2-proxy-metis.yaml b/services/maintenance/networking/oauth2-proxy-metis.yaml similarity index 98% rename from services/maintenance/oauth2-proxy-metis.yaml rename to services/maintenance/networking/oauth2-proxy-metis.yaml index eba18a4c6..6745e1042 100644 --- a/services/maintenance/oauth2-proxy-metis.yaml +++ b/services/maintenance/networking/oauth2-proxy-metis.yaml @@ -1,4 +1,4 @@ -# services/maintenance/oauth2-proxy-metis.yaml +# services/maintenance/networking/oauth2-proxy-metis.yaml apiVersion: v1 kind: Service metadata: diff --git a/services/maintenance/oauth2-proxy-soteria-networkpolicy.yaml b/services/maintenance/networking/oauth2-proxy-soteria-networkpolicy.yaml similarity index 86% rename from services/maintenance/oauth2-proxy-soteria-networkpolicy.yaml rename to services/maintenance/networking/oauth2-proxy-soteria-networkpolicy.yaml index b25eda105..3745623c2 100644 --- a/services/maintenance/oauth2-proxy-soteria-networkpolicy.yaml +++ b/services/maintenance/networking/oauth2-proxy-soteria-networkpolicy.yaml @@ -1,4 +1,4 @@ -# services/maintenance/oauth2-proxy-soteria-networkpolicy.yaml +# services/maintenance/networking/oauth2-proxy-soteria-networkpolicy.yaml apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/services/maintenance/oauth2-proxy-soteria.yaml b/services/maintenance/networking/oauth2-proxy-soteria.yaml similarity index 98% rename from services/maintenance/oauth2-proxy-soteria.yaml rename to services/maintenance/networking/oauth2-proxy-soteria.yaml index 92776ff9e..a21a9293c 100644 --- a/services/maintenance/oauth2-proxy-soteria.yaml +++ b/services/maintenance/networking/oauth2-proxy-soteria.yaml @@ -1,4 +1,4 @@ -# services/maintenance/oauth2-proxy-soteria.yaml +# services/maintenance/networking/oauth2-proxy-soteria.yaml apiVersion: v1 kind: Service metadata: diff --git a/services/maintenance/soteria-certificate.yaml b/services/maintenance/networking/soteria-certificate.yaml similarity index 79% rename from services/maintenance/soteria-certificate.yaml rename to services/maintenance/networking/soteria-certificate.yaml index 0c328fca0..abdba2396 100644 --- a/services/maintenance/soteria-certificate.yaml +++ b/services/maintenance/networking/soteria-certificate.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-certificate.yaml +# services/maintenance/networking/soteria-certificate.yaml apiVersion: cert-manager.io/v1 kind: Certificate metadata: diff --git a/services/maintenance/soteria-ingress.yaml b/services/maintenance/networking/soteria-ingress.yaml similarity index 92% rename from services/maintenance/soteria-ingress.yaml rename to services/maintenance/networking/soteria-ingress.yaml index ab4b52b9a..63d3d27e7 100644 --- a/services/maintenance/soteria-ingress.yaml +++ b/services/maintenance/networking/soteria-ingress.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-ingress.yaml +# services/maintenance/networking/soteria-ingress.yaml apiVersion: networking.k8s.io/v1 kind: Ingress metadata: diff --git a/services/maintenance/soteria-networkpolicy.yaml b/services/maintenance/networking/soteria-networkpolicy.yaml similarity index 89% rename from services/maintenance/soteria-networkpolicy.yaml rename to services/maintenance/networking/soteria-networkpolicy.yaml index fe6ff33c1..ae8e9bd93 100644 --- a/services/maintenance/soteria-networkpolicy.yaml +++ b/services/maintenance/networking/soteria-networkpolicy.yaml @@ -1,4 +1,4 @@ -# services/maintenance/soteria-networkpolicy.yaml +# services/maintenance/networking/soteria-networkpolicy.yaml apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: diff --git a/services/maintenance/disable-k3s-traefik-daemonset.yaml b/services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml similarity index 95% rename from services/maintenance/disable-k3s-traefik-daemonset.yaml rename to services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml index 71f0ece74..c9af0f309 100644 --- a/services/maintenance/disable-k3s-traefik-daemonset.yaml +++ b/services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/disable-k3s-traefik-daemonset.yaml +# services/maintenance/node-ops/disable-k3s-traefik-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/disable-k3s-traefik-rbac.yaml b/services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml similarity index 94% rename from services/maintenance/disable-k3s-traefik-rbac.yaml rename to services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml index 3bf0113ad..e0523e098 100644 --- a/services/maintenance/disable-k3s-traefik-rbac.yaml +++ b/services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml @@ -1,4 +1,4 @@ -# services/maintenance/disable-k3s-traefik-rbac.yaml +# services/maintenance/node-ops/disable-k3s-traefik-rbac.yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: diff --git a/services/maintenance/disable-k3s-traefik-serviceaccount.yaml b/services/maintenance/node-ops/disable-k3s-traefik-serviceaccount.yaml similarity index 57% rename from services/maintenance/disable-k3s-traefik-serviceaccount.yaml rename to services/maintenance/node-ops/disable-k3s-traefik-serviceaccount.yaml index 37bf6dcbe..f439f4660 100644 --- a/services/maintenance/disable-k3s-traefik-serviceaccount.yaml +++ b/services/maintenance/node-ops/disable-k3s-traefik-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/disable-k3s-traefik-serviceaccount.yaml +# services/maintenance/node-ops/disable-k3s-traefik-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/image-sweeper-cronjob.yaml b/services/maintenance/node-ops/image-sweeper-cronjob.yaml similarity index 96% rename from services/maintenance/image-sweeper-cronjob.yaml rename to services/maintenance/node-ops/image-sweeper-cronjob.yaml index 003920604..5a640276a 100644 --- a/services/maintenance/image-sweeper-cronjob.yaml +++ b/services/maintenance/node-ops/image-sweeper-cronjob.yaml @@ -1,4 +1,4 @@ -# services/maintenance/image-sweeper-cronjob.yaml +# services/maintenance/node-ops/image-sweeper-cronjob.yaml apiVersion: batch/v1 kind: CronJob metadata: diff --git a/services/maintenance/k3s-agent-restart-daemonset.yaml b/services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml similarity index 95% rename from services/maintenance/k3s-agent-restart-daemonset.yaml rename to services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml index e91e34891..60a282983 100644 --- a/services/maintenance/k3s-agent-restart-daemonset.yaml +++ b/services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/k3s-agent-restart-daemonset.yaml +# services/maintenance/node-ops/k3s-agent-restart-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/k3s-traefik-cleanup-rbac.yaml b/services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml similarity index 95% rename from services/maintenance/k3s-traefik-cleanup-rbac.yaml rename to services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml index 9adef8ba3..3628fd07a 100644 --- a/services/maintenance/k3s-traefik-cleanup-rbac.yaml +++ b/services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml @@ -1,4 +1,4 @@ -# services/maintenance/k3s-traefik-cleanup-rbac.yaml +# services/maintenance/node-ops/k3s-traefik-cleanup-rbac.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/node-ops/kustomization.yaml b/services/maintenance/node-ops/kustomization.yaml new file mode 100644 index 000000000..0ac0581f6 --- /dev/null +++ b/services/maintenance/node-ops/kustomization.yaml @@ -0,0 +1,60 @@ +# services/maintenance/node-ops/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: maintenance +resources: + - disable-k3s-traefik-serviceaccount.yaml + - disable-k3s-traefik-rbac.yaml + - k3s-traefik-cleanup-rbac.yaml + - node-nofile-serviceaccount.yaml + - titan-24-docker-serviceaccount.yaml + - pod-cleaner-rbac.yaml + - disable-k3s-traefik-daemonset.yaml + - node-nofile-daemonset.yaml + - rpi-resource-reservation-daemonset.yaml + - metis-sentinel-amd64-daemonset.yaml + - metis-sentinel-arm64-daemonset.yaml + - k3s-agent-restart-daemonset.yaml + - titan-24-docker-daemonset.yaml + - titan-22-link-keeper-daemonset.yaml + - node-image-sweeper-serviceaccount.yaml + - node-image-sweeper-daemonset.yaml + - image-sweeper-cronjob.yaml + - pod-cleaner-cronjob.yaml +configMapGenerator: + - name: disable-k3s-traefik-script + namespace: maintenance + files: + - disable_k3s_traefik.sh=scripts/disable_k3s_traefik.sh + options: + disableNameSuffixHash: true + - name: k3s-traefik-cleanup-script + namespace: maintenance + files: + - k3s_traefik_cleanup.sh=scripts/k3s_traefik_cleanup.sh + options: + disableNameSuffixHash: true + - name: node-nofile-script + namespace: maintenance + files: + - node_nofile.sh=scripts/node_nofile.sh + options: + disableNameSuffixHash: true + - name: node-image-sweeper-script + namespace: maintenance + files: + - node_image_sweeper.sh=scripts/node_image_sweeper.sh + options: + disableNameSuffixHash: true + - name: titan-24-docker-script + namespace: maintenance + files: + - titan_24_docker.sh=scripts/titan_24_docker.sh + options: + disableNameSuffixHash: true + - name: rpi-resource-reservation-script + namespace: maintenance + files: + - rpi_resource_reservation.sh=scripts/rpi_resource_reservation.sh + options: + disableNameSuffixHash: true diff --git a/services/maintenance/metis-sentinel-amd64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml similarity index 94% rename from services/maintenance/metis-sentinel-amd64-daemonset.yaml rename to services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml index 49782529c..3a8810097 100644 --- a/services/maintenance/metis-sentinel-amd64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-sentinel-amd64-daemonset.yaml +# services/maintenance/node-ops/metis-sentinel-amd64-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/metis-sentinel-arm64-daemonset.yaml b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml similarity index 94% rename from services/maintenance/metis-sentinel-arm64-daemonset.yaml rename to services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml index 5f772dce5..ccdbaed11 100644 --- a/services/maintenance/metis-sentinel-arm64-daemonset.yaml +++ b/services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/metis-sentinel-arm64-daemonset.yaml +# services/maintenance/node-ops/metis-sentinel-arm64-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/node-image-sweeper-daemonset.yaml b/services/maintenance/node-ops/node-image-sweeper-daemonset.yaml similarity index 96% rename from services/maintenance/node-image-sweeper-daemonset.yaml rename to services/maintenance/node-ops/node-image-sweeper-daemonset.yaml index 9dc65f5b3..41a04d8be 100644 --- a/services/maintenance/node-image-sweeper-daemonset.yaml +++ b/services/maintenance/node-ops/node-image-sweeper-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/node-image-sweeper-daemonset.yaml +# services/maintenance/node-ops/node-image-sweeper-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/node-image-sweeper-serviceaccount.yaml b/services/maintenance/node-ops/node-image-sweeper-serviceaccount.yaml similarity index 57% rename from services/maintenance/node-image-sweeper-serviceaccount.yaml rename to services/maintenance/node-ops/node-image-sweeper-serviceaccount.yaml index 854f041dc..dfb5a0478 100644 --- a/services/maintenance/node-image-sweeper-serviceaccount.yaml +++ b/services/maintenance/node-ops/node-image-sweeper-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/node-image-sweeper-serviceaccount.yaml +# services/maintenance/node-ops/node-image-sweeper-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/node-nofile-daemonset.yaml b/services/maintenance/node-ops/node-nofile-daemonset.yaml similarity index 95% rename from services/maintenance/node-nofile-daemonset.yaml rename to services/maintenance/node-ops/node-nofile-daemonset.yaml index 115de5b87..49d31c9c8 100644 --- a/services/maintenance/node-nofile-daemonset.yaml +++ b/services/maintenance/node-ops/node-nofile-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/node-nofile-daemonset.yaml +# services/maintenance/node-ops/node-nofile-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/node-nofile-serviceaccount.yaml b/services/maintenance/node-ops/node-nofile-serviceaccount.yaml similarity index 58% rename from services/maintenance/node-nofile-serviceaccount.yaml rename to services/maintenance/node-ops/node-nofile-serviceaccount.yaml index 1cc04996a..e1f75fccc 100644 --- a/services/maintenance/node-nofile-serviceaccount.yaml +++ b/services/maintenance/node-ops/node-nofile-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/node-nofile-serviceaccount.yaml +# services/maintenance/node-ops/node-nofile-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/pod-cleaner-cronjob.yaml b/services/maintenance/node-ops/pod-cleaner-cronjob.yaml similarity index 94% rename from services/maintenance/pod-cleaner-cronjob.yaml rename to services/maintenance/node-ops/pod-cleaner-cronjob.yaml index 99d13f672..8f20a1a98 100644 --- a/services/maintenance/pod-cleaner-cronjob.yaml +++ b/services/maintenance/node-ops/pod-cleaner-cronjob.yaml @@ -1,4 +1,4 @@ -# services/maintenance/pod-cleaner-cronjob.yaml +# services/maintenance/node-ops/pod-cleaner-cronjob.yaml apiVersion: batch/v1 kind: CronJob metadata: diff --git a/services/maintenance/pod-cleaner-rbac.yaml b/services/maintenance/node-ops/pod-cleaner-rbac.yaml similarity index 91% rename from services/maintenance/pod-cleaner-rbac.yaml rename to services/maintenance/node-ops/pod-cleaner-rbac.yaml index 26bb035e2..97deeda68 100644 --- a/services/maintenance/pod-cleaner-rbac.yaml +++ b/services/maintenance/node-ops/pod-cleaner-rbac.yaml @@ -1,4 +1,4 @@ -# services/maintenance/pod-cleaner-rbac.yaml +# services/maintenance/node-ops/pod-cleaner-rbac.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/rpi-resource-reservation-daemonset.yaml b/services/maintenance/node-ops/rpi-resource-reservation-daemonset.yaml similarity index 96% rename from services/maintenance/rpi-resource-reservation-daemonset.yaml rename to services/maintenance/node-ops/rpi-resource-reservation-daemonset.yaml index 574bd6164..d8065f0db 100644 --- a/services/maintenance/rpi-resource-reservation-daemonset.yaml +++ b/services/maintenance/node-ops/rpi-resource-reservation-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/rpi-resource-reservation-daemonset.yaml +# services/maintenance/node-ops/rpi-resource-reservation-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/scripts/disable_k3s_traefik.sh b/services/maintenance/node-ops/scripts/disable_k3s_traefik.sh similarity index 100% rename from services/maintenance/scripts/disable_k3s_traefik.sh rename to services/maintenance/node-ops/scripts/disable_k3s_traefik.sh diff --git a/services/maintenance/scripts/k3s_traefik_cleanup.sh b/services/maintenance/node-ops/scripts/k3s_traefik_cleanup.sh similarity index 100% rename from services/maintenance/scripts/k3s_traefik_cleanup.sh rename to services/maintenance/node-ops/scripts/k3s_traefik_cleanup.sh diff --git a/services/maintenance/scripts/node_image_sweeper.sh b/services/maintenance/node-ops/scripts/node_image_sweeper.sh similarity index 100% rename from services/maintenance/scripts/node_image_sweeper.sh rename to services/maintenance/node-ops/scripts/node_image_sweeper.sh diff --git a/services/maintenance/scripts/node_nofile.sh b/services/maintenance/node-ops/scripts/node_nofile.sh similarity index 100% rename from services/maintenance/scripts/node_nofile.sh rename to services/maintenance/node-ops/scripts/node_nofile.sh diff --git a/services/maintenance/scripts/pod_cleaner.sh b/services/maintenance/node-ops/scripts/pod_cleaner.sh similarity index 100% rename from services/maintenance/scripts/pod_cleaner.sh rename to services/maintenance/node-ops/scripts/pod_cleaner.sh diff --git a/services/maintenance/scripts/rpi_resource_reservation.sh b/services/maintenance/node-ops/scripts/rpi_resource_reservation.sh similarity index 100% rename from services/maintenance/scripts/rpi_resource_reservation.sh rename to services/maintenance/node-ops/scripts/rpi_resource_reservation.sh diff --git a/services/maintenance/scripts/titan_24_docker.sh b/services/maintenance/node-ops/scripts/titan_24_docker.sh similarity index 100% rename from services/maintenance/scripts/titan_24_docker.sh rename to services/maintenance/node-ops/scripts/titan_24_docker.sh diff --git a/services/maintenance/titan-22-link-keeper-daemonset.yaml b/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml similarity index 97% rename from services/maintenance/titan-22-link-keeper-daemonset.yaml rename to services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml index a0b88c957..b4dcb6b9a 100644 --- a/services/maintenance/titan-22-link-keeper-daemonset.yaml +++ b/services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/titan-22-link-keeper-daemonset.yaml +# services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/titan-24-docker-daemonset.yaml b/services/maintenance/node-ops/titan-24-docker-daemonset.yaml similarity index 95% rename from services/maintenance/titan-24-docker-daemonset.yaml rename to services/maintenance/node-ops/titan-24-docker-daemonset.yaml index 51d381398..aef2e1f21 100644 --- a/services/maintenance/titan-24-docker-daemonset.yaml +++ b/services/maintenance/node-ops/titan-24-docker-daemonset.yaml @@ -1,4 +1,4 @@ -# services/maintenance/titan-24-docker-daemonset.yaml +# services/maintenance/node-ops/titan-24-docker-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: diff --git a/services/maintenance/titan-24-docker-serviceaccount.yaml b/services/maintenance/node-ops/titan-24-docker-serviceaccount.yaml similarity index 58% rename from services/maintenance/titan-24-docker-serviceaccount.yaml rename to services/maintenance/node-ops/titan-24-docker-serviceaccount.yaml index 4be8b5eff..a6ba2a598 100644 --- a/services/maintenance/titan-24-docker-serviceaccount.yaml +++ b/services/maintenance/node-ops/titan-24-docker-serviceaccount.yaml @@ -1,4 +1,4 @@ -# services/maintenance/titan-24-docker-serviceaccount.yaml +# services/maintenance/node-ops/titan-24-docker-serviceaccount.yaml apiVersion: v1 kind: ServiceAccount metadata: diff --git a/services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml b/services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml similarity index 96% rename from services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml rename to services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml index 1649ce32b..70502a4d9 100644 --- a/services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml +++ b/services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml @@ -1,4 +1,4 @@ -# services/maintenance/oneoffs/k3s-traefik-cleanup-job.yaml +# services/maintenance/repair-jobs/k3s-traefik-cleanup-job.yaml # One-off job for maintenance/k3s-traefik-cleanup-2. # Purpose: k3s traefik cleanup 2 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/maintenance/repair-jobs/kustomization.yaml b/services/maintenance/repair-jobs/kustomization.yaml new file mode 100644 index 000000000..f005ad155 --- /dev/null +++ b/services/maintenance/repair-jobs/kustomization.yaml @@ -0,0 +1,7 @@ +# services/maintenance/repair-jobs/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: maintenance +resources: + - k3s-traefik-cleanup-job.yaml + - titan-24-rootfs-sweep-job.yaml diff --git a/services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml b/services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml similarity index 96% rename from services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml rename to services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml index 038f1fce4..8b9f127d2 100644 --- a/services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml +++ b/services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml @@ -1,4 +1,4 @@ -# services/maintenance/oneoffs/titan-24-rootfs-sweep-job.yaml +# services/maintenance/repair-jobs/titan-24-rootfs-sweep-job.yaml # One-off emergency cleanup for titan-24 rootfs pressure. # Safe to delete the finished Job/pod after it succeeds. apiVersion: batch/v1 diff --git a/services/monitoring/oneoffs/grafana-org-bootstrap.yaml b/services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml similarity index 98% rename from services/monitoring/oneoffs/grafana-org-bootstrap.yaml rename to services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml index b420ce94a..48f65afc4 100644 --- a/services/monitoring/oneoffs/grafana-org-bootstrap.yaml +++ b/services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml @@ -1,4 +1,4 @@ -# services/monitoring/oneoffs/grafana-org-bootstrap.yaml +# services/monitoring/bootstrap-jobs/grafana-org-bootstrap.yaml # One-off job for monitoring/grafana-org-bootstrap-3. # Purpose: grafana org bootstrap 3 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/monitoring/dashboards/atlas-overview.json b/services/monitoring/dashboards/atlas-overview.json index 9728f5767..7b27ab883 100644 --- a/services/monitoring/dashboards/atlas-overview.json +++ b/services/monitoring/dashboards/atlas-overview.json @@ -216,7 +216,7 @@ { "id": 27, "type": "stat", - "title": "Atlas Availability (365d)", + "title": "Atlas Serving Availability", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -229,7 +229,7 @@ }, "targets": [ { - "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\"}[24h])", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", "refId": "A", "instant": true } @@ -286,7 +286,7 @@ }, "textMode": "value" }, - "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data." + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." }, { "id": 4, @@ -2819,7 +2819,7 @@ }, "targets": [ { - "expr": "avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|performance|regression|reliability|security|smoke|system|ui\"})", + "expr": "(avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|performance|regression|reliability|security|smoke|system|ui\"})) or label_set(vector(0), \"category\", \"none\")", "refId": "A", "legendFormat": "{{category}}", "format": "time_series", diff --git a/services/monitoring/dashboards/atlas-testing.json b/services/monitoring/dashboards/atlas-testing.json index c6c9e26aa..d4d6638a2 100644 --- a/services/monitoring/dashboards/atlas-testing.json +++ b/services/monitoring/dashboards/atlas-testing.json @@ -1801,7 +1801,7 @@ }, "gridPos": { "h": 7, - "w": 8, + "w": 6, "x": 0, "y": 36 }, @@ -1881,8 +1881,8 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 8, + "w": 6, + "x": 6, "y": 36 }, "targets": [ @@ -1953,6 +1953,86 @@ { "id": 136, "type": "state-timeline", + "title": "Semgrep Failure Rate", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 36 + }, + "targets": [ + { + "expr": "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", + "refId": "A", + "legendFormat": "{{suite}}", + "format": "time_series", + "instant": false, + "range": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "mergeValues": true, + "showValue": "never", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "timeFrom": "7d" + }, + { + "id": 137, + "type": "state-timeline", "title": "Supply Chain Failure Rate", "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", "datasource": { @@ -1961,8 +2041,8 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 16, + "w": 6, + "x": 18, "y": 36 }, "targets": [ @@ -2046,7 +2126,7 @@ "collapsed": true, "panels": [ { - "id": 138, + "id": 160, "type": "state-timeline", "title": "Tests Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2126,7 +2206,7 @@ "timeFrom": "7d" }, { - "id": 139, + "id": 161, "type": "state-timeline", "title": "Coverage Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2206,7 +2286,7 @@ "timeFrom": "7d" }, { - "id": 140, + "id": 162, "type": "state-timeline", "title": "LOC Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2286,7 +2366,7 @@ "timeFrom": "7d" }, { - "id": 141, + "id": 163, "type": "state-timeline", "title": "Style Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2366,7 +2446,7 @@ "timeFrom": "7d" }, { - "id": 142, + "id": 164, "type": "state-timeline", "title": "Gate Glue Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2376,7 +2456,7 @@ }, "gridPos": { "h": 7, - "w": 8, + "w": 6, "x": 0, "y": 50 }, @@ -2446,7 +2526,7 @@ "timeFrom": "7d" }, { - "id": 143, + "id": 165, "type": "state-timeline", "title": "SonarQube Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2456,8 +2536,8 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 8, + "w": 6, + "x": 6, "y": 50 }, "targets": [ @@ -2526,7 +2606,87 @@ "timeFrom": "7d" }, { - "id": 144, + "id": 166, + "type": "state-timeline", + "title": "Semgrep Healthy Rate", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 50 + }, + "targets": [ + { + "expr": "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", + "refId": "A", + "legendFormat": "{{suite}}", + "format": "time_series", + "instant": false, + "range": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-orange", + "value": 90 + }, + { + "color": "dark-yellow", + "value": 93 + }, + { + "color": "dark-green", + "value": 95 + }, + { + "color": "dark-blue", + "value": 100 + } + ] + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "mergeValues": true, + "showValue": "never", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "timeFrom": "7d" + }, + { + "id": 167, "type": "state-timeline", "title": "Supply Chain Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2536,8 +2696,8 @@ }, "gridPos": { "h": 7, - "w": 8, - "x": 16, + "w": 6, + "x": 18, "y": 50 }, "targets": [ @@ -4643,6 +4803,452 @@ } ], "description": "SonarQube availability, projects, fetch errors, and gate status." + }, + { + "id": 506, + "type": "row", + "title": "Public ZAP Baseline", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "collapsed": true, + "panels": [ + { + "id": 168, + "type": "stat", + "title": "ZAP Targets Scanned", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 117 + }, + "targets": [ + { + "expr": "(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job=\"platform-security-zap\"})) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Number of public hostnames with a latest OWASP ZAP baseline scan sample." + }, + { + "id": 169, + "type": "stat", + "title": "ZAP High/Medium Alerts", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 117 + }, + "targets": [ + { + "expr": "(sum(platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium\"}) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Current high and medium ZAP alert instance count across public targets." + }, + { + "id": 170, + "type": "stat", + "title": "ZAP Scan Errors", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 117 + }, + "targets": [ + { + "expr": "(sum(platform_zap_baseline_scan_status{exported_job=\"platform-security-zap\",status=~\"fail|error\"}) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Targets whose latest ZAP baseline failed or did not produce a usable report." + }, + { + "id": 171, + "type": "piechart", + "title": "ZAP Alerts by Risk", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 117 + }, + "targets": [ + { + "expr": "sum by (risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk!=\"\"}) or on() vector(0)", + "refId": "A", + "legendFormat": "{{risk}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "pie", + "displayLabels": [], + "tooltip": { + "mode": "single" + }, + "colorScheme": "interpolateSpectral", + "colorBy": "value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "description": "Current ZAP alert distribution by risk level." + }, + { + "id": 172, + "type": "state-timeline", + "title": "ZAP Target Health", + "description": "Latest OWASP ZAP passive baseline health per public hostname. High alerts and scan errors drop a target to zero; medium/low alerts mark warning health.", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 117 + }, + "targets": [ + { + "expr": "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) or on() vector(0))", + "refId": "A", + "legendFormat": "{{host}}", + "format": "time_series", + "instant": false, + "range": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-orange", + "value": 90 + }, + { + "color": "dark-yellow", + "value": 93 + }, + { + "color": "dark-green", + "value": 95 + }, + { + "color": "dark-blue", + "value": 100 + } + ] + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "mergeValues": true, + "showValue": "never", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "timeFrom": "7d" + }, + { + "id": 173, + "type": "bargauge", + "title": "ZAP Alerts by Host", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 123 + }, + "targets": [ + { + "expr": "sort_desc(sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium|low\"}) or on() vector(0))", + "refId": "A", + "legendFormat": "{{host}} \u00b7 {{risk}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "none", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Current high, medium, and low ZAP alert counts by public hostname." + } + ], + "description": "OWASP ZAP passive baseline results for externally addressed Atlas domains." } ], "time": { diff --git a/services/monitoring/grafana-dashboard-overview.yaml b/services/monitoring/grafana-dashboard-overview.yaml index 55bcaef75..3f6ef5640 100644 --- a/services/monitoring/grafana-dashboard-overview.yaml +++ b/services/monitoring/grafana-dashboard-overview.yaml @@ -225,7 +225,7 @@ data: { "id": 27, "type": "stat", - "title": "Atlas Availability (365d)", + "title": "Atlas Serving Availability", "datasource": { "type": "prometheus", "uid": "atlas-vm" @@ -238,7 +238,7 @@ data: }, "targets": [ { - "expr": "last_over_time(atlas:availability:ratio_365d{scope=\"atlas\"}[24h])", + "expr": "(last_over_time(atlas:availability:ratio_365d{scope=\"atlas\",definition=\"serving-v2\"}[24h]) or on() avg_over_time((min(((sum(kube_node_status_condition{condition=\"Ready\",status=\"true\",node=~\"titan-0a|titan-0b|titan-0c\"}) >= bool 2)), ((sum(kube_deployment_status_replicas_available{namespace=~\"traefik|kube-system\",deployment=\"traefik\"}) > bool 0))))[1h:5m]))", "refId": "A", "instant": true } @@ -295,7 +295,7 @@ data: }, "textMode": "value" }, - "description": "Rolling 365-day availability from vmalert's precomputed atlas:availability:ratio_365d series. Grafana keeps the last successful rollup for up to 24h so one missed long-window evaluation does not render as No data." + "description": "Availability over observed samples, up to 365 days: at least two control-plane nodes Ready and at least one Traefik replica serving. Historical partial-capacity samples remain available when ingress kept serving; monitoring gaps are excluded. Grafana keeps the last successful rollup for up to 24h and falls back to the live binary serving state if no rollup is available." }, { "id": 4, @@ -2828,7 +2828,7 @@ data: }, "targets": [ { - "expr": "avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|performance|regression|reliability|security|smoke|system|ui\"})", + "expr": "(avg by (category) (platform_quality:test_category_health_rate:percent_1h{suite=~\"ariadne|metis|ananke|atlasbot|pegasus|soteria|titan_iac|bstein_home|data_prepper|lesavka\",branch!=\"\",branch=~\"main|master|origin/main|origin/master\",category=~\"api|chaos|compatibility|component|contract|e2e|integration|performance|regression|reliability|security|smoke|system|ui\"})) or label_set(vector(0), \"category\", \"none\")", "refId": "A", "legendFormat": "{{category}}", "format": "time_series", diff --git a/services/monitoring/grafana-dashboard-testing.yaml b/services/monitoring/grafana-dashboard-testing.yaml index 961d34b0e..5eef4e1d8 100644 --- a/services/monitoring/grafana-dashboard-testing.yaml +++ b/services/monitoring/grafana-dashboard-testing.yaml @@ -1810,7 +1810,7 @@ data: }, "gridPos": { "h": 7, - "w": 8, + "w": 6, "x": 0, "y": 36 }, @@ -1890,8 +1890,8 @@ data: }, "gridPos": { "h": 7, - "w": 8, - "x": 8, + "w": 6, + "x": 6, "y": 36 }, "targets": [ @@ -1962,6 +1962,86 @@ data: { "id": 136, "type": "state-timeline", + "title": "Semgrep Failure Rate", + "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 36 + }, + "targets": [ + { + "expr": "(((100 * (sum by (suite) (platform_quality:check_failed_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", + "refId": "A", + "legendFormat": "{{suite}}", + "format": "time_series", + "instant": false, + "range": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "mergeValues": true, + "showValue": "never", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "timeFrom": "7d" + }, + { + "id": 137, + "type": "state-timeline", "title": "Supply Chain Failure Rate", "description": "Latest bad-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are failing in the freshness window; this is not an event-count spike chart.", "datasource": { @@ -1970,8 +2050,8 @@ data: }, "gridPos": { "h": 7, - "w": 8, - "x": 16, + "w": 6, + "x": 18, "y": 36 }, "targets": [ @@ -2055,7 +2135,7 @@ data: "collapsed": true, "panels": [ { - "id": 138, + "id": 160, "type": "state-timeline", "title": "Tests Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2135,7 +2215,7 @@ data: "timeFrom": "7d" }, { - "id": 139, + "id": 161, "type": "state-timeline", "title": "Coverage Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2215,7 +2295,7 @@ data: "timeFrom": "7d" }, { - "id": 140, + "id": 162, "type": "state-timeline", "title": "LOC Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2295,7 +2375,7 @@ data: "timeFrom": "7d" }, { - "id": 141, + "id": 163, "type": "state-timeline", "title": "Style Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2375,7 +2455,7 @@ data: "timeFrom": "7d" }, { - "id": 142, + "id": 164, "type": "state-timeline", "title": "Gate Glue Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2385,7 +2465,7 @@ data: }, "gridPos": { "h": 7, - "w": 8, + "w": 6, "x": 0, "y": 50 }, @@ -2455,7 +2535,7 @@ data: "timeFrom": "7d" }, { - "id": 143, + "id": 165, "type": "state-timeline", "title": "SonarQube Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2465,8 +2545,8 @@ data: }, "gridPos": { "h": 7, - "w": 8, - "x": 8, + "w": 6, + "x": 6, "y": 50 }, "targets": [ @@ -2535,7 +2615,87 @@ data: "timeFrom": "7d" }, { - "id": 144, + "id": 166, + "type": "state-timeline", + "title": "Semgrep Healthy Rate", + "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 50 + }, + "targets": [ + { + "expr": "(((100 * (sum by (suite) (platform_quality:check_healthy_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})) / clamp_min((sum by (suite) (platform_quality:check_seen_flag:present_1h{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",check=~\"semgrep|sast\"})), 1))) or on(suite) ((0 * ((count by (suite) (platform_quality_gate_build_info{suite=~\"${suite:regex}\",branch!=\"\",branch=~\"${branch:regex}\",exported_job=\"platform-quality-ci\"}) >= bool 0)))))", + "refId": "A", + "legendFormat": "{{suite}}", + "format": "time_series", + "instant": false, + "range": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-orange", + "value": 90 + }, + { + "color": "dark-yellow", + "value": 93 + }, + { + "color": "dark-green", + "value": 95 + }, + { + "color": "dark-blue", + "value": 100 + } + ] + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "mergeValues": true, + "showValue": "never", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "timeFrom": "7d" + }, + { + "id": 167, "type": "state-timeline", "title": "Supply Chain Healthy Rate", "description": "Latest acceptable-state percentage for this check family, evaluated over time. Higher means more selected suites/checks are healthy in the freshness window; gaps mean there was no check evidence.", @@ -2545,8 +2705,8 @@ data: }, "gridPos": { "h": 7, - "w": 8, - "x": 16, + "w": 6, + "x": 18, "y": 50 }, "targets": [ @@ -4652,6 +4812,452 @@ data: } ], "description": "SonarQube availability, projects, fetch errors, and gate status." + }, + { + "id": 506, + "type": "row", + "title": "Public ZAP Baseline", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "collapsed": true, + "panels": [ + { + "id": 168, + "type": "stat", + "title": "ZAP Targets Scanned", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 117 + }, + "targets": [ + { + "expr": "(count(max by (host) (platform_zap_baseline_last_run_timestamp_seconds{exported_job=\"platform-security-zap\"})) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Number of public hostnames with a latest OWASP ZAP baseline scan sample." + }, + { + "id": 169, + "type": "stat", + "title": "ZAP High/Medium Alerts", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 117 + }, + "targets": [ + { + "expr": "(sum(platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium\"}) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Current high and medium ZAP alert instance count across public targets." + }, + { + "id": 170, + "type": "stat", + "title": "ZAP Scan Errors", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 117 + }, + "targets": [ + { + "expr": "(sum(platform_zap_baseline_scan_status{exported_job=\"platform-security-zap\",status=~\"fail|error\"}) or on() vector(0))", + "refId": "A", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "unit": "none", + "custom": { + "displayMode": "auto" + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "description": "Targets whose latest ZAP baseline failed or did not produce a usable report." + }, + { + "id": 171, + "type": "piechart", + "title": "ZAP Alerts by Risk", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 117 + }, + "targets": [ + { + "expr": "sum by (risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk!=\"\"}) or on() vector(0)", + "refId": "A", + "legendFormat": "{{risk}}" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "color": { + "mode": "palette-classic" + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "right" + }, + "pieType": "pie", + "displayLabels": [], + "tooltip": { + "mode": "single" + }, + "colorScheme": "interpolateSpectral", + "colorBy": "value", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "description": "Current ZAP alert distribution by risk level." + }, + { + "id": 172, + "type": "state-timeline", + "title": "ZAP Target Health", + "description": "Latest OWASP ZAP passive baseline health per public hostname. High alerts and scan errors drop a target to zero; medium/low alerts mark warning health.", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 117 + }, + "targets": [ + { + "expr": "(max by (host) (platform_zap_baseline_target_health_percent{exported_job=\"platform-security-zap\"}) or on() vector(0))", + "refId": "A", + "legendFormat": "{{host}}", + "format": "time_series", + "instant": false, + "range": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-orange", + "value": 90 + }, + { + "color": "dark-yellow", + "value": 93 + }, + { + "color": "dark-green", + "value": 95 + }, + { + "color": "dark-blue", + "value": 100 + } + ] + }, + "custom": { + "fillOpacity": 70, + "lineWidth": 0, + "spanNulls": true + }, + "min": 0, + "max": 100 + }, + "overrides": [] + }, + "options": { + "mergeValues": true, + "showValue": "never", + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "timeFrom": "7d" + }, + { + "id": 173, + "type": "bargauge", + "title": "ZAP Alerts by Host", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 123 + }, + "targets": [ + { + "expr": "sort_desc(sum by (host, risk) (platform_zap_baseline_alerts_total{exported_job=\"platform-security-zap\",risk=~\"high|medium|low\"}) or on() vector(0))", + "refId": "A", + "legendFormat": "{{host}} \u00b7 {{risk}}", + "instant": true + } + ], + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "unit": "none", + "min": 0, + "max": null, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "dark-green", + "value": 0.01 + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-orange", + "value": 3 + }, + { + "color": "dark-red", + "value": 5 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "displayMode": "basic", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + } + }, + "transformations": [ + { + "id": "sortBy", + "options": { + "fields": [ + "Value" + ], + "order": "desc" + } + }, + { + "id": "limit", + "options": { + "limit": 20 + } + } + ], + "description": "Current high, medium, and low ZAP alert counts by public hostname." + } + ], + "description": "OWASP ZAP passive baseline results for externally addressed Atlas domains." } ], "time": { diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index 15c86edaa..418db0e44 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -85,17 +85,20 @@ spec: extraArgs: retentionPeriod: "1y" # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11} promscrape.configCheckInterval: "30s" + search.maxConcurrentRequests: "4" + search.maxQueryDuration: "1m" + search.maxQueueDuration: "30s" persistentVolume: enabled: true size: 100Gi resources: requests: - cpu: 250m - memory: 1Gi - limits: - cpu: "1" + cpu: 500m memory: 2Gi + limits: + cpu: "2" + memory: 4Gi affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: @@ -108,10 +111,34 @@ spec: - key: kubernetes.io/hostname operator: NotIn values: + # Longhorn engine-image probes repeatedly fail on these + # attachment hosts and can surface volume I/O errors. + - titan-14 + - titan-18 - titan-20 - titan-21 - titan-22 - titan-24 + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi4 + - weight: 25 + preference: + matchExpressions: + - key: node-role.kubernetes.io/storage-backbone + operator: Exists # Enable built-in Kubernetes scraping scrape: diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index 16df6f3f4..8e2f9cd2c 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -31,8 +31,8 @@ resources: - grafana-alerting-config.yaml - grafana-folders.yaml - helmrelease.yaml - - oneoffs/grafana-org-bootstrap.yaml - - oneoffs/grafana-user-dedupe-job.yaml + - bootstrap-jobs/grafana-org-bootstrap.yaml + - repair-jobs/grafana-user-dedupe-job.yaml configMapGenerator: - name: postmark-exporter-script diff --git a/services/monitoring/oneoffs/grafana-user-dedupe-job.yaml b/services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml similarity index 98% rename from services/monitoring/oneoffs/grafana-user-dedupe-job.yaml rename to services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml index 6bcd69adb..bd974323f 100644 --- a/services/monitoring/oneoffs/grafana-user-dedupe-job.yaml +++ b/services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml @@ -1,4 +1,4 @@ -# services/monitoring/oneoffs/grafana-user-dedupe-job.yaml +# services/monitoring/repair-jobs/grafana-user-dedupe-job.yaml # One-off job for monitoring/grafana-user-dedupe-api-v8. # Purpose: grafana user dedupe api v8 (see container args/env in this file). # Run by setting spec.suspend to false, reconcile, then set it back to true. diff --git a/services/monitoring/vmalert-atlas-availability.yaml b/services/monitoring/vmalert-atlas-availability.yaml index a1138d753..1bfcc1e49 100644 --- a/services/monitoring/vmalert-atlas-availability.yaml +++ b/services/monitoring/vmalert-atlas-availability.yaml @@ -17,132 +17,34 @@ data: min( ( sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 + >= bool 2 ), ( sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) + > bool 0 ) ) )[1h:5m]) labels: + definition: serving-v2 scope: atlas rollup: hourly - record: atlas:availability:ratio_365d expr: | - clamp_max(( - ( - sum(sum_over_time(( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(0) + share_gt_over_time(( + max by (scope, rollup) ( + atlas:availability:ratio_1h{scope="atlas",definition="serving-v2"} + or atlas:availability:ratio_1h{scope="atlas",definition=""} ) - + - clamp_min( - 8761 - - - ( - clamp_min( - floor( - ( - time() - - - ( - min(min_over_time(timestamp( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(time() + 3600) - ) - ) - / 3600 - ) - + 1, - 0 - ) - ), - 0 - ) - ) - / - clamp_min( - ( - ( - sum(count_over_time(( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(0) - ) - + - clamp_min( - 8761 - - - ( - clamp_min( - floor( - ( - time() - - - ( - min(min_over_time(timestamp( - min( - ( - sum(kube_node_status_condition{condition="Ready",status="true",node=~"titan-0a|titan-0b|titan-0c"}) - / 3 - ), - ( - sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"}) - / clamp_min(sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1) - ) - ) - )[365d:1h])) - or on() vector(time() + 3600) - ) - ) - / 3600 - ) - + 1, - 0 - ) - ), - 0 - ) - ), - 1 - ), 1) + )[365d:15m], 0) labels: + definition: serving-v2 scope: atlas rollup: yearly platform-quality.yaml: | groups: - name: platform.quality - interval: 1m + interval: 5m rules: - record: platform_quality:test_case_status:count_1h expr: | @@ -419,6 +321,7 @@ spec: - -datasource.queryStep=1h - -remoteWrite.url=http://victoria-metrics-single-server:8428 - -rule=/etc/vmalert/rules/*.yaml + - -configCheckInterval=30s - -evaluationInterval=15m - -httpListenAddr=:8880 ports: diff --git a/services/pegasus/kustomization.yaml b/services/pegasus/kustomization.yaml index 05c3baa59..8e7b793ba 100644 --- a/services/pegasus/kustomization.yaml +++ b/services/pegasus/kustomization.yaml @@ -1,6 +1,7 @@ # services/pegasus/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: jellyfin resources: - configmap.yaml - image.yaml diff --git a/services/quality/kustomization.yaml b/services/quality/kustomization.yaml index 61ebb2a1f..8e055669f 100644 --- a/services/quality/kustomization.yaml +++ b/services/quality/kustomization.yaml @@ -1,6 +1,7 @@ # services/quality/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: quality resources: - namespace.yaml - sonarqube-serviceaccount.yaml @@ -12,5 +13,7 @@ resources: - sonarqube-exporter-service.yaml - sonarqube-exporter-deployment.yaml - oauth2-proxy-sonarqube.yaml + - zap-baseline-configmap.yaml + - zap-baseline-cronjob.yaml - sonarqube-certificate.yaml - sonarqube-ingress.yaml diff --git a/services/quality/zap-baseline-configmap.yaml b/services/quality/zap-baseline-configmap.yaml new file mode 100644 index 000000000..a22747448 --- /dev/null +++ b/services/quality/zap-baseline-configmap.yaml @@ -0,0 +1,180 @@ +# services/quality/zap-baseline-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: zap-baseline-config + namespace: quality +data: + targets.txt: | + https://bstein.dev + https://chat.ai.bstein.dev + https://sso.bstein.dev + https://auth.bstein.dev + https://scm.bstein.dev + https://ci.bstein.dev + https://registry.bstein.dev + https://quality.bstein.dev + https://secret.bstein.dev + https://vault.bstein.dev + https://logs.bstein.dev + https://metrics.bstein.dev + https://stream.bstein.dev + https://pegasus.bstein.dev + https://tasks.bstein.dev + https://cloud.bstein.dev + https://notes.bstein.dev + https://budget.bstein.dev + https://money.bstein.dev + https://health.bstein.dev + https://agent.bstein.dev + https://cassandra.bstein.dev + https://veles.bstein.dev + https://matrix.live.bstein.dev + https://live.bstein.dev + https://wolf.bstein.dev + https://mail.bstein.dev + https://recovery.bstein.dev + https://backup.bstein.dev + run_zap_baseline.sh: | + #!/usr/bin/env bash + set -euo pipefail + + target_file="${ZAP_TARGET_FILE:-/zap/config/targets.txt}" + pushgateway="${PUSHGATEWAY_URL:-http://platform-quality-gateway.monitoring.svc.cluster.local:9091}" + job_name="${QUALITY_GATE_JOB_NAME:-platform-security-zap}" + spider_minutes="${ZAP_SPIDER_MINUTES:-1}" + max_scan_minutes="${ZAP_MAX_SCAN_MINUTES:-5}" + work_dir="${ZAP_WORK_DIR:-/zap/wrk}" + metrics_path="${work_dir}/zap-baseline.prom" + reports_dir="${work_dir}/reports" + + mkdir -p "${reports_dir}" + : > "${metrics_path}" + + python3 - <<'PY' > "${metrics_path}" + print("# TYPE platform_zap_baseline_target_up gauge") + print("# TYPE platform_zap_baseline_scan_status gauge") + print("# TYPE platform_zap_baseline_alerts_total gauge") + print("# TYPE platform_zap_baseline_target_health_percent gauge") + print("# TYPE platform_zap_baseline_last_run_timestamp_seconds gauge") + PY + + while IFS= read -r raw_target || [ -n "${raw_target}" ]; do + target="$(printf '%s' "${raw_target}" | sed 's/#.*//' | xargs)" + [ -n "${target}" ] || continue + + host="$(python3 - "${target}" <<'PY' + import sys + from urllib.parse import urlparse + + print(urlparse(sys.argv[1]).hostname or "unknown") + PY + )" + safe_host="$(printf '%s' "${host}" | tr -c 'A-Za-z0-9_.-' '_')" + json_report="${reports_dir}/${safe_host}.json" + html_report="${reports_dir}/${safe_host}.html" + markdown_report="${reports_dir}/${safe_host}.md" + xml_report="${reports_dir}/${safe_host}.xml" + + set +e + zap-baseline.py \ + -t "${target}" \ + -m "${spider_minutes}" \ + -T "${max_scan_minutes}" \ + -I \ + -J "${json_report}" \ + -r "${html_report}" \ + -w "${markdown_report}" \ + -x "${xml_report}" + zap_rc=$? + set -e + + python3 - "${target}" "${host}" "${json_report}" "${zap_rc}" >> "${metrics_path}" <<'PY' + import json + import sys + import time + from pathlib import Path + + target, host, report_path, zap_rc_raw = sys.argv[1:5] + try: + zap_rc = int(zap_rc_raw) + except ValueError: + zap_rc = 3 + + def esc(value: str) -> str: + return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"') + + def labels(extra: dict[str, str]) -> str: + base = {"target": target, "host": host} + base.update(extra) + return "{" + ",".join(f'{key}="{esc(value)}"' for key, value in base.items()) + "}" + + risk_counts = {"high": 0, "medium": 0, "low": 0, "informational": 0, "unknown": 0} + completed = Path(report_path).exists() + if completed: + try: + payload = json.loads(Path(report_path).read_text(encoding="utf-8")) + except json.JSONDecodeError: + completed = False + payload = {} + for site in payload.get("site", []) if isinstance(payload, dict) else []: + if not isinstance(site, dict): + continue + alerts = site.get("alerts", []) + if not isinstance(alerts, list): + continue + for alert in alerts: + if not isinstance(alert, dict): + continue + risk_code = str(alert.get("riskcode") or "").strip() + risk_name = str(alert.get("risk") or alert.get("riskdesc") or "unknown").split()[0].lower() + risk = { + "3": "high", + "2": "medium", + "1": "low", + "0": "informational", + }.get(risk_code, risk_name) + if risk not in risk_counts: + risk = "unknown" + instances = alert.get("instances") + count = len(instances) if isinstance(instances, list) and instances else 1 + risk_counts[risk] += count + + if not completed or zap_rc not in {0, 1, 2}: + status = "error" + elif risk_counts["high"] > 0: + status = "fail" + elif risk_counts["medium"] > 0 or risk_counts["low"] > 0: + status = "warn" + else: + status = "ok" + + health = 0 if status in {"error", "fail"} else 85 if status == "warn" else 100 + print(f"platform_zap_baseline_target_up{labels({})} {1 if completed else 0}") + for candidate in ["ok", "warn", "fail", "error"]: + print(f'platform_zap_baseline_scan_status{labels({"status": candidate})} {1 if status == candidate else 0}') + for risk, count in risk_counts.items(): + print(f'platform_zap_baseline_alerts_total{labels({"risk": risk})} {count}') + print(f"platform_zap_baseline_target_health_percent{labels({})} {health}") + print(f"platform_zap_baseline_last_run_timestamp_seconds{labels({})} {int(time.time())}") + PY + done < "${target_file}" + + python3 - "${pushgateway}" "${job_name}" "${metrics_path}" <<'PY' + import sys + import urllib.request + from pathlib import Path + + pushgateway, job_name, metrics_path = sys.argv[1:4] + payload = Path(metrics_path).read_bytes() + url = f"{pushgateway.rstrip('/')}/metrics/job/{job_name}" + request = urllib.request.Request( + url, + data=payload, + method="PUT", + headers={"Content-Type": "text/plain"}, + ) + with urllib.request.urlopen(request, timeout=15) as response: + if response.status >= 400: + raise SystemExit(f"pushgateway returned HTTP {response.status}") + PY diff --git a/services/quality/zap-baseline-cronjob.yaml b/services/quality/zap-baseline-cronjob.yaml new file mode 100644 index 000000000..2f98cc0ce --- /dev/null +++ b/services/quality/zap-baseline-cronjob.yaml @@ -0,0 +1,77 @@ +# services/quality/zap-baseline-cronjob.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: zap-baseline + namespace: quality +spec: + schedule: "17 4 * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + ttlSecondsAfterFinished: 604800 + backoffLimit: 1 + template: + metadata: + labels: + app: zap-baseline + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5"] + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi4"] + containers: + - name: zap + image: ghcr.io/zaproxy/zaproxy:2.17.0 + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - /zap/config/run_zap_baseline.sh + env: + - name: PUSHGATEWAY_URL + value: http://platform-quality-gateway.monitoring.svc.cluster.local:9091 + - name: QUALITY_GATE_JOB_NAME + value: platform-security-zap + - name: ZAP_TARGET_FILE + value: /zap/config/targets.txt + - name: ZAP_SPIDER_MINUTES + value: "1" + - name: ZAP_MAX_SCAN_MINUTES + value: "5" + resources: + requests: + cpu: 100m + memory: 1Gi + limits: + cpu: "2" + memory: 2Gi + volumeMounts: + - name: config + mountPath: /zap/config + readOnly: true + - name: work + mountPath: /zap/wrk + volumes: + - name: config + configMap: + name: zap-baseline-config + defaultMode: 0755 + - name: work + emptyDir: {} diff --git a/services/sui-metrics/overlays/atlas/kustomization.yaml b/services/sui-metrics/overlays/atlas/kustomization.yaml index 826e5aa65..5aa4c81fa 100644 --- a/services/sui-metrics/overlays/atlas/kustomization.yaml +++ b/services/sui-metrics/overlays/atlas/kustomization.yaml @@ -1,6 +1,7 @@ # services/sui-metrics/overlays/atlas/kustomization.yaml apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +namespace: sui-metrics resources: - ../../base patches: diff --git a/services/veles/ingress.yaml b/services/veles/ingress.yaml index ae2597f79..8a2e569ca 100644 --- a/services/veles/ingress.yaml +++ b/services/veles/ingress.yaml @@ -7,6 +7,7 @@ metadata: annotations: cert-manager.io/cluster-issuer: letsencrypt traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.middlewares: veles-redirect-to-cassandra@kubernetescrd traefik.ingress.kubernetes.io/router.tls: "true" spec: ingressClassName: traefik diff --git a/services/veles/kustomization.yaml b/services/veles/kustomization.yaml index 8aacbd184..f1af10dce 100644 --- a/services/veles/kustomization.yaml +++ b/services/veles/kustomization.yaml @@ -20,6 +20,7 @@ resources: - frontend-deployment.yaml - retention-cronjob.yaml - image.yaml + - redirect-middleware.yaml - ingress.yaml - bootstrap-jobs/veles-secrets-ensure-job.yaml - migration-jobs/cassandra-artifact-export.yaml diff --git a/services/veles/redirect-middleware.yaml b/services/veles/redirect-middleware.yaml new file mode 100644 index 000000000..4e60e9cce --- /dev/null +++ b/services/veles/redirect-middleware.yaml @@ -0,0 +1,11 @@ +# services/veles/redirect-middleware.yaml +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: redirect-to-cassandra + namespace: veles +spec: + redirectRegex: + regex: ^https?://veles\.bstein\.dev/?(.*) + replacement: https://cassandra.bstein.dev/${1} + permanent: true diff --git a/terraform/aether/locals.tf b/terraform/aether/locals.tf new file mode 100644 index 000000000..b3b4ce208 --- /dev/null +++ b/terraform/aether/locals.tf @@ -0,0 +1,22 @@ +data "aws_caller_identity" "current" {} + +locals { + name = "${var.name_prefix}-${var.environment}" + + tags = { + Environment = var.environment + ManagedBy = "terraform" + Plane = "aether" + System = "titan-lab" + } + + paid_component_requests = compact([ + var.enable_external_backup_bucket ? "external-backup-bucket" : "", + ]) + + external_backup_bucket_name = lower(replace( + "${local.name}-${data.aws_caller_identity.current.account_id}-${var.aws_region}-backup", + "_", + "-", + )) +} diff --git a/terraform/aether/main.tf b/terraform/aether/main.tf new file mode 100644 index 000000000..2caa7d27d --- /dev/null +++ b/terraform/aether/main.tf @@ -0,0 +1,79 @@ +resource "terraform_data" "paid_resource_guard" { + count = length(local.paid_component_requests) > 0 ? 1 : 0 + + input = local.paid_component_requests + + lifecycle { + precondition { + condition = var.permit_paid_resources + error_message = "Set permit_paid_resources=true before enabling Aether paid components." + } + } +} + +resource "aws_budgets_budget" "monthly" { + count = var.enable_budget ? 1 : 0 + + name = "${local.name}-monthly" + budget_type = "COST" + limit_amount = tostring(var.monthly_budget_usd) + limit_unit = "USD" + time_unit = "MONTHLY" + + dynamic "notification" { + for_each = var.budget_alert_email == null ? [] : [var.budget_alert_email] + + content { + comparison_operator = "GREATER_THAN" + notification_type = "FORECASTED" + subscriber_email_addresses = [notification.value] + threshold = 80 + threshold_type = "PERCENTAGE" + } + } +} + +resource "aws_s3_bucket" "external_backup" { + count = var.enable_external_backup_bucket ? 1 : 0 + + bucket = local.external_backup_bucket_name + force_destroy = false + + tags = { + Purpose = "external-backup-test" + } + + depends_on = [terraform_data.paid_resource_guard] +} + +resource "aws_s3_bucket_public_access_block" "external_backup" { + count = var.enable_external_backup_bucket ? 1 : 0 + + bucket = aws_s3_bucket.external_backup[0].id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "external_backup" { + count = var.enable_external_backup_bucket ? 1 : 0 + + bucket = aws_s3_bucket.external_backup[0].id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_versioning" "external_backup" { + count = var.enable_external_backup_bucket ? 1 : 0 + + bucket = aws_s3_bucket.external_backup[0].id + + versioning_configuration { + status = "Enabled" + } +} diff --git a/terraform/aether/outputs.tf b/terraform/aether/outputs.tf new file mode 100644 index 000000000..5c9ea46a8 --- /dev/null +++ b/terraform/aether/outputs.tf @@ -0,0 +1,27 @@ +output "account_id" { + value = data.aws_caller_identity.current.account_id + description = "AWS account Terraform is configured to use." +} + +output "enabled_components" { + value = { + budget = var.enable_budget + external_backup_bucket = var.enable_external_backup_bucket + } + description = "Aether components currently switched on." +} + +output "paid_component_requests" { + value = local.paid_component_requests + description = "Aether components that require the paid resource opt-in." +} + +output "external_backup_bucket_name" { + value = try(aws_s3_bucket.external_backup[0].bucket, null) + description = "S3 bucket name when the external backup test bucket is enabled." +} + +output "flux_root" { + value = abspath("${path.module}/../../clusters/aether/flux-system") + description = "Aether Flux render root." +} diff --git a/terraform/aether/providers.tf b/terraform/aether/providers.tf new file mode 100644 index 000000000..7ad1636ab --- /dev/null +++ b/terraform/aether/providers.tf @@ -0,0 +1,9 @@ +provider "aws" { + region = var.aws_region + profile = var.aws_profile + allowed_account_ids = length(var.allowed_account_ids) == 0 ? null : var.allowed_account_ids + + default_tags { + tags = local.tags + } +} diff --git a/terraform/aether/terraform.tfvars.example b/terraform/aether/terraform.tfvars.example new file mode 100644 index 000000000..fc34774e9 --- /dev/null +++ b/terraform/aether/terraform.tfvars.example @@ -0,0 +1,13 @@ +aws_region = "us-east-1" +aws_profile = "aether" +allowed_account_ids = [] + +name_prefix = "aether" +environment = "lab" + +enable_budget = false +monthly_budget_usd = 1 +budget_alert_email = null + +permit_paid_resources = false +enable_external_backup_bucket = false diff --git a/terraform/aether/variables.tf b/terraform/aether/variables.tf new file mode 100644 index 000000000..32f9332d8 --- /dev/null +++ b/terraform/aether/variables.tf @@ -0,0 +1,74 @@ +variable "aws_region" { + type = string + description = "AWS region used for Aether lab resources." + default = "us-east-1" +} + +variable "aws_profile" { + type = string + description = "Optional AWS CLI profile name for the target account." + default = null +} + +variable "allowed_account_ids" { + type = list(string) + description = "Optional allow-list for the AWS account IDs Terraform may touch." + default = [] +} + +variable "name_prefix" { + type = string + description = "Short name used as the prefix for Aether resources." + default = "aether" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{1,30}[a-z0-9]$", var.name_prefix)) + error_message = "name_prefix must be lowercase kebab-case and start/end with an alphanumeric character." + } +} + +variable "environment" { + type = string + description = "Aether environment name." + default = "lab" + + validation { + condition = contains(["lab", "sandbox", "test"], var.environment) + error_message = "environment must be one of: lab, sandbox, test." + } +} + +variable "enable_budget" { + type = bool + description = "Create a small AWS Budget for the account." + default = false +} + +variable "monthly_budget_usd" { + type = number + description = "Monthly budget amount when enable_budget is true." + default = 1 + + validation { + condition = var.monthly_budget_usd >= 1 && var.monthly_budget_usd <= 25 + error_message = "monthly_budget_usd must stay between 1 and 25 for the Aether lab." + } +} + +variable "budget_alert_email" { + type = string + description = "Optional email subscriber for AWS Budget notifications." + default = null +} + +variable "permit_paid_resources" { + type = bool + description = "Set true before enabling any resource that can create AWS spend." + default = false +} + +variable "enable_external_backup_bucket" { + type = bool + description = "Create a versioned S3 bucket for small Atlas offsite backup tests." + default = false +} diff --git a/terraform/aether/versions.tf b/terraform/aether/versions.tf new file mode 100644 index 000000000..c1195b506 --- /dev/null +++ b/terraform/aether/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.6.0" + + backend "local" { + path = "terraform.tfstate" + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 5.0, < 7.0" + } + } +} diff --git a/testing/quality_contract.json b/testing/quality_contract.json index 83086c4c8..e8793d846 100644 --- a/testing/quality_contract.json +++ b/testing/quality_contract.json @@ -16,6 +16,7 @@ "managed_modules": [ "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", + "ci/scripts/semgrep_report.py", "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", @@ -29,12 +30,14 @@ "ci/tests/glue/test_glue_metrics.py", "testing/tests/test_publish_test_metrics.py", "testing/tests/test_supply_chain_report.py", + "testing/tests/test_semgrep_report.py", "testing/tests/test_quality_contract.py", "testing/tests/test_quality_gate.py" ], "lint_paths": [ "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", + "ci/scripts/semgrep_report.py", "ci/tests/glue", "scripts/tests", "services/comms/scripts/tests", @@ -54,7 +57,7 @@ "junit": "build/junit-unit.xml", "coverage_sources": [ "ci/scripts", - "scripts.dashboards_render_atlas", + "scripts.render.dashboards_render_atlas", "services/mailu/scripts", "testing" ], @@ -89,15 +92,15 @@ }, "manual_scripts": [ { - "path": "scripts/test_atlas_user_cleanup.py", + "path": "scripts/manual-tests/test_atlas_user_cleanup.py", "description": "Manual cleanup validation for Atlas user lifecycle automation." }, { - "path": "scripts/test_user_cleanup.py", + "path": "scripts/manual-tests/test_user_cleanup.py", "description": "Manual cleanup validation for shared user lifecycle automation." }, { - "path": "scripts/test_vaultwarden_user_cleanup.py", + "path": "scripts/manual-tests/test_vaultwarden_user_cleanup.py", "description": "Manual cleanup validation for Vaultwarden user lifecycle automation." }, { @@ -167,6 +170,7 @@ "tracked_files": [ "ci/scripts/publish_test_metrics.py", "ci/scripts/publish_test_metrics_quality.py", + "ci/scripts/semgrep_report.py", "ci/scripts/supply_chain_report.py", "services/mailu/scripts/mailu_sync.py", "services/mailu/scripts/mailu_sync_listener.py", diff --git a/testing/tests/test_publish_test_metrics.py b/testing/tests/test_publish_test_metrics.py index aa183a6b5..3f1c45b86 100644 --- a/testing/tests/test_publish_test_metrics.py +++ b/testing/tests/test_publish_test_metrics.py @@ -114,6 +114,7 @@ def test_build_check_statuses_maps_legacy_names_and_reports(): workspace_line_coverage_percent=97.0, source_lines_over_500=0, sonarqube_report={"projectStatus": {"status": "OK"}}, + semgrep_report={"status": "ok"}, supply_chain_report={"status": "compliant"}, supply_chain_required=True, ) @@ -125,6 +126,7 @@ def test_build_check_statuses_maps_legacy_names_and_reports(): "docs_naming": "ok", "gate_glue": "failed", "sonarqube": "ok", + "semgrep": "ok", "supply_chain": "ok", } @@ -136,6 +138,7 @@ def test_build_check_statuses_handles_missing_reports(): workspace_line_coverage_percent=0.0, source_lines_over_500=2, sonarqube_report={}, + semgrep_report={}, supply_chain_report={}, supply_chain_required=False, ) @@ -146,6 +149,7 @@ def test_build_check_statuses_handles_missing_reports(): assert check_statuses["docs_naming"] == "not_applicable" assert check_statuses["gate_glue"] == "not_applicable" assert check_statuses["sonarqube"] == "not_applicable" + assert check_statuses["semgrep"] == "not_applicable" assert check_statuses["supply_chain"] == "not_applicable" @@ -170,6 +174,8 @@ def test_status_normalization_and_optional_reports(tmp_path: Path): assert publish_test_metrics._infer_supply_chain_status({"status": "failed"}, required=False) == "not_applicable" assert publish_test_metrics._infer_supply_chain_status({"status": None}, required=False) == "not_applicable" assert publish_test_metrics._infer_supply_chain_status({"status": "not_applicable"}, required=True) == "failed" + assert publish_test_metrics._infer_semgrep_status({"blocking_findings": 0, "errors_total": 0}) == "ok" + assert publish_test_metrics._infer_semgrep_status({"blocking_findings": 1, "errors_total": 0}) == "failed" def test_build_check_statuses_handles_non_dict_results_and_fallbacks(): @@ -187,6 +193,7 @@ def test_build_check_statuses_handles_non_dict_results_and_fallbacks(): workspace_line_coverage_percent=94.0, source_lines_over_500=0, sonarqube_report={"status": "ERROR"}, + semgrep_report={"status": "failed"}, supply_chain_report={"status": "not_applicable"}, supply_chain_required=True, ) @@ -197,6 +204,7 @@ def test_build_check_statuses_handles_non_dict_results_and_fallbacks(): assert check_statuses["docs_naming"] == "ok" assert check_statuses["gate_glue"] == "ok" assert check_statuses["sonarqube"] == "failed" + assert check_statuses["semgrep"] == "failed" assert check_statuses["supply_chain"] == "failed" @@ -342,12 +350,14 @@ def test_build_payload_includes_canonical_checks(): "docs_naming": "ok", "gate_glue": "ok", "sonarqube": "failed", + "semgrep": "ok", "supply_chain": "failed", }, ) assert 'platform_quality_gate_runs_total{suite="titan-iac",status="ok"} 7' in payload assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="docs_naming",result="ok"} 1' in payload + assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="semgrep",result="ok"} 1' in payload assert 'titan_iac_quality_gate_checks_total{suite="titan-iac",check="tests",result="failed"} 1' in payload @@ -430,7 +440,7 @@ def test_main_marks_successful_run(tmp_path: Path, monkeypatch, capsys): summary = json.loads(capsys.readouterr().out) assert rc == 0 assert summary["status"] == "ok" - assert summary["checks_recorded"] == 7 + assert summary["checks_recorded"] == 8 def test_main_requires_supply_chain_when_images_are_published(tmp_path: Path, monkeypatch): diff --git a/testing/tests/test_publish_test_metrics_quality.py b/testing/tests/test_publish_test_metrics_quality.py index 2b6d2cb77..eb96f9be7 100644 --- a/testing/tests/test_publish_test_metrics_quality.py +++ b/testing/tests/test_publish_test_metrics_quality.py @@ -68,6 +68,7 @@ def test_build_check_statuses_uses_fallbacks_for_tests_docs_and_gate_glue() -> N workspace_line_coverage_percent=0.0, source_lines_over_500=0, sonarqube_report={}, + semgrep_report={}, supply_chain_report={}, supply_chain_required=False, ) @@ -94,6 +95,7 @@ def test_build_check_statuses_preserves_explicit_loc_docs_and_glue_results() -> workspace_line_coverage_percent=96.0, source_lines_over_500=9, sonarqube_report={}, + semgrep_report={"status": "ok"}, supply_chain_report={}, supply_chain_required=False, ) @@ -103,3 +105,4 @@ def test_build_check_statuses_preserves_explicit_loc_docs_and_glue_results() -> assert statuses["loc"] == "ok" assert statuses["docs_naming"] == "ok" assert statuses["gate_glue"] == "ok" + assert statuses["semgrep"] == "ok" diff --git a/testing/tests/test_repo_structure.py b/testing/tests/test_repo_structure.py new file mode 100644 index 000000000..a98718c67 --- /dev/null +++ b/testing/tests/test_repo_structure.py @@ -0,0 +1,126 @@ +"""Repository structure guardrails for the GitOps control plane.""" + +from __future__ import annotations + +import datetime as dt +import json +import stat +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] + + +def _yaml_docs(path: Path) -> list[dict]: + docs = [] + for doc in yaml.safe_load_all(path.read_text(encoding="utf-8")): + if isinstance(doc, dict): + docs.append(doc) + return docs + + +def test_root_jenkinsfile_mirrors_canonical_pipeline() -> None: + """The root Jenkinsfile exists for discovery, but CI logic lives in ci/.""" + root = (REPO_ROOT / "Jenkinsfile").read_text(encoding="utf-8").splitlines() + canonical = (REPO_ROOT / "ci/Jenkinsfile.titan-iac").read_text( + encoding="utf-8" + ).splitlines() + + if root and root[0].startswith("// Mirror of "): + root = root[1:] + + assert root == canonical + + +def test_suspended_flux_kustomizations_have_reasons() -> None: + """Suspended reconciliations should say why they are parked.""" + missing = [] + for path in sorted((REPO_ROOT / "clusters/atlas/flux-system").rglob("*.yaml")): + for doc in _yaml_docs(path): + if doc.get("kind") != "Kustomization": + continue + if not str(doc.get("apiVersion") or "").startswith( + "kustomize.toolkit.fluxcd.io/" + ): + continue + spec = doc.get("spec") or {} + if spec.get("suspend") is not True: + continue + annotations = (doc.get("metadata") or {}).get("annotations") or {} + if not annotations.get("atlas.bstein.dev/suspend-reason"): + missing.append(str(path.relative_to(REPO_ROOT))) + + assert missing == [] + + +def test_trivy_waiver_targets_still_exist() -> None: + """Waivers should stay temporary and point at files that still exist.""" + path = REPO_ROOT / "ci/titan-iac-trivy-waivers.json" + payload = json.loads(path.read_text(encoding="utf-8")) + expires = dt.date.fromisoformat(payload["default_expires_at"]) + assert expires >= dt.date.today() + + missing = [] + for entry in payload.get("misconfigurations", []): + for target in entry.get("targets", []): + if target and not (REPO_ROOT / target).exists(): + missing.append(f"{entry.get('id')}:{target}") + + assert missing == [] + + +def test_oceanus_is_not_a_standalone_flux_cluster() -> None: + """Oceanus now belongs to Atlas capacity, not a separate Flux root.""" + assert not (REPO_ROOT / "clusters/oceanus").exists() + assert not (REPO_ROOT / "infrastructure/modules/profiles/oceanus-validator").exists() + + +def test_service_job_directories_use_lifecycle_names() -> None: + """Manual job folders should say when they are used.""" + old_dirs = list((REPO_ROOT / "services").rglob("oneoffs")) + assert old_dirs == [] + + stale_refs = [] + for root in ["clusters", "services", "ci", "testing"]: + for path in (REPO_ROOT / root).rglob("*"): + if not path.is_file(): + continue + if path == Path(__file__).resolve(): + continue + if path.suffix not in {".json", ".py", ".yaml", ".yml"}: + continue + if "oneoffs" in path.read_text(encoding="utf-8"): + stale_refs.append(str(path.relative_to(REPO_ROOT))) + + assert stale_refs == [] + + +def test_knowledge_service_mirror_matches_source() -> None: + """Atlasbot should serve the generated knowledge tree, not a fork.""" + source = REPO_ROOT / "knowledge" + mirror = REPO_ROOT / "services/comms/knowledge" + source_files = sorted(path.relative_to(source) for path in source.rglob("*") if path.is_file()) + mirror_files = sorted(path.relative_to(mirror) for path in mirror.rglob("*") if path.is_file()) + + assert mirror_files == source_files + mismatched = [ + str(rel) + for rel in source_files + if (source / rel).read_bytes() != (mirror / rel).read_bytes() + ] + assert mismatched == [] + + +def test_scripts_shell_entrypoints_are_executable_with_shebangs() -> None: + """Runnable shell and Fish helpers should not need tribal chmod knowledge.""" + broken = [] + for path in sorted((REPO_ROOT / "scripts").rglob("*")): + if path.suffix not in {".sh", ".fish"} or not path.is_file(): + continue + mode = path.stat().st_mode + first_line = path.read_text(encoding="utf-8").splitlines()[0] + if not mode & stat.S_IXUSR or not first_line.startswith("#!"): + broken.append(str(path.relative_to(REPO_ROOT))) + + assert broken == [] diff --git a/testing/tests/test_semgrep_report.py b/testing/tests/test_semgrep_report.py new file mode 100644 index 000000000..e0cf1735e --- /dev/null +++ b/testing/tests/test_semgrep_report.py @@ -0,0 +1,170 @@ +"""Unit tests for Semgrep report normalization.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from ci.scripts import semgrep_report + + +def test_build_report_counts_severities_and_blocks_errors() -> None: + """ERROR findings should fail the report while lower severities remain visible.""" + + report = semgrep_report.build_report( + { + "results": [ + {"extra": {"severity": "ERROR"}}, + {"extra": {"severity": "WARNING"}}, + {"extra": {"severity": "INFO"}}, + {"extra": {"severity": "unknown-ish"}}, + ], + "errors": [], + }, + semgrep_exit_code=1, + ) + + assert report["status"] == "failed" + assert report["findings_total"] == 4 + assert report["blocking_findings"] == 1 + assert report["severity_counts"] == { + "ERROR": 1, + "WARNING": 1, + "INFO": 1, + "UNKNOWN": 1, + } + + +def test_build_report_marks_clean_scan_ok() -> None: + """Clean Semgrep JSON with exit code zero should publish an OK report.""" + + report = semgrep_report.build_report( + {"results": [], "errors": []}, + semgrep_exit_code=0, + ) + + assert report["status"] == "ok" + assert report["blocking_findings"] == 0 + assert report["errors_total"] == 0 + + +def test_build_sonar_issues_converts_findings() -> None: + """Semgrep findings should be importable by SonarQube generic issues.""" + + issues = semgrep_report.build_sonar_issues( + { + "results": [ + { + "check_id": "python.lang.security.audit.danger", + "path": "app/main.py", + "start": {"line": 7}, + "end": {"line": 9}, + "extra": {"severity": "ERROR", "message": "dangerous call"}, + }, + {"check_id": "missing-path", "extra": {"severity": "INFO"}}, + ] + } + ) + + assert len(issues["issues"]) == 1 + assert issues["issues"][0]["engineId"] == "semgrep" + assert issues["issues"][0]["severity"] == "CRITICAL" + assert issues["issues"][0]["primaryLocation"]["filePath"] == "app/main.py" + assert issues["issues"][0]["primaryLocation"]["textRange"] == { + "startLine": 7, + "endLine": 9, + } + + +def test_build_sonar_issues_uses_safe_defaults_for_sparse_findings() -> None: + """Sparse findings should still produce valid file-level Sonar issues.""" + + issues = semgrep_report.build_sonar_issues( + { + "results": [ + { + "path": "app/config.yaml", + "start": {"line": "bad"}, + "end": {"line": 0}, + "extra": "not-a-dict", + } + ] + } + ) + + issue = issues["issues"][0] + assert issue["ruleId"] == "semgrep.unknown" + assert issue["type"] == "CODE_SMELL" + assert issue["severity"] == "INFO" + assert issue["primaryLocation"]["message"] == "Semgrep finding" + assert issue["primaryLocation"]["textRange"] == {"startLine": 1, "endLine": 1} + + +def test_read_json_handles_invalid_json_and_non_object(tmp_path: Path) -> None: + """Report loading should fail closed for bad JSON and wrong top-level shapes.""" + + bad_json = tmp_path / "bad.json" + bad_json.write_text("{bad", encoding="utf-8") + list_json = tmp_path / "list.json" + list_json.write_text("[]", encoding="utf-8") + + assert semgrep_report._read_json(bad_json)["errors"][0]["message"].startswith("invalid JSON:") + assert semgrep_report._read_json(list_json) == { + "errors": [{"message": "report payload is not an object"}], + "results": [], + } + + +def test_main_handles_missing_or_invalid_input(tmp_path: Path) -> None: + """Missing or invalid Semgrep JSON should still produce a failed report file.""" + + output = tmp_path / "build" / "semgrep-report.json" + + rc = semgrep_report.main( + [ + "--semgrep-json", + str(tmp_path / "missing.json"), + "--exit-code", + "2", + "--output", + str(output), + "--sonar-issues-output", + str(tmp_path / "build" / "sonar-issues.json"), + ] + ) + + assert rc == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["status"] == "failed" + assert payload["errors_total"] == 1 + sonar_payload = json.loads((tmp_path / "build" / "sonar-issues.json").read_text(encoding="utf-8")) + assert sonar_payload == {"issues": []} + + +def test_main_honors_custom_blocking_severity_without_sonar_output(tmp_path: Path) -> None: + """CLI callers can choose stricter blocking severities without writing Sonar issues.""" + + semgrep_json = tmp_path / "semgrep.json" + semgrep_json.write_text( + json.dumps({"results": [{"path": "app.py", "extra": {"severity": "WARNING"}}], "errors": []}), + encoding="utf-8", + ) + output = tmp_path / "semgrep-report.json" + + rc = semgrep_report.main( + [ + "--semgrep-json", + str(semgrep_json), + "--exit-code", + "1", + "--output", + str(output), + "--blocking-severity", + "WARNING", + ] + ) + + assert rc == 0 + payload = json.loads(output.read_text(encoding="utf-8")) + assert payload["status"] == "failed" + assert payload["blocking_findings"] == 1 diff --git a/testing/tests/test_supply_chain_report.py b/testing/tests/test_supply_chain_report.py index d47b5841a..5c249d306 100644 --- a/testing/tests/test_supply_chain_report.py +++ b/testing/tests/test_supply_chain_report.py @@ -106,6 +106,28 @@ def test_build_report_fails_expired_waivers_and_secrets(tmp_path: Path): assert report["waived_misconfigurations"] == 0 +def test_build_report_fails_when_only_waivers_are_expired(tmp_path: Path): + """Expiry alone is enough to fail the supply-chain baseline.""" + waivers = tmp_path / "waivers.json" + _write_json( + waivers, + { + "default_expires_at": "2026-04-01", + "misconfigurations": [ + {"id": "KSV-0014", "targets": ["services/example/deployment.yaml"]} + ], + }, + ) + + report = supply_chain_report.build_report( + {"Results": []}, waivers, today_override="2026-04-22" + ) + + assert report["status"] == "failed" + assert report["compliant"] is False + assert report["expired_waivers"] == 1 + + def test_build_report_handles_missing_and_malformed_waiver_entries(tmp_path: Path): """Malformed waiver rows are ignored instead of hiding real findings.""" missing_waivers = tmp_path / "missing.json"