From e84399d0b1c7b70367d16461cdb2fc15781a3018 Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Sun, 5 Apr 2026 10:25:29 -0300
Subject: [PATCH] maintenance: source metis SSH keys from Vault

---
 services/keycloak/kustomization.yaml          |   1 +
 .../metis-ssh-keys-secret-ensure-job.yaml     | 113 ++++++++++++++++++
 services/maintenance/secretproviderclass.yaml |  18 +++
 .../vault/scripts/vault_k8s_auth_configure.sh |   6 +-
 4 files changed, 135 insertions(+), 3 deletions(-)
 create mode 100644 services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml

diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml
index 7e69b28f..c532a6fa 100644
--- a/services/keycloak/kustomization.yaml
+++ b/services/keycloak/kustomization.yaml
@@ -23,6 +23,7 @@ resources:
   - oneoffs/synapse-oidc-secret-ensure-job.yaml
   - oneoffs/logs-oidc-secret-ensure-job.yaml
   - oneoffs/metis-oidc-secret-ensure-job.yaml
+  - oneoffs/metis-ssh-keys-secret-ensure-job.yaml
   - oneoffs/harbor-oidc-secret-ensure-job.yaml
   - oneoffs/vault-oidc-secret-ensure-job.yaml
   - oneoffs/actual-oidc-secret-ensure-job.yaml
diff --git a/services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml b/services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml
new file mode 100644
index 00000000..14e259f1
--- /dev/null
+++ b/services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml
@@ -0,0 +1,113 @@
+# services/keycloak/oneoffs/metis-ssh-keys-secret-ensure-job.yaml
+# One-off job for sso/metis-ssh-keys-secret-ensure-1.
+# Purpose: ensure Vault path maintenance/metis-ssh-keys exists for Metis key injection.
+# Migration behavior: if Vault path is missing, seed from existing maintenance/metis-ssh-keys Kubernetes Secret.
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: metis-ssh-keys-secret-ensure-1
+  namespace: sso
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    metadata:
+      annotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/agent-pre-populate-only: "true"
+        vault.hashicorp.com/role: "sso-secrets"
+        vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
+        vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
+          {{ with secret "kv/data/atlas/shared/keycloak-admin" }}
+          export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
+          {{ end }}
+    spec:
+      serviceAccountName: mas-secrets-ensure
+      restartPolicy: Never
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: node-role.kubernetes.io/worker
+                    operator: Exists
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: ["arm64"]
+      containers:
+        - name: apply
+          image: registry.bstein.dev/bstein/kubectl:1.35.0
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+
+              vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
+              vault_role="${VAULT_ROLE:-sso-secrets}"
+              vault_path="kv/data/atlas/maintenance/metis-ssh-keys"
+
+              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
+              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
+              vault_token="$(curl -sS --request POST --data "${login_payload}" \
+                "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
+              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
+                echo "vault login failed" >&2
+                exit 1
+              fi
+
+              read_status="$(curl -sS -o /tmp/metis-ssh-read.json -w "%{http_code}" \
+                -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/${vault_path}" || true)"
+              if [ "${read_status}" = "200" ]; then
+                bastion_existing="$(jq -r '.data.data.bastion_pub // empty' /tmp/metis-ssh-read.json)"
+                brad_existing="$(jq -r '.data.data.brad_pub // empty' /tmp/metis-ssh-read.json)"
+                hecate_existing="$(jq -r '.data.data.hecate_tethys_pub // empty' /tmp/metis-ssh-read.json)"
+                if [ -n "${bastion_existing}" ] && [ -n "${brad_existing}" ] && [ -n "${hecate_existing}" ]; then
+                  echo "Vault metis-ssh-keys already present"
+                  exit 0
+                fi
+              elif [ "${read_status}" != "404" ]; then
+                echo "Vault read failed (status ${read_status})" >&2
+                cat /tmp/metis-ssh-read.json >&2 || true
+                exit 1
+              fi
+
+              bastion_pub="$(kubectl -n maintenance get secret metis-ssh-keys -o jsonpath='{.data.bastion_pub}' 2>/dev/null | base64 -d || true)"
+              brad_pub="$(kubectl -n maintenance get secret metis-ssh-keys -o jsonpath='{.data.brad_pub}' 2>/dev/null | base64 -d || true)"
+              hecate_tethys_pub="$(kubectl -n maintenance get secret metis-ssh-keys -o jsonpath='{.data.hecate_tethys_pub}' 2>/dev/null | base64 -d || true)"
+
+              if [ -z "${bastion_pub}" ] || [ -z "${brad_pub}" ] || [ -z "${hecate_tethys_pub}" ]; then
+                echo "Cannot seed Vault metis-ssh-keys: maintenance/metis-ssh-keys missing required keys" >&2
+                exit 1
+              fi
+
+              payload="$(jq -nc \
+                --arg bastion_pub "${bastion_pub}" \
+                --arg brad_pub "${brad_pub}" \
+                --arg hecate_tethys_pub "${hecate_tethys_pub}" \
+                '{data:{bastion_pub:$bastion_pub,brad_pub:$brad_pub,hecate_tethys_pub:$hecate_tethys_pub}}')"
+              write_status="$(curl -sS -o /tmp/metis-ssh-write.json -w "%{http_code}" -X POST \
+                -H "X-Vault-Token: ${vault_token}" \
+                -H 'Content-Type: application/json' \
+                -d "${payload}" \
+                "${vault_addr}/v1/${vault_path}")"
+              if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
+                echo "Vault write failed (status ${write_status})" >&2
+                cat /tmp/metis-ssh-write.json >&2 || true
+                exit 1
+              fi
+
+              verify_status="$(curl -sS -o /tmp/metis-ssh-verify.json -w "%{http_code}" \
+                -H "X-Vault-Token: ${vault_token}" \
+                "${vault_addr}/v1/${vault_path}" || true)"
+              if [ "${verify_status}" != "200" ]; then
+                echo "Vault verify failed (status ${verify_status})" >&2
+                cat /tmp/metis-ssh-verify.json >&2 || true
+                exit 1
+              fi
+
+              echo "Metis SSH key material now persisted in Vault at maintenance/metis-ssh-keys"
diff --git a/services/maintenance/secretproviderclass.yaml b/services/maintenance/secretproviderclass.yaml
index fae83c78..ad99ff30 100644
--- a/services/maintenance/secretproviderclass.yaml
+++ b/services/maintenance/secretproviderclass.yaml
@@ -16,6 +16,15 @@ spec:
       - objectName: "harbor-core__harbor_admin_password"
         secretPath: "kv/data/atlas/harbor/harbor-core"
         secretKey: "harbor_admin_password"
+      - objectName: "metis-ssh-keys__bastion_pub"
+        secretPath: "kv/data/atlas/maintenance/metis-ssh-keys"
+        secretKey: "bastion_pub"
+      - objectName: "metis-ssh-keys__brad_pub"
+        secretPath: "kv/data/atlas/maintenance/metis-ssh-keys"
+        secretKey: "brad_pub"
+      - objectName: "metis-ssh-keys__hecate_tethys_pub"
+        secretPath: "kv/data/atlas/maintenance/metis-ssh-keys"
+        secretKey: "hecate_tethys_pub"
   secretObjects:
     - secretName: harbor-regcred
       type: kubernetes.io/dockerconfigjson
@@ -27,3 +36,12 @@ spec:
       data:
         - objectName: harbor-core__harbor_admin_password
           key: METIS_HARBOR_PASSWORD
+    - secretName: metis-ssh-keys
+      type: Opaque
+      data:
+        - objectName: metis-ssh-keys__bastion_pub
+          key: bastion_pub
+        - objectName: metis-ssh-keys__brad_pub
+          key: brad_pub
+        - objectName: metis-ssh-keys__hecate_tethys_pub
+          key: hecate_tethys_pub
diff --git a/services/vault/scripts/vault_k8s_auth_configure.sh b/services/vault/scripts/vault_k8s_auth_configure.sh
index 84b2625a..82d90057 100644
--- a/services/vault/scripts/vault_k8s_auth_configure.sh
+++ b/services/vault/scripts/vault_k8s_auth_configure.sh
@@ -231,7 +231,7 @@ write_policy_and_role "crypto" "crypto" "crypto-vault-sync" \
 write_policy_and_role "health" "health" "health-vault-sync" \
   "health/*" ""
 write_policy_and_role "maintenance" "maintenance" "ariadne,maintenance-vault-sync" \
-  "maintenance/ariadne-db maintenance/metis-oidc portal/atlas-portal-db portal/bstein-dev-home-keycloak-admin mailu/mailu-db-secret mailu/mailu-initial-account-secret nextcloud/nextcloud-db nextcloud/nextcloud-admin health/wger-admin finance/firefly-secrets comms/mas-admin-client-runtime comms/atlasbot-credentials-runtime comms/synapse-db comms/synapse-admin vault/vault-oidc-config shared/harbor-pull harbor/harbor-core" ""
+  "maintenance/ariadne-db maintenance/metis-oidc maintenance/metis-ssh-keys portal/atlas-portal-db portal/bstein-dev-home-keycloak-admin mailu/mailu-db-secret mailu/mailu-initial-account-secret nextcloud/nextcloud-db nextcloud/nextcloud-admin health/wger-admin finance/firefly-secrets comms/mas-admin-client-runtime comms/atlasbot-credentials-runtime comms/synapse-db comms/synapse-admin vault/vault-oidc-config shared/harbor-pull harbor/harbor-core" ""
 write_policy_and_role "finance" "finance" "finance-vault" \
   "finance/* shared/postmark-relay" ""
 write_policy_and_role "finance-secrets" "finance" "finance-secrets-ensure" \
@@ -245,8 +245,8 @@ write_policy_and_role "vault" "vault" "vault" \
   "vault/*" ""
 
 write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
-  "shared/keycloak-admin" \
-  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc"
+  "shared/keycloak-admin maintenance/metis-ssh-keys" \
+  "harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/metis-ssh-keys"
 write_policy_and_role "crypto-secrets" "crypto" "crypto-secrets-ensure" \
   "" \
   "crypto/wallet-monero-temp-rpc-auth"