nodes: repair credential audit and preserve Vault records

scripts/node_admin_access.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+"""Audit or restore existing node passwords from JSON on encrypted SSH stdin.
+
+Run as root with {"hostname": "titan-12", "passwords": {"atlas": "...",
+"root": "..."}} on stdin. Passwords must come from the node's Vault record.
+No passwords or hashes are written to files or output. SSH policy is unchanged.
+"""
+
+import argparse
+import ctypes
+import ctypes.util
+import hmac
+import json
+import os
+import pwd
+import socket
+import subprocess
+import sys
+
+
+def password_status(username, password):
+    """Compare an existing shadow hash locally; return non-secret state only."""
+    with open("/etc/shadow", encoding="utf-8") as stream:
+        row = next((line.rstrip("\n").split(":") for line in stream
+                    if line.startswith(username + ":")), None)
+    if row is None:
+        raise ValueError("account_missing")
+    stored = row[1]
+    locked = stored.startswith(("!", "*")) or not stored
+    matches = False
+    if not locked:
+        library = ctypes.CDLL(ctypes.util.find_library("crypt"))
+        library.crypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p]
+        library.crypt.restype = ctypes.c_char_p
+        calculated = library.crypt(password.encode(), stored.encode())
+        matches = bool(calculated and hmac.compare_digest(calculated, stored.encode()))
+    return {"locked": locked, "vault_password_matches": matches}
+
+
+def run(payload, apply=False):
+    """Validate all input before applying only missing/mismatched passwords."""
+    if os.geteuid() != 0:
+        raise ValueError("root_required")
+    hostname = socket.gethostname().split(".")[0]
+    if payload.get("hostname") != hostname:
+        raise ValueError("hostname_mismatch")
+    passwords = payload.get("passwords")
+    if not isinstance(passwords, dict) or not passwords:
+        raise ValueError("passwords_required")
+    for username, password in passwords.items():
+        if username not in {"atlas", "root"}:
+            raise ValueError("account_not_allowed")
+        if not isinstance(password, str) or not password or any(c in password for c in "\r\n\x00"):
+            raise ValueError("invalid_password")
+        pwd.getpwnam(username)
+    before = {user: password_status(user, value) for user, value in passwords.items()}
+    changed = []
+    if apply:
+        for user, value in passwords.items():
+            if not before[user]["vault_password_matches"]:
+                completed = subprocess.run(["/usr/sbin/chpasswd"], input=user + ":" + value + "\n",
+                                           text=True, capture_output=True, timeout=15)
+                if completed.returncode:
+                    raise ValueError("password_update_failed")
+                changed.append(user)
+    after = {user: password_status(user, value) for user, value in passwords.items()}
+    if apply and not all(state["vault_password_matches"] for state in after.values()):
+        raise ValueError("password_verification_failed")
+    return {"hostname": hostname, "applied": apply, "changed_accounts": changed,
+            "before": before, "after": after}
+
+
+def main():
+    """Read one bounded payload and emit only safe operational metadata."""
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--apply", action="store_true")
+    args = parser.parse_args()
+    try:
+        content = sys.stdin.read(65537)
+        if len(content) > 65536:
+            raise ValueError("payload_too_large")
+        result = run(json.loads(content), args.apply)
+    except Exception as error:
+        # Do not echo exception messages: malformed input can contain credentials.
+        print(json.dumps({"error": "node_access_operation_failed", "type": type(error).__name__}))
+        return 1
+    print(json.dumps(result, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml

@@ -1,15 +1,14 @@
 # services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml
-# One-off job for sso/metis-node-passwords-secret-ensure-4.
+# One-off job for sso/metis-node-passwords-secret-ensure-5.
 # Purpose: ensure per-node Metis recovery placeholders exist in Vault.
 # Atlas/root values are preserved while intranet IPs are standardized per node.
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: metis-node-passwords-secret-ensure-4
+  name: metis-node-passwords-secret-ensure-5
   namespace: sso
 spec:
   backoffLimit: 0
-  ttlSecondsAfterFinished: 3600
   template:
     spec:
       serviceAccountName: mas-secrets-ensure
@@ -35,49 +34,53 @@ spec:
           args:
             - |
               set -eu
-
+              umask 077
+              work_dir="$(mktemp -d)"
+              trap 'rm -rf "${work_dir}"' EXIT HUP INT TERM
               vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
               vault_role="${VAULT_ROLE:-sso-secrets}"
-
-              jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
-              login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
-              vault_token="$(curl -sS --request POST --data "${login_payload}"                 "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
-              if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
-                echo "vault login failed" >&2
-                exit 1
-              fi
-
+              jwt_file="${SERVICE_ACCOUNT_TOKEN_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}"
+              jq -nc --rawfile jwt "${jwt_file}" --arg role "${vault_role}" \
+                '{jwt:($jwt|rtrimstr("\n")),role:$role}' > "${work_dir}/login.json"
+              curl -fsS --max-time 30 --request POST \
+                --data-binary "@${work_dir}/login.json" \
+                "${vault_addr}/v1/auth/kubernetes/login" > "${work_dir}/auth.json"
+              jq -er '.auth.client_token | select(type == "string" and length > 0) |
+                "header = " + (("X-Vault-Token: " + .) | @json)' \
+                "${work_dir}/auth.json" > "${work_dir}/auth.curl"
               ensured=0
               while read -r node intranet_ip; do
-                if [ -z "${node}" ] || [ -z "${intranet_ip}" ]; then
+                [ -n "${node}" ] && [ -n "${intranet_ip}" ] || continue
+                secret_path="kv/data/atlas/nodes/${node}"
+                read_status="$(curl -sS --max-time 30 --config "${work_dir}/auth.curl" \
+                  -o "${work_dir}/read.json" -w '%{http_code}' \
+                  "${vault_addr}/v1/${secret_path}")"
+                case "${read_status}" in
+                  200) ;;
+                  404) printf '%s\n' '{"data":{"data":{},"metadata":{"version":0}}}' > "${work_dir}/read.json" ;;
+                  *) echo "Vault read failed for ${node} (HTTP ${read_status})" >&2; exit 1 ;;
+                esac
+                # Preserve all existing fields. CAS prevents a concurrent password update
+                # from being overwritten; unchanged records do not create new versions.
+                jq -e --arg ip "${intranet_ip}" '
+                  .data as $existing |
+                  {options:{cas:$existing.metadata.version},
+                   data:({atlas_password:"",root_password:""} + $existing.data + {intranet_ip:$ip})}
+                ' "${work_dir}/read.json" > "${work_dir}/write.json"
+                if jq -e --slurpfile update "${work_dir}/write.json" \
+                  '.data.data == $update[0].data' "${work_dir}/read.json" >/dev/null; then
                   continue
                 fi
-
-                secret_path="kv/data/atlas/nodes/${node}"
-                read_status="$(curl -sS -o /tmp/node-read.json -w "%{http_code}"                   -H "X-Vault-Token: ${vault_token}"                   "${vault_addr}/v1/${secret_path}" || true)"
-                if [ "${read_status}" = "200" ]; then
-                  atlas_password="$(jq -r '.data.data.atlas_password // empty' /tmp/node-read.json)"
-                  root_password="$(jq -r '.data.data.root_password // empty' /tmp/node-read.json)"
-                elif [ "${read_status}" = "404" ]; then
-                  atlas_password=""
-                  root_password=""
-                else
-                  echo "Vault read failed for ${node} (status ${read_status})" >&2
-                  cat /tmp/node-read.json >&2 || true
-                  exit 1
-                fi
-
-                payload="$(jq -nc                   --arg atlas_password "${atlas_password}"                   --arg root_password "${root_password}"                   --arg intranet_ip "${intranet_ip}"                   '{data:{atlas_password:$atlas_password,root_password:$root_password,intranet_ip:$intranet_ip}}')"
-
-                write_status="$(curl -sS -o /tmp/node-write.json -w "%{http_code}" -X POST                   -H "X-Vault-Token: ${vault_token}"                   -H 'Content-Type: application/json'                   -d "${payload}"                   "${vault_addr}/v1/${secret_path}")"
-                if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then
-                  echo "Vault write failed for ${node} (status ${write_status})" >&2
-                  cat /tmp/node-write.json >&2 || true
-                  exit 1
-                fi
-
+                write_status="$(curl -sS --max-time 30 --config "${work_dir}/auth.curl" \
+                  -o "${work_dir}/result.json" -w '%{http_code}' --request POST \
+                  -H 'Content-Type: application/json' --data-binary "@${work_dir}/write.json" \
+                  "${vault_addr}/v1/${secret_path}")"
+                case "${write_status}" in
+                  200|204) ;;
+                  *) echo "Vault update failed for ${node} (HTTP ${write_status}); no retry over concurrent writes" >&2; exit 1 ;;
+                esac
                 ensured=$((ensured + 1))
-                echo "Ensured node secret placeholder for ${node} (${intranet_ip})"
+                echo "Updated node metadata: ${node}"
               done <<'EOF_NODES'
               titan-jh 192.168.22.8
               titan-db 192.168.22.10
@@ -87,7 +90,7 @@ spec:
               titan-20 192.168.22.20
               titan-21 192.168.22.21
               titan-22 192.168.22.22
-              titan-23 192.168.22.23
+              titan-23 192.168.22.24
               titan-24 192.168.22.26
               titan-04 192.168.22.30
               titan-05 192.168.22.31
@@ -107,4 +110,4 @@ spec:
               titan-19 192.168.22.47
               EOF_NODES
 
-              echo "Ensured ${ensured} Metis node placeholders in Vault"
+              echo "Updated ${ensured} Metis node records; unchanged records preserved"

testing/tests/test_node_admin_access.py

@@ -0,0 +1,75 @@
+"""Node credential repair must be targeted, idempotent and silent about secrets."""
+import importlib.util
+from pathlib import Path
+import unittest
+from unittest.mock import patch, Mock
+
+spec = importlib.util.spec_from_file_location(
+    "node_admin_access", Path(__file__).resolve().parents[2] / "scripts/node_admin_access.py")
+module = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(module)
+
+
+class NodeAccessTests(unittest.TestCase):
+    def setUp(self):
+        self.payload = {"hostname": "titan-test", "passwords": {"atlas": "synthetic-only"}}
+        self.root = patch.object(module.os, "geteuid", return_value=0)
+        self.host = patch.object(module.socket, "gethostname", return_value="titan-test")
+        self.account = patch.object(module.pwd, "getpwnam")
+        for item in [self.root, self.host, self.account]:
+            item.start()
+            self.addCleanup(item.stop)
+
+    def test_wrong_host_never_changes_password(self):
+        with patch.object(module.subprocess, "run") as run:
+            self.payload["hostname"] = "wrong-node"
+            with self.assertRaisesRegex(ValueError, "hostname_mismatch"):
+                module.run(self.payload, True)
+            run.assert_not_called()
+
+    def test_validate_all_accounts_before_mutation(self):
+        with patch.object(module.subprocess, "run") as run:
+            self.payload["passwords"]["other"] = "synthetic"
+            with self.assertRaisesRegex(ValueError, "account_not_allowed"):
+                module.run(self.payload, True)
+            run.assert_not_called()
+
+    def test_password_record_injection_rejected(self):
+        self.payload["passwords"]["atlas"] = "bad\nroot:injected"
+        with self.assertRaisesRegex(ValueError, "invalid_password"):
+            module.run(self.payload, True)
+
+    def test_audit_is_read_only(self):
+        with patch.object(module, "password_status", return_value={"vault_password_matches": False}), \
+             patch.object(module.subprocess, "run") as run:
+            result = module.run(self.payload)
+            run.assert_not_called()
+            self.assertEqual(result["changed_accounts"], [])
+
+    def test_matching_password_is_unchanged(self):
+        with patch.object(module, "password_status", return_value={"vault_password_matches": True}), \
+             patch.object(module.subprocess, "run") as run:
+            module.run(self.payload, True)
+            run.assert_not_called()
+
+    def test_restore_uses_stdin_and_returns_no_secret(self):
+        with patch.object(module, "password_status", side_effect=[
+            {"vault_password_matches": False, "locked": True},
+            {"vault_password_matches": True, "locked": False}]), \
+             patch.object(module.subprocess, "run", return_value=Mock(returncode=0)) as run:
+            result = module.run(self.payload, True)
+            self.assertEqual(run.call_args.args[0], ["/usr/sbin/chpasswd"])
+            self.assertEqual(run.call_args.kwargs["input"], "atlas:synthetic-only\n")
+            self.assertNotIn("synthetic-only", str(result))
+            self.assertEqual(result["changed_accounts"], ["atlas"])
+            self.assertTrue(result["after"]["atlas"]["vault_password_matches"])
+
+    def test_failed_update_does_not_echo_subprocess(self):
+        with patch.object(module, "password_status", return_value={"vault_password_matches": False}), \
+             patch.object(module.subprocess, "run", return_value=Mock(returncode=1, stderr="SECRET")):
+            with self.assertRaisesRegex(ValueError, "^password_update_failed$"):
+                module.run(self.payload, True)
+
+
+if __name__ == "__main__":
+    unittest.main()

testing/tests/test_node_secret_bootstrap.py

@@ -0,0 +1,79 @@
+"""Exercise the real bootstrap shell with a local fake Vault transport."""
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+import yaml
+
+ROOT = Path(__file__).resolve().parents[2]
+MANIFEST = ROOT / 'services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml'
+FAKE_CURL = r'''#!/usr/bin/env python3
+import json,os,pathlib,sys
+args=sys.argv[1:]
+state=pathlib.Path(os.environ['FAKE_STATE'])
+s=json.loads(state.read_text())
+if args[-1].endswith('/login'):
+ print(json.dumps({'auth':{'client_token':'synthetic-token'}})); sys.exit(0)
+node=args[-1].rsplit('/',1)[-1]
+out=pathlib.Path(args[args.index('-o')+1])
+if '--data-binary' in args:
+ p=json.loads(pathlib.Path(args[args.index('--data-binary')+1][1:]).read_text())
+ s['writes'].append(p)
+ status='400' if s.get('conflict') else '200'
+ out.write_text('UNSAFE BODY MUST NOT APPEAR IN LOGS')
+else:
+ d=s['records'].get(node)
+ status='200' if d else '404'
+ out.write_text(json.dumps({'data':{'data':d,'metadata':{'version':7}}}))
+state.write_text(json.dumps(s)); print(status,end='')
+'''
+
+
+class BootstrapTests(unittest.TestCase):
+    def exercise(self, records, conflict=False):
+        manifest = yaml.safe_load(MANIFEST.read_text())
+        self.assertNotIn('ttlSecondsAfterFinished', manifest['spec'])
+        script = manifest['spec']['template']['spec']['containers'][0]['args'][0]
+        with tempfile.TemporaryDirectory() as d:
+            folder = Path(d)
+            fake = folder / 'curl'
+            fake.write_text(FAKE_CURL)
+            fake.chmod(0o755)
+            state = folder / 'state.json'
+            state.write_text(json.dumps({'records': records, 'writes': [], 'conflict': conflict}))
+            jwt = folder / 'jwt'
+            jwt.write_text('synthetic-jwt')
+            env = dict(os.environ, PATH=d + os.pathsep + os.environ['PATH'],
+                       FAKE_STATE=str(state), SERVICE_ACCOUNT_TOKEN_FILE=str(jwt))
+            result = subprocess.run(['/bin/sh', '-c', script], env=env,
+                                    capture_output=True, text=True, timeout=30)
+            self.assertNotIn('UNSAFE BODY', result.stdout + result.stderr)
+            self.assertNotIn('synthetic-password', result.stdout + result.stderr)
+            return result, json.loads(state.read_text())
+
+    def test_preserves_passwords_extra_fields_and_cas(self):
+        result, state = self.exercise({'titan-jh': {'atlas_password': 'synthetic-password',
+                        'root_password': 'synthetic-root', 'operator_note': 'keep'}})
+        self.assertEqual(result.returncode, 0, result.stderr)
+        first = state['writes'][0]
+        self.assertEqual(first['options']['cas'], 7)
+        self.assertEqual(first['data']['atlas_password'], 'synthetic-password')
+        self.assertEqual(first['data']['operator_note'], 'keep')
+        self.assertEqual(state['writes'][1]['options']['cas'], 0)
+
+    def test_unchanged_secret_has_no_new_version(self):
+        result, state = self.exercise({'titan-jh': {'atlas_password': 'synthetic-password',
+               'root_password': 'synthetic-root', 'intranet_ip': '192.168.22.8'}})
+        self.assertEqual(result.returncode, 0)
+        self.assertFalse(any(w['data']['intranet_ip'] == '192.168.22.8' for w in state['writes']))
+
+    def test_concurrent_write_fails_closed(self):
+        result, state = self.exercise({}, conflict=True)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(len(state['writes']), 1)
+
+
+if __name__ == '__main__':
+    unittest.main()