From 1bd8ae3197112e79cd6cf3bb8decc5842a5d6f08 Mon Sep 17 00:00:00 2001 From: jenkins Date: Sun, 4 Oct 2026 00:09:20 -0500 Subject: [PATCH] nodes: repair credential audit and preserve Vault records --- scripts/node_admin_access.py | 92 +++++++++++++++++++ ...etis-node-passwords-secret-ensure-job.yaml | 85 ++++++++--------- testing/tests/test_node_admin_access.py | 75 +++++++++++++++ testing/tests/test_node_secret_bootstrap.py | 79 ++++++++++++++++ 4 files changed, 290 insertions(+), 41 deletions(-) create mode 100644 scripts/node_admin_access.py create mode 100644 testing/tests/test_node_admin_access.py create mode 100644 testing/tests/test_node_secret_bootstrap.py diff --git a/scripts/node_admin_access.py b/scripts/node_admin_access.py new file mode 100644 index 00000000..71d619b9 --- /dev/null +++ b/scripts/node_admin_access.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Audit or restore existing node passwords from JSON on encrypted SSH stdin. + +Run as root with {"hostname": "titan-12", "passwords": {"atlas": "...", +"root": "..."}} on stdin. Passwords must come from the node's Vault record. +No passwords or hashes are written to files or output. SSH policy is unchanged. +""" + +import argparse +import ctypes +import ctypes.util +import hmac +import json +import os +import pwd +import socket +import subprocess +import sys + + +def password_status(username, password): + """Compare an existing shadow hash locally; return non-secret state only.""" + with open("/etc/shadow", encoding="utf-8") as stream: + row = next((line.rstrip("\n").split(":") for line in stream + if line.startswith(username + ":")), None) + if row is None: + raise ValueError("account_missing") + stored = row[1] + locked = stored.startswith(("!", "*")) or not stored + matches = False + if not locked: + library = ctypes.CDLL(ctypes.util.find_library("crypt")) + library.crypt.argtypes = [ctypes.c_char_p, ctypes.c_char_p] + library.crypt.restype = ctypes.c_char_p + calculated = library.crypt(password.encode(), stored.encode()) + matches = bool(calculated and hmac.compare_digest(calculated, stored.encode())) + return {"locked": locked, "vault_password_matches": matches} + + +def run(payload, apply=False): + """Validate all input before applying only missing/mismatched passwords.""" + if os.geteuid() != 0: + raise ValueError("root_required") + hostname = socket.gethostname().split(".")[0] + if payload.get("hostname") != hostname: + raise ValueError("hostname_mismatch") + passwords = payload.get("passwords") + if not isinstance(passwords, dict) or not passwords: + raise ValueError("passwords_required") + for username, password in passwords.items(): + if username not in {"atlas", "root"}: + raise ValueError("account_not_allowed") + if not isinstance(password, str) or not password or any(c in password for c in "\r\n\x00"): + raise ValueError("invalid_password") + pwd.getpwnam(username) + before = {user: password_status(user, value) for user, value in passwords.items()} + changed = [] + if apply: + for user, value in passwords.items(): + if not before[user]["vault_password_matches"]: + completed = subprocess.run(["/usr/sbin/chpasswd"], input=user + ":" + value + "\n", + text=True, capture_output=True, timeout=15) + if completed.returncode: + raise ValueError("password_update_failed") + changed.append(user) + after = {user: password_status(user, value) for user, value in passwords.items()} + if apply and not all(state["vault_password_matches"] for state in after.values()): + raise ValueError("password_verification_failed") + return {"hostname": hostname, "applied": apply, "changed_accounts": changed, + "before": before, "after": after} + + +def main(): + """Read one bounded payload and emit only safe operational metadata.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true") + args = parser.parse_args() + try: + content = sys.stdin.read(65537) + if len(content) > 65536: + raise ValueError("payload_too_large") + result = run(json.loads(content), args.apply) + except Exception as error: + # Do not echo exception messages: malformed input can contain credentials. + print(json.dumps({"error": "node_access_operation_failed", "type": type(error).__name__})) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml b/services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml index 2c40ed3e..44de7b61 100644 --- a/services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml +++ b/services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml @@ -1,15 +1,14 @@ # services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml -# One-off job for sso/metis-node-passwords-secret-ensure-4. +# One-off job for sso/metis-node-passwords-secret-ensure-5. # Purpose: ensure per-node Metis recovery placeholders exist in Vault. # Atlas/root values are preserved while intranet IPs are standardized per node. apiVersion: batch/v1 kind: Job metadata: - name: metis-node-passwords-secret-ensure-4 + name: metis-node-passwords-secret-ensure-5 namespace: sso spec: backoffLimit: 0 - ttlSecondsAfterFinished: 3600 template: spec: serviceAccountName: mas-secrets-ensure @@ -35,49 +34,53 @@ spec: args: - | set -eu - + umask 077 + work_dir="$(mktemp -d)" + trap 'rm -rf "${work_dir}"' EXIT HUP INT TERM vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}" vault_role="${VAULT_ROLE:-sso-secrets}" - - jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" - login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')" - vault_token="$(curl -sS --request POST --data "${login_payload}" "${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')" - if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then - echo "vault login failed" >&2 - exit 1 - fi - + jwt_file="${SERVICE_ACCOUNT_TOKEN_FILE:-/var/run/secrets/kubernetes.io/serviceaccount/token}" + jq -nc --rawfile jwt "${jwt_file}" --arg role "${vault_role}" \ + '{jwt:($jwt|rtrimstr("\n")),role:$role}' > "${work_dir}/login.json" + curl -fsS --max-time 30 --request POST \ + --data-binary "@${work_dir}/login.json" \ + "${vault_addr}/v1/auth/kubernetes/login" > "${work_dir}/auth.json" + jq -er '.auth.client_token | select(type == "string" and length > 0) | + "header = " + (("X-Vault-Token: " + .) | @json)' \ + "${work_dir}/auth.json" > "${work_dir}/auth.curl" ensured=0 while read -r node intranet_ip; do - if [ -z "${node}" ] || [ -z "${intranet_ip}" ]; then + [ -n "${node}" ] && [ -n "${intranet_ip}" ] || continue + secret_path="kv/data/atlas/nodes/${node}" + read_status="$(curl -sS --max-time 30 --config "${work_dir}/auth.curl" \ + -o "${work_dir}/read.json" -w '%{http_code}' \ + "${vault_addr}/v1/${secret_path}")" + case "${read_status}" in + 200) ;; + 404) printf '%s\n' '{"data":{"data":{},"metadata":{"version":0}}}' > "${work_dir}/read.json" ;; + *) echo "Vault read failed for ${node} (HTTP ${read_status})" >&2; exit 1 ;; + esac + # Preserve all existing fields. CAS prevents a concurrent password update + # from being overwritten; unchanged records do not create new versions. + jq -e --arg ip "${intranet_ip}" ' + .data as $existing | + {options:{cas:$existing.metadata.version}, + data:({atlas_password:"",root_password:""} + $existing.data + {intranet_ip:$ip})} + ' "${work_dir}/read.json" > "${work_dir}/write.json" + if jq -e --slurpfile update "${work_dir}/write.json" \ + '.data.data == $update[0].data' "${work_dir}/read.json" >/dev/null; then continue fi - - secret_path="kv/data/atlas/nodes/${node}" - read_status="$(curl -sS -o /tmp/node-read.json -w "%{http_code}" -H "X-Vault-Token: ${vault_token}" "${vault_addr}/v1/${secret_path}" || true)" - if [ "${read_status}" = "200" ]; then - atlas_password="$(jq -r '.data.data.atlas_password // empty' /tmp/node-read.json)" - root_password="$(jq -r '.data.data.root_password // empty' /tmp/node-read.json)" - elif [ "${read_status}" = "404" ]; then - atlas_password="" - root_password="" - else - echo "Vault read failed for ${node} (status ${read_status})" >&2 - cat /tmp/node-read.json >&2 || true - exit 1 - fi - - payload="$(jq -nc --arg atlas_password "${atlas_password}" --arg root_password "${root_password}" --arg intranet_ip "${intranet_ip}" '{data:{atlas_password:$atlas_password,root_password:$root_password,intranet_ip:$intranet_ip}}')" - - write_status="$(curl -sS -o /tmp/node-write.json -w "%{http_code}" -X POST -H "X-Vault-Token: ${vault_token}" -H 'Content-Type: application/json' -d "${payload}" "${vault_addr}/v1/${secret_path}")" - if [ "${write_status}" != "200" ] && [ "${write_status}" != "204" ]; then - echo "Vault write failed for ${node} (status ${write_status})" >&2 - cat /tmp/node-write.json >&2 || true - exit 1 - fi - + write_status="$(curl -sS --max-time 30 --config "${work_dir}/auth.curl" \ + -o "${work_dir}/result.json" -w '%{http_code}' --request POST \ + -H 'Content-Type: application/json' --data-binary "@${work_dir}/write.json" \ + "${vault_addr}/v1/${secret_path}")" + case "${write_status}" in + 200|204) ;; + *) echo "Vault update failed for ${node} (HTTP ${write_status}); no retry over concurrent writes" >&2; exit 1 ;; + esac ensured=$((ensured + 1)) - echo "Ensured node secret placeholder for ${node} (${intranet_ip})" + echo "Updated node metadata: ${node}" done <<'EOF_NODES' titan-jh 192.168.22.8 titan-db 192.168.22.10 @@ -87,7 +90,7 @@ spec: titan-20 192.168.22.20 titan-21 192.168.22.21 titan-22 192.168.22.22 - titan-23 192.168.22.23 + titan-23 192.168.22.24 titan-24 192.168.22.26 titan-04 192.168.22.30 titan-05 192.168.22.31 @@ -107,4 +110,4 @@ spec: titan-19 192.168.22.47 EOF_NODES - echo "Ensured ${ensured} Metis node placeholders in Vault" + echo "Updated ${ensured} Metis node records; unchanged records preserved" diff --git a/testing/tests/test_node_admin_access.py b/testing/tests/test_node_admin_access.py new file mode 100644 index 00000000..18812d68 --- /dev/null +++ b/testing/tests/test_node_admin_access.py @@ -0,0 +1,75 @@ +"""Node credential repair must be targeted, idempotent and silent about secrets.""" +import importlib.util +from pathlib import Path +import unittest +from unittest.mock import patch, Mock + +spec = importlib.util.spec_from_file_location( + "node_admin_access", Path(__file__).resolve().parents[2] / "scripts/node_admin_access.py") +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class NodeAccessTests(unittest.TestCase): + def setUp(self): + self.payload = {"hostname": "titan-test", "passwords": {"atlas": "synthetic-only"}} + self.root = patch.object(module.os, "geteuid", return_value=0) + self.host = patch.object(module.socket, "gethostname", return_value="titan-test") + self.account = patch.object(module.pwd, "getpwnam") + for item in [self.root, self.host, self.account]: + item.start() + self.addCleanup(item.stop) + + def test_wrong_host_never_changes_password(self): + with patch.object(module.subprocess, "run") as run: + self.payload["hostname"] = "wrong-node" + with self.assertRaisesRegex(ValueError, "hostname_mismatch"): + module.run(self.payload, True) + run.assert_not_called() + + def test_validate_all_accounts_before_mutation(self): + with patch.object(module.subprocess, "run") as run: + self.payload["passwords"]["other"] = "synthetic" + with self.assertRaisesRegex(ValueError, "account_not_allowed"): + module.run(self.payload, True) + run.assert_not_called() + + def test_password_record_injection_rejected(self): + self.payload["passwords"]["atlas"] = "bad\nroot:injected" + with self.assertRaisesRegex(ValueError, "invalid_password"): + module.run(self.payload, True) + + def test_audit_is_read_only(self): + with patch.object(module, "password_status", return_value={"vault_password_matches": False}), \ + patch.object(module.subprocess, "run") as run: + result = module.run(self.payload) + run.assert_not_called() + self.assertEqual(result["changed_accounts"], []) + + def test_matching_password_is_unchanged(self): + with patch.object(module, "password_status", return_value={"vault_password_matches": True}), \ + patch.object(module.subprocess, "run") as run: + module.run(self.payload, True) + run.assert_not_called() + + def test_restore_uses_stdin_and_returns_no_secret(self): + with patch.object(module, "password_status", side_effect=[ + {"vault_password_matches": False, "locked": True}, + {"vault_password_matches": True, "locked": False}]), \ + patch.object(module.subprocess, "run", return_value=Mock(returncode=0)) as run: + result = module.run(self.payload, True) + self.assertEqual(run.call_args.args[0], ["/usr/sbin/chpasswd"]) + self.assertEqual(run.call_args.kwargs["input"], "atlas:synthetic-only\n") + self.assertNotIn("synthetic-only", str(result)) + self.assertEqual(result["changed_accounts"], ["atlas"]) + self.assertTrue(result["after"]["atlas"]["vault_password_matches"]) + + def test_failed_update_does_not_echo_subprocess(self): + with patch.object(module, "password_status", return_value={"vault_password_matches": False}), \ + patch.object(module.subprocess, "run", return_value=Mock(returncode=1, stderr="SECRET")): + with self.assertRaisesRegex(ValueError, "^password_update_failed$"): + module.run(self.payload, True) + + +if __name__ == "__main__": + unittest.main() diff --git a/testing/tests/test_node_secret_bootstrap.py b/testing/tests/test_node_secret_bootstrap.py new file mode 100644 index 00000000..b8dbf531 --- /dev/null +++ b/testing/tests/test_node_secret_bootstrap.py @@ -0,0 +1,79 @@ +"""Exercise the real bootstrap shell with a local fake Vault transport.""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest +import yaml + +ROOT = Path(__file__).resolve().parents[2] +MANIFEST = ROOT / 'services/keycloak/bootstrap-jobs/metis-node-passwords-secret-ensure-job.yaml' +FAKE_CURL = r'''#!/usr/bin/env python3 +import json,os,pathlib,sys +args=sys.argv[1:] +state=pathlib.Path(os.environ['FAKE_STATE']) +s=json.loads(state.read_text()) +if args[-1].endswith('/login'): + print(json.dumps({'auth':{'client_token':'synthetic-token'}})); sys.exit(0) +node=args[-1].rsplit('/',1)[-1] +out=pathlib.Path(args[args.index('-o')+1]) +if '--data-binary' in args: + p=json.loads(pathlib.Path(args[args.index('--data-binary')+1][1:]).read_text()) + s['writes'].append(p) + status='400' if s.get('conflict') else '200' + out.write_text('UNSAFE BODY MUST NOT APPEAR IN LOGS') +else: + d=s['records'].get(node) + status='200' if d else '404' + out.write_text(json.dumps({'data':{'data':d,'metadata':{'version':7}}})) +state.write_text(json.dumps(s)); print(status,end='') +''' + + +class BootstrapTests(unittest.TestCase): + def exercise(self, records, conflict=False): + manifest = yaml.safe_load(MANIFEST.read_text()) + self.assertNotIn('ttlSecondsAfterFinished', manifest['spec']) + script = manifest['spec']['template']['spec']['containers'][0]['args'][0] + with tempfile.TemporaryDirectory() as d: + folder = Path(d) + fake = folder / 'curl' + fake.write_text(FAKE_CURL) + fake.chmod(0o755) + state = folder / 'state.json' + state.write_text(json.dumps({'records': records, 'writes': [], 'conflict': conflict})) + jwt = folder / 'jwt' + jwt.write_text('synthetic-jwt') + env = dict(os.environ, PATH=d + os.pathsep + os.environ['PATH'], + FAKE_STATE=str(state), SERVICE_ACCOUNT_TOKEN_FILE=str(jwt)) + result = subprocess.run(['/bin/sh', '-c', script], env=env, + capture_output=True, text=True, timeout=30) + self.assertNotIn('UNSAFE BODY', result.stdout + result.stderr) + self.assertNotIn('synthetic-password', result.stdout + result.stderr) + return result, json.loads(state.read_text()) + + def test_preserves_passwords_extra_fields_and_cas(self): + result, state = self.exercise({'titan-jh': {'atlas_password': 'synthetic-password', + 'root_password': 'synthetic-root', 'operator_note': 'keep'}}) + self.assertEqual(result.returncode, 0, result.stderr) + first = state['writes'][0] + self.assertEqual(first['options']['cas'], 7) + self.assertEqual(first['data']['atlas_password'], 'synthetic-password') + self.assertEqual(first['data']['operator_note'], 'keep') + self.assertEqual(state['writes'][1]['options']['cas'], 0) + + def test_unchanged_secret_has_no_new_version(self): + result, state = self.exercise({'titan-jh': {'atlas_password': 'synthetic-password', + 'root_password': 'synthetic-root', 'intranet_ip': '192.168.22.8'}}) + self.assertEqual(result.returncode, 0) + self.assertFalse(any(w['data']['intranet_ip'] == '192.168.22.8' for w in state['writes'])) + + def test_concurrent_write_fails_closed(self): + result, state = self.exercise({}, conflict=True) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(len(state['writes']), 1) + + +if __name__ == '__main__': + unittest.main()