diff --git a/docs/CLUSTER_OPERATIONS.md b/docs/CLUSTER_OPERATIONS.md index 0ef44b1b..f223d8ae 100644 --- a/docs/CLUSTER_OPERATIONS.md +++ b/docs/CLUSTER_OPERATIONS.md @@ -92,6 +92,15 @@ with Ananke, and retains a root-only backup. The coordinator remains Titan-db; Titan-24 remains its peer. Credentials go to sudo stdin for predefined actions, not arbitrary command arguments or logs. No new controller was introduced. +The peer's public key must also be present in the worker's atlas authorized_keys. +It is recorded in `kv/atlas/maintenance/metis-ssh-keys`, field `ananke_tethys_pub`. +`scripts/install_ananke_peer_key.py` accepts a hostname and verified public key on +stdin, preserves existing keys, and restricts new entries to Titan-24's source +address 192.168.22.26. It grants no sudo permissions. Verify the public key through +an already authenticated connection; never replace trusted host keys just to +make SSH connect. On October 4, both the coordinator and peer passed actual +privileged read-only worker checks after this repair. + This automated worker-password path requires the Kubernetes API. It is not a standalone cold-start credential store. A read-only access check is not a power failure/recovery drill. When onboarding another recovered worker, first validate diff --git a/docs/CLUSTER_STABILIZATION.md b/docs/CLUSTER_STABILIZATION.md index 43598b51..9ec6dd40 100644 --- a/docs/CLUSTER_STABILIZATION.md +++ b/docs/CLUSTER_STABILIZATION.md @@ -7,7 +7,7 @@ Use [Cluster operations](CLUSTER_OPERATIONS.md) for the ordinary operator path. ## Six-priority progress tracker -Started against the user's approved list on 2026-10-03. Updated: 2026-10-04 05:55 UTC. +Started against the user's approved list on 2026-10-03. Updated: 2026-10-04 06:10 UTC. "Verified" means the stated check passed; it does not imply a completed soak or failure drill. Physical repairs and disruptive recovery drills remain separate from routine software work. @@ -80,10 +80,16 @@ replacement. No node was reflashed or rebooted during this work. template and corrects Titan-24's administrator to tethys. Both live host configs were patched without changing unrelated settings and passed Ananke validation. - [x] Verify read-only privileged actions using Ananke's own coordinator SSH - identity on all 13 affected workers. The peer's independent SSH check on - 12/13/19 failed before sudo; that identity/known-host path is being diagnosed. Both - Ananke services were restarted separately and returned active/running with - zero automatic restarts. Jenkins credential-helper generation remained 796. + identity on all 13 affected workers. Both Ananke services were restarted + separately and returned active/running with zero automatic restarts. Jenkins + credential-helper generation remained 796. +- [x] Restore the peer's missing SSH public key on 04/07/08/11/12/13/19/20/21. + The key was derived on the authenticated Titan-24 host and matched the existing + Vault recovery public key. New entries restrict their source to 192.168.22.26; + existing administrator keys and SSH policies were preserved. The peer's own + identity then passed password-backed sudo on all 13 workers, plus its existing + privileged path on 0a/0b/0c/db/22. These are actual remote systemctl version + checks, not just successful key installation. - [ ] Resolve the remaining network/physical findings below. ### Current node distinctions @@ -104,8 +110,9 @@ root accounts (0a/0b/0c/04/07/08/11/db/jh); their administrator account plus sud still provides full root control. Root-account lock state is recorded explicitly in Vault instead of implying that every root_password field is a usable login. -Validation: 16 Atlas regression tests passed (credential targeting, secret-safe -output, legacy-grant preservation, Vault no-op/CAS behavior, and preservation/idempotency of the Ananke config patch); relevant Kustomize +Validation: 19 Atlas regression tests passed (credential targeting, secret-safe +output, legacy-grant preservation, Vault no-op/CAS behavior, preservation/idempotency +of the Ananke config patch, and restricted peer-key enrollment); relevant Kustomize builds, client dry-runs and focused Flux diffs passed. Metis's complete Go tests and separate docs/LOC/vet/per-file coverage gate passed. Tests include a real throwaway ext4 image for systemd link injection and mocked password-setup failure, @@ -131,6 +138,9 @@ Native Ananke config backups are root-only under its lookup settings would remove automated password-backed access; retain the working administrator passwords. Its source profiles and the versioned Atlas configuration helper document the change. +Peer key enrollment uses `scripts/install_ananke_peer_key.py`. Rollback removes +only the newly appended `ananke-tethys-recovery` line after checking another +administrator session still works; do not replace the whole authorized_keys file. ## Verified changes diff --git a/scripts/install_ananke_peer_key.py b/scripts/install_ananke_peer_key.py new file mode 100755 index 00000000..f8272dc7 --- /dev/null +++ b/scripts/install_ananke_peer_key.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Install a verified Ananke peer public key for atlas, restricted to Titan-24. + +Read {"hostname":"titan-12", "public_key":"ssh-ed25519 ..."} from stdin. +Obtain the key from the authenticated peer, never from an unverified key scan. +""" +import base64 +import json +import os +from pathlib import Path +import pwd +import socket +import sys + + +def install(payload): + """Append one public key without replacing existing administrator keys.""" + if os.geteuid() != 0 or payload["hostname"] != socket.gethostname().split(".")[0]: + raise ValueError("root_and_matching_hostname_required") + parts = payload["public_key"].split() + if len(parts) < 2 or parts[0] != "ssh-ed25519": + raise ValueError("ed25519_public_key_required") + blob = base64.b64decode(parts[1], validate=True) + if len(blob) != 51 or not blob.startswith(b"\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20"): + raise ValueError("invalid_public_key") + account = pwd.getpwnam("atlas") + directory = Path(account.pw_dir) / ".ssh" + target = directory / "authorized_keys" + if directory.is_symlink() or target.is_symlink(): + raise ValueError("symlink_requires_review") + directory.mkdir(exist_ok=True, mode=0o700) + directory.chmod(0o700) + os.chown(directory, account.pw_uid, account.pw_gid) + old = target.read_text() if target.exists() else "" + key = " ".join(parts[:2]) + if any(key in line for line in old.splitlines() if not line.lstrip().startswith("#")): + return {"hostname": payload["hostname"], "key_already_present": True} + prefix = "\n" if old and not old.endswith("\n") else "" + entry = prefix + 'from="192.168.22.26",restrict ' + key + " ananke-tethys-recovery\n" + fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_APPEND | os.O_NOFOLLOW, 0o600) + with os.fdopen(fd, "w") as stream: + os.fchmod(stream.fileno(), 0o600) + os.fchown(stream.fileno(), account.pw_uid, account.pw_gid) + stream.write(entry) + return {"hostname": payload["hostname"], "key_added": True, "source_ip": "192.168.22.26"} + + +if __name__ == "__main__": + print(json.dumps(install(json.load(sys.stdin)))) diff --git a/testing/tests/test_ananke_peer_key.py b/testing/tests/test_ananke_peer_key.py new file mode 100644 index 00000000..7833bf40 --- /dev/null +++ b/testing/tests/test_ananke_peer_key.py @@ -0,0 +1,52 @@ +"""Peer SSH enrollment preserves administrator keys and restricts the source.""" +import base64 +import importlib.util +import os +from pathlib import Path +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('peer_key', + Path(__file__).resolve().parents[2] / 'scripts/install_ananke_peer_key.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class PeerKeyTests(unittest.TestCase): + def test_append_is_restricted_and_idempotent(self): + blob = b'\x00\x00\x00\x0bssh-ed25519\x00\x00\x00\x20' + b'B' * 32 + payload = {'hostname': 'titan-test', 'public_key': 'ssh-ed25519 ' + base64.b64encode(blob).decode()} + with tempfile.TemporaryDirectory() as directory: + home = Path(directory) + (home / '.ssh').mkdir() + keys = home / '.ssh/authorized_keys' + keys.write_text('# existing administrator key remains\nssh-ed25519 existing-admin\n') + account = SimpleNamespace(pw_dir=directory, pw_uid=os.getuid(), pw_gid=os.getgid()) + with patch.object(module.os, 'geteuid', return_value=0), \ + patch.object(module.socket, 'gethostname', return_value='titan-test'), \ + patch.object(module.pwd, 'getpwnam', return_value=account): + self.assertTrue(module.install(payload)['key_added']) + self.assertTrue(module.install(payload)['key_already_present']) + content = keys.read_text() + self.assertIn('existing-admin', content) + self.assertEqual(content.count(payload['public_key']), 1) + self.assertIn('from="192.168.22.26",restrict ', content) + self.assertEqual(keys.stat().st_mode & 0o777, 0o600) + + def test_wrong_node_is_rejected_before_account_lookup(self): + with patch.object(module.os, 'geteuid', return_value=0), \ + patch.object(module.socket, 'gethostname', return_value='different-node'), \ + patch.object(module.pwd, 'getpwnam') as lookup: + with self.assertRaisesRegex(ValueError, 'root_and_matching_hostname_required'): + module.install({'hostname': 'titan-test'}) + lookup.assert_not_called() + + def test_invalid_key_does_not_touch_authorized_keys(self): + with patch.object(module.os, 'geteuid', return_value=0), \ + patch.object(module.socket, 'gethostname', return_value='titan-test'), \ + patch.object(module.pwd, 'getpwnam') as lookup: + with self.assertRaises(ValueError): + module.install({'hostname': 'titan-test', 'public_key': 'ssh-ed25519 invalid'}) + lookup.assert_not_called()