atlas-iac/testing/tests/test_hermes_node_account_hardening.py
2026-08-17 02:00:36 -03:00

596 lines
22 KiB
Python

"""Contracts for the dedicated, unprivileged Hermes node SSH identity."""
from __future__ import annotations
import base64
import importlib.util
import os
import sys
from pathlib import Path
import pytest
import yaml
ROOT = Path(__file__).parents[2]
SCRIPT = ROOT / "services/hermes/scripts/node_account_hardening.py"
sys.path.insert(0, str(SCRIPT.parent))
def _key(material: bytes, comment: str = "") -> str:
key_type = b"ssh-ed25519"
blob = len(key_type).to_bytes(4, "big") + key_type + material
suffix = f" {comment}" if comment else ""
return "ssh-ed25519 " + base64.b64encode(blob).decode() + suffix
def _typed_key(key_type: str, material: bytes, comment: str = "") -> str:
encoded_type = key_type.encode()
blob = len(encoded_type).to_bytes(4, "big") + encoded_type + material
suffix = f" {comment}" if comment else ""
return f"{key_type} {base64.b64encode(blob).decode()}{suffix}"
def _load():
spec = importlib.util.spec_from_file_location("node_account_hardening_test", SCRIPT)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def _fixture(tmp_path: Path, monkeypatch):
module = _load()
host_etc = tmp_path / "etc"
host_home = tmp_path / "home"
host_etc.mkdir()
host_home.mkdir()
originals = {
"passwd": (
"root:x:0:0:root:/root:/bin/bash\n"
"atlas:x:2000:2000:Atlas:/home/atlas:/bin/bash\n"
"oceanus:x:2001:2001:Oceanus:/home/oceanus:/bin/bash\n"
),
"group": (
"root:x:0:\n"
"atlas:x:2000:\n"
"oceanus:x:2001:\n"
"disk:x:6:atlas\n"
"sudo:x:27:oceanus\n"
),
"shadow": (
"root:!:1:0:99999:7:::\n"
"atlas:!:1:0:99999:7:::\n"
"oceanus:!:1:0:99999:7:::\n"
),
"gshadow": (
"root:!::\n"
"atlas:!::\n"
"oceanus:!::\n"
"disk:!::atlas\n"
"sudo:!::oceanus\n"
),
}
for name, value in originals.items():
(host_etc / name).write_text(value, encoding="utf-8")
key = _key(b"synthetic-hermes-key")
other = _key(b"human-operator-key")
for user in ("atlas", "oceanus"):
ssh = host_home / user / ".ssh"
ssh.mkdir(parents=True)
(ssh / "authorized_keys").write_text(
f"{other} {user}\n{key}\n", encoding="utf-8"
)
public_key = tmp_path / "public-key"
public_key.write_text(key + "\n", encoding="utf-8")
monkeypatch.setattr(module, "HOST_ETC", host_etc)
monkeypatch.setattr(module, "HOST_HOME", host_home)
monkeypatch.setattr(module, "ACCOUNT_UID", os.getuid())
monkeypatch.setattr(module, "ACCOUNT_GID", os.getgid())
monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid())
monkeypatch.setattr(module, "HOST_POLKIT_SHARE", tmp_path / "missing-polkit")
# ACL behavior has its own executable test below. These account-database
# tests must not depend on the host running pytest with CAP_FOWNER.
monkeypatch.setattr(module, "_deny_sensitive_roots", lambda: None)
return module, originals, key, other, public_key
def test_reconciler_creates_locked_groupless_account_and_moves_only_exact_key(
tmp_path: Path, monkeypatch
):
module, originals, key, other, public_key = _fixture(tmp_path, monkeypatch)
module.reconcile(public_key)
first = {
name: (module.HOST_ETC / name).read_text(encoding="utf-8")
for name in originals
}
module.reconcile(public_key)
second = {
name: (module.HOST_ETC / name).read_text(encoding="utf-8")
for name in originals
}
assert first == second
expected = module._expected_records()
for name, original in originals.items():
assert first[name] == original + ":".join(expected[name]) + "\n"
backup = module.HOST_ETC / f"{name}.hermes-boundary-backup"
assert backup.read_text(encoding="utf-8") == original
assert expected["shadow"][1] == "!"
assert expected["group"][-1] == ""
assert "disk:x:6:atlas\n" in first["group"]
assert "sudo:x:27:oceanus\n" in first["group"]
hermes_keys = (
module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys"
).read_text(encoding="utf-8")
assert hermes_keys == key + "\n"
for user in module.LEGACY_ACCOUNTS:
legacy = (
module.HOST_HOME / user / ".ssh/authorized_keys"
).read_text(encoding="utf-8")
assert legacy == f"{other} {user}\n"
backup = module.HOST_HOME / user / ".ssh/authorized_keys.hermes-boundary-backup"
assert backup.read_text(encoding="utf-8") == f"{other} {user}\n{key}\n"
def test_identity_conflict_fails_closed_without_editing_account_databases(
tmp_path: Path, monkeypatch
):
module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch)
conflict = originals["passwd"] + (
f"unrelated:x:{module.ACCOUNT_UID}:{module.ACCOUNT_GID}:Other:/home/other:/bin/bash\n"
)
(module.HOST_ETC / "passwd").write_text(conflict, encoding="utf-8")
with pytest.raises(module.HardeningError, match="identity conflicts"):
module.reconcile(public_key)
assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == conflict
for name in ("group", "shadow", "gshadow"):
assert (module.HOST_ETC / name).read_text(encoding="utf-8") == originals[name]
assert not (module.HOST_HOME / module.ACCOUNT).exists()
def test_malformed_account_file_fails_closed(tmp_path: Path, monkeypatch):
module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch)
(module.HOST_ETC / "group").write_text("malformed\n", encoding="utf-8")
with pytest.raises(module.HardeningError, match="invalid record"):
module.reconcile(public_key)
assert (module.HOST_ETC / "passwd").read_text(encoding="utf-8") == originals[
"passwd"
]
assert (module.HOST_ETC / "group").read_text(encoding="utf-8") == "malformed\n"
def test_preexisting_home_fails_before_account_database_or_key_changes(
tmp_path: Path, monkeypatch
):
module, originals, _key, _other, public_key = _fixture(tmp_path, monkeypatch)
home = module.HOST_HOME / module.ACCOUNT
home.mkdir()
(home / "unrelated").write_text("preserve\n", encoding="utf-8")
with pytest.raises(module.HardeningError, match="home already exists"):
module.reconcile(public_key)
for name, original in originals.items():
assert (module.HOST_ETC / name).read_text(encoding="utf-8") == original
assert not (module.HOST_ETC / f"{name}.hermes-boundary-backup").exists()
assert (home / "unrelated").read_text(encoding="utf-8") == "preserve\n"
def test_flux_daemonset_reconciles_every_node_without_mutating_human_groups():
documents = list(
yaml.safe_load_all(
(ROOT / "services/hermes/node-ssh-access.yaml").read_text(encoding="utf-8")
)
)
daemonset = next(item for item in documents if item["kind"] == "DaemonSet")
spec = daemonset["spec"]["template"]["spec"]
assert spec["tolerations"] == [{"operator": "Exists"}]
command = spec["containers"][0]["args"][0]
assert "/opt/node-hardener/node_account_hardening.py" in command
assert "sleep 300" in command
mounts = {item["name"]: item for item in spec["volumes"]}
assert mounts["host-home"]["hostPath"]["path"] == "/home"
assert mounts["host-etc"]["hostPath"]["path"] == "/etc"
assert mounts["host-k3s"]["hostPath"]["path"] == "/var/lib/rancher/k3s"
assert mounts["host-kubelet"]["hostPath"]["path"] == "/var/lib/kubelet"
assert mounts["host-run-k3s"]["hostPath"]["path"] == "/run/k3s"
assert mounts["host-run-containerd"]["hostPath"]["path"] == "/run/containerd"
assert "usermod" not in command
assert "groupmod" not in command
policies = list(
yaml.safe_load_all(
(ROOT / "services/hermes/networkpolicy.yaml").read_text(encoding="utf-8")
)
)
isolation = next(
item
for item in policies
if item["metadata"]["name"] == "hermes-node-ssh-access-isolation"
)
assert isolation["spec"]["ingress"] == []
assert isolation["spec"]["egress"] == []
source = SCRIPT.read_text(encoding="utf-8")
assert "ACCOUNT = \"hermes-agent\"" in source
assert "ACCOUNT_UID = 1200" in source
assert "ACCOUNT_GID = 1200" in source
assert 'LEGACY_ACCOUNTS = ("atlas", "oceanus")' in source
assert "audit_membership" in source
image = spec["containers"][0]["image"]
assert image == (
"python@sha256:6d43704baacd1bfbe7c295d7f13079d5d8104ed33568873133f8fc69980419df"
)
assert mounts["host-polkit-share"]["hostPath"]["path"] == "/usr/share/polkit-1"
def test_sensitive_roots_get_explicit_zero_permission_acl_for_hermes(
tmp_path: Path, monkeypatch
):
module = _load()
sensitive = tmp_path / "k3s"
sensitive.mkdir(mode=0o755)
host_etc = tmp_path / "etc"
host_etc.mkdir()
monkeypatch.setattr(module, "ACCOUNT_UID", 1200)
monkeypatch.setattr(module, "HOST_ETC", host_etc)
monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid())
monkeypatch.setattr(module, "HOST_ROOT_GID", os.getgid())
# Production requires uid 0. The test process owns its fixture but runs the
# real xattr/ACL implementation using the test owner's uid as that boundary.
module._deny_sensitive_root(sensitive)
first = module._read_acl(sensitive)
module._deny_sensitive_root(sensitive)
second = module._read_acl(sensitive)
assert first == second
entries = module._decode_acl(first, 0o755)
assert (module.ACL_USER, 0, 1200) in entries
assert (module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID) in entries
assert (module.ACL_GROUP_OBJ, 0o5, module.ACL_UNDEFINED_ID) in entries
assert (module.ACL_OTHER, 0o5, module.ACL_UNDEFINED_ID) in entries
backup = host_etc / "hermes-node-boundary/k3s.acl"
assert backup.read_bytes() == b"N"
assert backup.stat().st_mode & 0o777 == 0o600
def test_sensitive_root_rejects_non_directory_and_non_root_owner(
tmp_path: Path, monkeypatch
):
module = _load()
regular = tmp_path / "not-a-directory"
regular.write_text("preserve", encoding="utf-8")
with pytest.raises(module.HardeningError, match="unsafe sensitive directory"):
module._deny_sensitive_root(regular)
directory = tmp_path / "not-root-owned"
directory.mkdir()
if directory.stat().st_uid == 0:
pytest.skip("cannot construct a non-root-owned fixture as root")
with pytest.raises(module.HardeningError, match="not root-owned"):
module._deny_sensitive_root(directory)
def test_acl_parser_rejects_malformed_or_incomplete_values():
module = _load()
with pytest.raises(module.HardeningError, match="malformed"):
module._decode_acl(b"too short", 0o755)
incomplete = module._encode_acl(
[(module.ACL_USER_OBJ, 0o7, module.ACL_UNDEFINED_ID)]
)
with pytest.raises(module.HardeningError, match="incomplete"):
module._decode_acl(incomplete, 0o755)
def test_acl_failure_stops_before_installing_the_ssh_key(tmp_path: Path, monkeypatch):
module = _load()
host_etc = tmp_path / "etc"
host_etc.mkdir()
monkeypatch.setattr(module, "HOST_ETC", host_etc)
monkeypatch.setattr(module, "HOST_ROOT_UID", os.getuid())
public_key = tmp_path / "public-key"
public_key.write_text(
_key(b"synthetic-key") + "\n",
encoding="utf-8",
)
moved = False
monkeypatch.setattr(module, "_reconcile_databases", lambda: None)
def fail_acl():
raise module.HardeningError("ACL unavailable")
def move_key(_path):
nonlocal moved
moved = True
monkeypatch.setattr(module, "_deny_sensitive_roots", fail_acl)
monkeypatch.setattr(module, "_move_key", move_key)
with pytest.raises(module.HardeningError, match="ACL unavailable"):
module.reconcile(public_key)
assert moved is False
def test_runtime_ssh_config_forces_dedicated_account_for_every_titan():
stage = (
ROOT / "services/hermes/scripts/stage_runtime_access.py"
).read_text(encoding="utf-8")
assert '"Host titan-*\\n User hermes-agent\\n"' in stage
agent = yaml.safe_load(
(ROOT / "services/hermes/agent-deployment.yaml").read_text(encoding="utf-8")
)
assert "User atlas" not in yaml.safe_dump(agent)
assert "User oceanus" not in yaml.safe_dump(agent)
@pytest.mark.parametrize("database,record", [
("group", "disk:x:6:atlas,hermes-agent\n"),
("gshadow", "disk:!:hermes-agent:atlas\n"),
("gshadow", "disk:!::atlas,hermes-agent\n"),
])
def test_unexpected_group_or_gshadow_membership_fails_before_writes(
tmp_path: Path, monkeypatch, database: str, record: str
):
module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
path = module.HOST_ETC / database
path.write_text(
"\n".join(line for line in originals[database].splitlines() if not line.startswith("disk:"))
+ "\n"
+ record,
encoding="utf-8",
)
before = {name: (module.HOST_ETC / name).read_bytes() for name in originals}
with pytest.raises(module.HardeningError, match="group access"):
module.reconcile(public_key)
assert {name: (module.HOST_ETC / name).read_bytes() for name in originals} == before
@pytest.mark.parametrize(
"relative,value",
[
("sudoers.d/hermes", "hermes-agent ALL=(ALL:ALL) NOPASSWD: ALL\n"),
("sudoers", "ALL ALL=(ALL:ALL) ALL\n"),
(
"polkit-1/rules.d/90-root.rules",
'if (subject.user == "hermes-agent") return polkit.Result.YES;\n',
),
(
"polkit-1/rules.d/90-root.rules",
'unix-user:* org.freedesktop.policykit.exec polkit.Result.YES\n',
),
],
)
def test_sudo_and_root_equivalent_polkit_authority_fail_closed(
tmp_path: Path, monkeypatch, relative: str, value: str
):
module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
policy = module.HOST_ETC / relative
policy.parent.mkdir(parents=True, exist_ok=True)
policy.write_text(value, encoding="utf-8")
with pytest.raises(module.HardeningError, match="sudo|polkit"):
module.reconcile(public_key)
assert all((module.HOST_ETC / name).read_text() == original for name, original in originals.items())
def test_external_group_authority_source_is_rejected(tmp_path: Path, monkeypatch):
module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
(module.HOST_ETC / "nsswitch.conf").write_text("passwd: files\ngroup: files ldap\n")
with pytest.raises(module.HardeningError, match="external group"):
module.reconcile(public_key)
assert all((module.HOST_ETC / name).read_text() == original for name, original in originals.items())
def test_writable_privilege_policy_is_rejected(tmp_path: Path, monkeypatch):
module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
sudoers = module.HOST_ETC / "sudoers"
sudoers.write_text("root ALL=(ALL:ALL) ALL\n")
sudoers.chmod(0o666)
with pytest.raises(module.HardeningError, match="unsafe mutation"):
module.reconcile(public_key)
assert all((module.HOST_ETC / name).read_text() == original for name, original in originals.items())
def test_key_identity_ignores_options_and_comments_and_removes_legacy_first(
tmp_path: Path, monkeypatch
):
module, _originals, key, other, public_key = _fixture(tmp_path, monkeypatch)
key_fields = key.split()
decorated = f'restrict,command="echo denied" {key_fields[0]} {key_fields[1]} old-comment'
for user in module.LEGACY_ACCOUNTS:
path = module.HOST_HOME / user / ".ssh/authorized_keys"
path.write_text(f"{other} {user}\n{decorated}\n", encoding="utf-8")
writes: list[Path] = []
real_write = module.atomic_write
def recording_write(path, value, metadata):
writes.append(path)
real_write(path, value, metadata)
monkeypatch.setattr(module, "atomic_write", recording_write)
module.reconcile(public_key)
key_writes = [path for path in writes if path.name == "authorized_keys"]
assert key_writes[-1].parts[-3:] == (module.ACCOUNT, ".ssh", "authorized_keys")
assert all(module.ACCOUNT not in path.parts for path in key_writes[:-1])
def test_unrelated_modern_human_key_type_is_preserved(tmp_path: Path, monkeypatch):
module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch)
modern = _typed_key("sk-ssh-ed25519@openssh.com", b"human-security-key", "human")
for user in module.LEGACY_ACCOUNTS:
path = module.HOST_HOME / user / ".ssh/authorized_keys"
path.write_text(f"{modern}\n{key}\n", encoding="utf-8")
module.reconcile(public_key)
for user in module.LEGACY_ACCOUNTS:
path = module.HOST_HOME / user / ".ssh/authorized_keys"
assert path.read_text() == modern + "\n"
def test_crash_during_legacy_key_removal_never_installs_duplicate_key(
tmp_path: Path, monkeypatch
):
module, _originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
real_write = module.atomic_write
key_writes = 0
def crash_on_second_key(path, value, metadata):
nonlocal key_writes
if path.name == "authorized_keys":
key_writes += 1
if key_writes == 2:
raise module.HardeningError("synthetic crash")
real_write(path, value, metadata)
monkeypatch.setattr(module, "atomic_write", crash_on_second_key)
with pytest.raises(module.HardeningError, match="synthetic crash"):
module.reconcile(public_key)
assert not (module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys").exists()
def test_preexisting_target_key_is_removed_before_legacy_cleanup_and_reinstalled(
tmp_path: Path, monkeypatch
):
module, _originals, key, _other, public_key = _fixture(tmp_path, monkeypatch)
target = module.HOST_HOME / module.ACCOUNT / ".ssh/authorized_keys"
target.parent.mkdir(parents=True)
target.write_text(key + " existing\n")
writes: list[Path] = []
real_write = module.atomic_write
def recording_write(path, value, metadata):
if path.name == "authorized_keys":
writes.append(path)
real_write(path, value, metadata)
monkeypatch.setattr(module, "atomic_write", recording_write)
module._move_key(public_key)
assert writes[0] == target
assert writes[-1] == target
assert all(path != target for path in writes[1:-1])
assert target.read_text() == key + "\n"
def test_account_database_write_preserves_mode_owner_and_xattrs(
tmp_path: Path, monkeypatch
):
module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
passwd = module.HOST_ETC / "passwd"
passwd.chmod(0o640)
os.setxattr(passwd, "user.hermes-test", b"preserve")
before = passwd.stat()
module.reconcile(public_key)
after = passwd.stat()
assert after.st_mode & 0o777 == 0o640
assert (after.st_uid, after.st_gid) == (before.st_uid, before.st_gid)
assert os.getxattr(passwd, "user.hermes-test") == b"preserve"
backup = module.HOST_ETC / "passwd.hermes-boundary-backup"
assert os.getxattr(backup, "user.hermes-test") == b"preserve"
assert backup.read_text() == originals["passwd"]
def test_atomic_write_restores_acl_and_security_label_xattrs(tmp_path: Path, monkeypatch):
module = _load()
io_module = sys.modules[module.atomic_write.__module__]
target = tmp_path / "passwd"
target.write_bytes(b"old\n")
metadata = io_module.FileSnapshot(
value=b"old\n",
device=target.stat().st_dev,
inode=target.stat().st_ino,
mode=0o600,
uid=os.getuid(),
gid=os.getgid(),
size=4,
mtime_ns=target.stat().st_mtime_ns,
ctime_ns=target.stat().st_ctime_ns,
xattrs=(
("system.posix_acl_access", b"synthetic-acl"),
("security.selinux", b"synthetic-label"),
("user.audit", b"synthetic-xattr"),
),
)
restored: list[tuple[str, bytes]] = []
monkeypatch.setattr(
io_module.os,
"setxattr",
lambda _fd, name, value: restored.append((name, value)),
)
io_module.atomic_write(target, b"new\n", metadata)
assert target.read_bytes() == b"new\n"
assert restored == list(metadata.xattrs)
def test_concurrent_database_change_is_detected_before_any_account_write(
tmp_path: Path, monkeypatch
):
module, originals, _key_value, _other, public_key = _fixture(tmp_path, monkeypatch)
real_backup = module.backup_once
backup_count = 0
writes: list[Path] = []
def racing_backup(path, snapshot, maximum):
nonlocal backup_count
result = real_backup(path, snapshot, maximum)
backup_count += 1
if backup_count == 4:
group = module.HOST_ETC / "group"
group.write_text(originals["group"] + "race:x:4000:\n")
return result
monkeypatch.setattr(module, "backup_once", racing_backup)
monkeypatch.setattr(module, "atomic_write", lambda path, *_args: writes.append(path))
with pytest.raises(module.HardeningError, match="concurrent host account change"):
module.reconcile(public_key)
assert writes == []
def test_flux_orders_observer_rbac_before_hermes_prunes_old_authority():
observer = yaml.safe_load(
(ROOT / "clusters/atlas/flux-system/applications/hermes-observer-rbac/kustomization.yaml").read_text()
)
hermes = yaml.safe_load(
(ROOT / "clusters/atlas/flux-system/applications/hermes/kustomization.yaml").read_text()
)
assert "dependsOn" not in observer["spec"]
assert {item["name"] for item in hermes["spec"]["dependsOn"]} >= {
"hermes-observer-rbac",
"hermes-scm-broker",
}
def test_sensitive_root_acl_backups_have_unique_host_path_names(monkeypatch):
module = _load()
seen = []
monkeypatch.setattr(
module,
"_deny_sensitive_root",
lambda path, backup_name=None: seen.append((path, backup_name)),
)
module._deny_sensitive_roots()
assert [name for _path, name in seen] == [
"var-lib-rancher-k3s",
"var-lib-kubelet",
"run-k3s",
"run-containerd",
]
assert len({name for _path, name in seen}) == 4