516 lines
19 KiB
Python
516 lines
19 KiB
Python
"""Independent build, Harbor evidence, and Flux handoff for Hermes WebUI."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import sys
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
PIPELINE = ROOT / "ci/Jenkinsfile.hermes-webui-image"
|
|
RELEASE = ROOT / "ci/scripts/hermes_webui_release.py"
|
|
DOCKERFILE = ROOT / "dockerfiles/Dockerfile.hermes-webui"
|
|
CHAT = ROOT / "services/hermes/chat-statefulset.yaml"
|
|
DASHBOARD = ROOT / "services/hermes/deployment.yaml"
|
|
POLICY = ROOT / "services/harbor/scripts/harbor_hermes_webui_immutability_ensure.py"
|
|
sys.path.insert(0, str(RELEASE.parent))
|
|
|
|
|
|
def _load(path: Path, name: str):
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
assert spec is not None and spec.loader is not None
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _pod_spec() -> dict:
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
pod_yaml = source.split('yaml """', 1)[1].split('"""', 1)[0]
|
|
return yaml.safe_load(pod_yaml)["spec"]
|
|
|
|
|
|
def _release_fixture(tmp_path: Path):
|
|
module = _load(RELEASE, f"hermes_webui_release_{tmp_path.name}")
|
|
digest = "sha256:" + "7" * 64
|
|
revision = "8" * 40
|
|
build = "23"
|
|
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-{build}"
|
|
digest_file = tmp_path / "hermes-webui.digest"
|
|
image_file = tmp_path / "hermes-webui.image"
|
|
digest_file.write_text(digest + "\n", encoding="utf-8")
|
|
image_file.write_text(f"{destination}@{digest}\n", encoding="utf-8")
|
|
output = tmp_path / "release"
|
|
kwargs = {
|
|
"digest_file": digest_file,
|
|
"image_file": image_file,
|
|
"source_revision": revision,
|
|
"build_number": build,
|
|
"destination": destination,
|
|
"chat_manifest": CHAT,
|
|
"dashboard_manifest": DASHBOARD,
|
|
"output_dir": output,
|
|
}
|
|
module.write_release_artifacts(
|
|
digest=digest,
|
|
source_revision=revision,
|
|
build_number=build,
|
|
destination=destination,
|
|
chat_manifest=CHAT,
|
|
dashboard_manifest=DASHBOARD,
|
|
output_dir=output,
|
|
)
|
|
return module, digest, kwargs
|
|
|
|
|
|
def test_webui_job_is_independent_bounded_and_main_only() -> None:
|
|
"""WebUI has its own bounded job using the same Vault-injected release token."""
|
|
config = yaml.safe_load(
|
|
(ROOT / "services/jenkins/configmap-jcasc.yaml").read_text(encoding="utf-8")
|
|
)
|
|
jobs = config["data"]["jobs.yaml"]
|
|
assert jobs.count("pipelineJob('hermes-agent-image')") == 1
|
|
assert jobs.count("pipelineJob('hermes-webui-image')") == 1
|
|
block = jobs.split("pipelineJob('hermes-webui-image')", 1)[1].split(
|
|
"multibranchPipelineJob(", 1
|
|
)[0]
|
|
assert "branches('*/main')" in block
|
|
assert "scriptPath('ci/Jenkinsfile.hermes-webui-image')" in block
|
|
assert "pipelineTriggers" not in block
|
|
assert (
|
|
"authenticationToken(System.getenv('HERMES_AGENT_IMAGE_BUILD_TOKEN'))" in block
|
|
)
|
|
assert "booleanParam('PUBLISH_IMAGE', false" in block
|
|
assert "stringParam('EXPECTED_SOURCE_REVISION', ''" in block
|
|
assert "stringParam('CONFIRM_PUBLISH', ''" in block
|
|
|
|
|
|
def test_pipeline_builds_latest_main_containing_reviewed_anchor() -> None:
|
|
"""Publish is explicit, evidence-bound, and handed only to Flux."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
assert 'test "${CONFIRM_PUBLISH}" = "PUBLISH HERMES WEBUI"' in source
|
|
assert 'test "${actual_revision}" = "$(git rev-parse origin/main)"' in source
|
|
assert "git fetch --no-tags origin main" not in source
|
|
assert 'git merge-base --is-ancestor "${EXPECTED_SOURCE_REVISION}"' in source
|
|
assert "dockerfiles/Dockerfile.hermes-webui" in source
|
|
assert "ci/scripts/hermes_webui_release.py" in source
|
|
assert "registry.bstein.dev/bstein/hermes-webui" in source
|
|
assert "Dockerfile.hermes-agent" not in source
|
|
assert "hermes_image_release.py" not in source
|
|
assert "HERMES_KANIKO_HEREDOC_COMPAT" not in source
|
|
assert "--digest-file=" in source
|
|
assert "--image-name-tag-with-digest-file=" in source
|
|
assert "org.opencontainers.image.revision=${source_revision}" in source
|
|
assert "assert-absent" in source and "verify-evidence" in source
|
|
assert "ci/scripts/hermes_oci_promote.py" in source
|
|
assert "test_hermes_webui_brand.py" in source
|
|
assert "test_hermes_webui_release.py" in source
|
|
for forbidden in ("kubectl ", "flux reconcile", "git push", "git commit"):
|
|
assert forbidden not in source
|
|
|
|
spec = _pod_spec()
|
|
assert spec["serviceAccountName"] == "hermes-image-builder"
|
|
assert spec["automountServiceAccountToken"] is False
|
|
assert spec["enableServiceLinks"] is False
|
|
containers = {item["name"]: item for item in spec["containers"]}
|
|
assert "kaniko" in containers
|
|
assert "docker.sock" not in source and "hostPath" not in source
|
|
for container in containers.values():
|
|
assert container["securityContext"]["allowPrivilegeEscalation"] is False
|
|
assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]
|
|
|
|
|
|
def test_webui_dockerfile_is_kaniko_safe_and_uses_reviewed_repo_source() -> None:
|
|
"""The dedicated build consumes tracked patches/assets without RUN heredocs."""
|
|
source = DOCKERFILE.read_text(encoding="utf-8")
|
|
assert "<<" not in source
|
|
assert "hermes-webui-base-patch.py" in source
|
|
assert "hermes-webui-brand-patch.py" in source
|
|
assert "hermes-webui-manifest-patch.py" in source
|
|
assert "hermes-webui-smoke.py" in source
|
|
assert "hermes-webui-stt-patch.py" in source
|
|
assert "hermes-webui-atlas-voice.js" in source
|
|
assert "hermes-webui-manifest.json" in source
|
|
assert source.count("RUN /opt/hermes/.venv/bin/python /tmp/hermes-webui-") == 1
|
|
assert "urljoin(manifest_url, source)" in (
|
|
ROOT / "dockerfiles/hermes-webui-smoke.py"
|
|
).read_text(encoding="utf-8")
|
|
|
|
|
|
def test_renderer_updates_exact_chat_and_dashboard_webui_only(tmp_path: Path) -> None:
|
|
"""One digest patch spans the two same-policy WebUI consumers and nothing else."""
|
|
module, digest, kwargs = _release_fixture(tmp_path)
|
|
output = kwargs["output_dir"]
|
|
patch = (output / "hermes-webui-image-update.patch").read_text(encoding="utf-8")
|
|
assert patch.count(f"+ image: {module.DEFAULT_IMAGE}@{digest}") == 2
|
|
assert "services/hermes/chat-statefulset.yaml" in patch
|
|
assert "services/hermes/deployment.yaml" in patch
|
|
assert "hermes-agent@sha256" not in "\n".join(
|
|
line for line in patch.splitlines() if line.startswith("+")
|
|
)
|
|
assert CHAT.read_text(encoding="utf-8") != (
|
|
output / "hermes-chat-statefulset.yaml"
|
|
).read_text(encoding="utf-8")
|
|
assert DASHBOARD.read_text(encoding="utf-8") != (
|
|
output / "hermes-dashboard-deployment.yaml"
|
|
).read_text(encoding="utf-8")
|
|
metadata = json.loads((output / "hermes-webui-image.json").read_text())
|
|
assert metadata["digest"] == digest
|
|
assert metadata["flux_image"] == f"{module.DEFAULT_IMAGE}@{digest}"
|
|
assert metadata["flux_targets"] == [
|
|
"apps/StatefulSet/hermes/hermes-chat-tenant",
|
|
"apps/Deployment/hermes/hermes",
|
|
]
|
|
module.validate_release_artifacts(**kwargs)
|
|
|
|
|
|
def test_renderer_accepts_flux_tagged_digest_reference() -> None:
|
|
"""Flux's whole-image setter may retain the selected release tag."""
|
|
module = _load(RELEASE, "hermes_webui_release_tagged_digest")
|
|
old_digest = "sha256:" + "1" * 64
|
|
new_digest = "sha256:" + "2" * 64
|
|
source = (
|
|
"apiVersion: apps/v1\n"
|
|
"kind: Deployment\n"
|
|
"metadata:\n"
|
|
" name: hermes\n"
|
|
"spec:\n"
|
|
" image: registry.bstein.dev/bstein/hermes-webui:"
|
|
f"git-{'a' * 40}-build-7-release@{old_digest} "
|
|
'# {"$imagepolicy": "hermes:hermes-webui-release"}\n'
|
|
)
|
|
|
|
rendered = module.render_workload(
|
|
source, new_digest, kind="Deployment", name="hermes"
|
|
)
|
|
|
|
assert f"image: {module.DEFAULT_IMAGE}@{new_digest}" in rendered
|
|
assert '"$imagepolicy": "hermes:hermes-webui-release"' in rendered
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("source", "kind", "name", "match"),
|
|
[
|
|
(
|
|
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: wrong\n",
|
|
"Deployment",
|
|
"hermes",
|
|
"identity changed",
|
|
),
|
|
(
|
|
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes\n",
|
|
"Deployment",
|
|
"hermes",
|
|
"found 0",
|
|
),
|
|
(
|
|
"apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: hermes\n"
|
|
"spec:\n image: registry.bstein.dev/bstein/hermes-webui:latest\n",
|
|
"Deployment",
|
|
"hermes",
|
|
"found 0",
|
|
),
|
|
],
|
|
)
|
|
def test_renderer_fails_closed_on_flux_target_drift(
|
|
source: str, kind: str, name: str, match: str
|
|
) -> None:
|
|
module = _load(RELEASE, f"webui_renderer_{abs(hash(source))}")
|
|
with pytest.raises(ValueError, match=match):
|
|
module.render_workload(source, "sha256:" + "a" * 64, kind=kind, name=name)
|
|
|
|
|
|
def test_evidence_revalidation_rejects_extra_or_changed_files(tmp_path: Path) -> None:
|
|
"""Archived output is an exact deterministic set, not a best-effort bundle."""
|
|
module, _digest, kwargs = _release_fixture(tmp_path)
|
|
module.validate_release_artifacts(**kwargs)
|
|
extra = kwargs["output_dir"] / "unexpected"
|
|
extra.write_text("surprise\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="exactly four"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
extra.unlink()
|
|
metadata = kwargs["output_dir"] / "hermes-webui-image.json"
|
|
metadata.write_text("{}\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="incomplete or mismatched"):
|
|
module.validate_release_artifacts(**kwargs)
|
|
|
|
|
|
class _Response(io.BytesIO):
|
|
status = 200
|
|
|
|
def __init__(self, body: bytes, headers: dict[str, str] | None = None):
|
|
super().__init__(body)
|
|
self.headers = headers or {}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
self.close()
|
|
|
|
|
|
def test_release_verifies_exact_webui_harbor_artifact_and_policy() -> None:
|
|
"""Independent evidence resolves the tag and exact WebUI immutability rule."""
|
|
module = _load(RELEASE, "webui_registry_contract")
|
|
revision = "a" * 40
|
|
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
|
|
digest = "sha256:" + "b" * 64
|
|
seen = []
|
|
|
|
def artifact_open(request, timeout):
|
|
seen.append((request, timeout))
|
|
return _Response(
|
|
json.dumps(
|
|
{
|
|
"digest": digest,
|
|
"tags": [{"name": f"git-{revision}-build-9", "immutable": True}],
|
|
"extra_attrs": {
|
|
"config": {
|
|
"Labels": {
|
|
"org.opencontainers.image.revision": revision,
|
|
}
|
|
}
|
|
},
|
|
}
|
|
).encode()
|
|
)
|
|
|
|
module.verify_registry_digest(
|
|
destination,
|
|
digest,
|
|
revision,
|
|
username="robot",
|
|
password="private",
|
|
opener=artifact_open,
|
|
)
|
|
request = seen[0][0]
|
|
assert "/repositories/hermes-webui/artifacts/" in request.full_url
|
|
assert request.full_url.startswith("https://registry.bstein.dev/api/v2.0/")
|
|
assert request.get_header("Authorization").startswith("Basic ")
|
|
assert seen[0][1] == 20
|
|
|
|
def policy_open(request, timeout):
|
|
assert request.full_url.endswith(
|
|
"/projects/bstein/immutabletagrules?page=1&page_size=100"
|
|
)
|
|
return _Response(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"disabled": False,
|
|
"action": "immutable",
|
|
"template": "immutable_template",
|
|
"tag_selectors": [
|
|
{
|
|
"kind": "doublestar",
|
|
"decoration": "matches",
|
|
"pattern": "git-*-build-*",
|
|
}
|
|
],
|
|
"scope_selectors": {
|
|
"repository": [
|
|
{
|
|
"kind": "doublestar",
|
|
"decoration": "repoMatches",
|
|
"pattern": "hermes-webui",
|
|
}
|
|
]
|
|
},
|
|
}
|
|
]
|
|
).encode(),
|
|
{"X-Total-Count": "1"},
|
|
)
|
|
|
|
module.verify_immutable_policy(
|
|
username="robot", password="private", opener=policy_open
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"labels", [None, {}, {"org.opencontainers.image.revision": "c" * 40}]
|
|
)
|
|
def test_release_rejects_missing_or_wrong_harbor_source_revision(labels) -> None:
|
|
"""A tag derived from Git cannot substitute for the persisted OCI label."""
|
|
module = _load(RELEASE, f"webui_registry_revision_{labels!r}")
|
|
revision = "a" * 40
|
|
destination = f"{module.DEFAULT_IMAGE}:git-{revision}-build-9"
|
|
digest = "sha256:" + "b" * 64
|
|
|
|
def artifact_open(_request, _timeout):
|
|
config = {} if labels is None else {"Labels": labels}
|
|
return _Response(
|
|
json.dumps(
|
|
{
|
|
"digest": digest,
|
|
"tags": [{"name": f"git-{revision}-build-9", "immutable": True}],
|
|
"extra_attrs": {"config": config},
|
|
}
|
|
).encode()
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="OCI (image labels|source-revision label)"):
|
|
module.verify_registry_digest(
|
|
destination,
|
|
digest,
|
|
revision,
|
|
username="robot",
|
|
password="private",
|
|
opener=artifact_open,
|
|
)
|
|
|
|
|
|
def test_flux_tracks_webui_policy_before_jenkins() -> None:
|
|
"""The immutable Harbor rule is reviewed desired state, not a pipeline wish."""
|
|
harbor = yaml.safe_load(
|
|
(
|
|
ROOT / "clusters/atlas/flux-system/applications/harbor/kustomization.yaml"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
checks = harbor["spec"]["healthChecks"]
|
|
assert {
|
|
"apiVersion": "batch/v1",
|
|
"kind": "Job",
|
|
"name": "harbor-hermes-webui-immutability-ensure-1",
|
|
"namespace": "harbor",
|
|
} in checks
|
|
jenkins = yaml.safe_load(
|
|
(
|
|
ROOT / "clusters/atlas/flux-system/applications/jenkins/kustomization.yaml"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert "harbor" in {item["name"] for item in jenkins["spec"]["dependsOn"]}
|
|
|
|
policy = _load(POLICY, "webui_policy_contract")
|
|
assert policy.REPOSITORY_PATTERN == "hermes-webui"
|
|
assert policy.TAG_PATTERN == "git-*-build-*"
|
|
assert policy.EXPECTED_RULE["disabled"] is False
|
|
assert "robot" not in POLICY.read_text(encoding="utf-8").lower()
|
|
|
|
job = yaml.safe_load(
|
|
(ROOT / "services/harbor/hermes-webui-immutability-job.yaml").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
)
|
|
template = job["spec"]["template"]
|
|
annotations = template["metadata"]["annotations"]
|
|
assert annotations["vault.hashicorp.com/role"] == "harbor-policy-bootstrap"
|
|
assert (
|
|
annotations["vault.hashicorp.com/agent-inject-secret-harbor-admin-password"]
|
|
== "kv/data/atlas/harbor/harbor-core"
|
|
)
|
|
pod = template["spec"]
|
|
assert pod["serviceAccountName"] == "harbor-policy-bootstrap"
|
|
assert pod["enableServiceLinks"] is False
|
|
container = pod["containers"][0]
|
|
assert "@sha256:" in container["image"]
|
|
assert container["securityContext"]["readOnlyRootFilesystem"] is True
|
|
assert container["securityContext"]["runAsNonRoot"] is True
|
|
assert container["securityContext"]["capabilities"]["drop"] == ["ALL"]
|
|
|
|
release_docs = (ROOT / "docs/hermes_webui_release.md").read_text(encoding="utf-8")
|
|
assert "immutable-tag:list" in release_docs
|
|
assert "harbor-hermes-agent-immutability-ensure-1" in release_docs
|
|
|
|
|
|
class _FakePolicyClient:
|
|
origin = "https://registry.bstein.dev/api/v2.0"
|
|
|
|
def __init__(self, responses):
|
|
self.responses = list(responses)
|
|
self.calls = []
|
|
|
|
def request(self, method, path, payload=None):
|
|
self.calls.append((method, path, payload))
|
|
return self.responses.pop(0)
|
|
|
|
|
|
def test_webui_policy_is_idempotent_and_create_is_reread() -> None:
|
|
"""The desired Harbor rule validates in place or verifies its exact new ID."""
|
|
policy = _load(POLICY, "webui_policy_idempotency")
|
|
existing = {"id": 17, **policy.EXPECTED_RULE}
|
|
client = _FakePolicyClient(
|
|
[(200, json.dumps([existing]).encode(), {"X-Total-Count": "1"})]
|
|
)
|
|
assert policy.ensure_rule(client) == 17
|
|
assert [call[0] for call in client.calls] == ["GET"]
|
|
|
|
created = {"id": 23, **policy.EXPECTED_RULE}
|
|
client = _FakePolicyClient(
|
|
[
|
|
(200, b"[]", {"X-Total-Count": "0"}),
|
|
(
|
|
201,
|
|
b"",
|
|
{"Location": ("/api/v2.0/projects/bstein/immutabletagrules/23")},
|
|
),
|
|
(200, json.dumps([created]).encode(), {"X-Total-Count": "1"}),
|
|
]
|
|
)
|
|
assert policy.ensure_rule(client) == 23
|
|
assert client.calls[1] == (
|
|
"POST",
|
|
"/projects/bstein/immutabletagrules",
|
|
policy.EXPECTED_RULE,
|
|
)
|
|
|
|
|
|
def test_webui_policy_rejects_disabled_duplicate_or_truncated_rules() -> None:
|
|
"""Ambiguous or incomplete Harbor evidence can never unblock publication."""
|
|
policy = _load(POLICY, "webui_policy_rejections")
|
|
disabled = {"id": 17, **policy.EXPECTED_RULE, "disabled": True}
|
|
with pytest.raises(RuntimeError, match="not enabled and exact"):
|
|
policy.ensure_rule(
|
|
_FakePolicyClient(
|
|
[
|
|
(
|
|
200,
|
|
json.dumps([disabled]).encode(),
|
|
{"X-Total-Count": "1"},
|
|
)
|
|
]
|
|
)
|
|
)
|
|
|
|
duplicate = [
|
|
{"id": 17, **policy.EXPECTED_RULE},
|
|
{"id": 18, **policy.EXPECTED_RULE},
|
|
]
|
|
with pytest.raises(RuntimeError, match="multiple Hermes WebUI"):
|
|
policy.ensure_rule(
|
|
_FakePolicyClient(
|
|
[
|
|
(
|
|
200,
|
|
json.dumps(duplicate).encode(),
|
|
{"X-Total-Count": "2"},
|
|
)
|
|
]
|
|
)
|
|
)
|
|
|
|
with pytest.raises(RuntimeError, match="truncated"):
|
|
policy.list_rules(_FakePolicyClient([(200, b"[]", {"X-Total-Count": "1"})]))
|
|
|
|
|
|
def test_pipeline_archives_exact_eight_files_before_release() -> None:
|
|
"""Release cannot pass with missing digest, workload, or metadata evidence."""
|
|
source = PIPELINE.read_text(encoding="utf-8")
|
|
evidence = source.split("stage('Verify and archive release evidence')", 1)[1]
|
|
evidence = evidence.split("stage('Publish Flux release tag')", 1)[0]
|
|
archive = evidence.split("artifacts: '", 1)[1].split("'", 1)[0].split(",")
|
|
assert len(archive) == len(set(archive)) == 8
|
|
assert all("*" not in path for path in archive)
|
|
assert "find build -type f" in evidence
|
|
assert "allowEmptyArchive: false" in evidence
|
|
assert "build/hermes-webui.source-revision" in archive
|
|
assert "hermes-chat-statefulset.yaml" in evidence
|
|
assert "hermes-dashboard-deployment.yaml" in evidence
|
|
assert " post {" not in source
|