diff --git a/ci/Jenkinsfile.hermes-agent-image b/ci/Jenkinsfile.hermes-agent-image index c10ff9e4..c94e30ca 100644 --- a/ci/Jenkinsfile.hermes-agent-image +++ b/ci/Jenkinsfile.hermes-agent-image @@ -178,7 +178,15 @@ spec: container('python') { sh ''' set -eu + # Install the pinned test deps fully offline from the reviewed, + # in-repo wheelhouse (ci/vendor/hermes-agent-test-wheels). --no-index + # forbids any network index, so this stage never resolves + # pypi.org/files.pythonhosted.org and cannot fail on public-internet + # DNS. The wheels match this pipeline's arm64 python:3.12 container + # (PyYAML is the cp312 manylinux aarch64 build; the rest are + # py3-none-any). Bump the wheelhouse when these pins change. python3 -m pip install --disable-pip-version-check --no-cache-dir \ + --no-index --find-links="${WORKSPACE}/ci/vendor/hermes-agent-test-wheels" \ --target=/tmp/hermes-agent-release-test-deps \ pytest==8.3.4 PyYAML==6.0.2 PYTHONPATH=/tmp/hermes-agent-release-test-deps \ diff --git a/ci/vendor/hermes-agent-test-wheels/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl b/ci/vendor/hermes-agent-test-wheels/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl new file mode 100644 index 00000000..c2b5d354 Binary files /dev/null and b/ci/vendor/hermes-agent-test-wheels/PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl differ diff --git a/ci/vendor/hermes-agent-test-wheels/iniconfig-2.3.0-py3-none-any.whl b/ci/vendor/hermes-agent-test-wheels/iniconfig-2.3.0-py3-none-any.whl new file mode 100644 index 00000000..f8cd4b9e Binary files /dev/null and b/ci/vendor/hermes-agent-test-wheels/iniconfig-2.3.0-py3-none-any.whl differ diff --git a/ci/vendor/hermes-agent-test-wheels/packaging-26.3-py3-none-any.whl b/ci/vendor/hermes-agent-test-wheels/packaging-26.3-py3-none-any.whl new file mode 100644 index 00000000..d54f5712 Binary files /dev/null and b/ci/vendor/hermes-agent-test-wheels/packaging-26.3-py3-none-any.whl differ diff --git a/ci/vendor/hermes-agent-test-wheels/pluggy-1.6.0-py3-none-any.whl b/ci/vendor/hermes-agent-test-wheels/pluggy-1.6.0-py3-none-any.whl new file mode 100644 index 00000000..1f7e6268 Binary files /dev/null and b/ci/vendor/hermes-agent-test-wheels/pluggy-1.6.0-py3-none-any.whl differ diff --git a/ci/vendor/hermes-agent-test-wheels/pytest-8.3.4-py3-none-any.whl b/ci/vendor/hermes-agent-test-wheels/pytest-8.3.4-py3-none-any.whl new file mode 100644 index 00000000..86f23262 Binary files /dev/null and b/ci/vendor/hermes-agent-test-wheels/pytest-8.3.4-py3-none-any.whl differ diff --git a/dockerfiles/Dockerfile.hermes-agent b/dockerfiles/Dockerfile.hermes-agent index 2ec4b6e5..65c1325a 100644 --- a/dockerfiles/Dockerfile.hermes-agent +++ b/dockerfiles/Dockerfile.hermes-agent @@ -10,7 +10,19 @@ # arm64 build is unchanged; Kaniko/containerd auto-selects the matching leaf per # build platform (arm64 rpi5 pod vs amd64 titan-24 pod). Do NOT replace this with # a per-arch leaf digest -- that would break the amd64 build leg. -FROM nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 +# +# The reference below points at the in-cluster Harbor "mirror" project, NOT +# docker.io. The identical upstream index (same content-addressed digest +# 9c841866..., both leaves) is mirrored into Harbor with +# skopeo copy --all docker://nousresearch/hermes-agent@sha256:9c841866... \ +# docker://harbor-core.harbor.svc.cluster.local/mirror/hermes-agent@sha256:9c841866... +# by the Flux-managed one-shot Job services/harbor/hermes-agent-base-mirror-job.yaml. +# Because the digest is content-addressed, mirroring reproduces the exact index +# and both leaf digests, so this remains fully digest-pinned and multi-arch while +# Kaniko pulls it over the internal insecure registry (see the pipeline's +# --insecure-registry/--registry-mirror flags) with no docker.io fallback. +# Re-run that Job whenever this digest is bumped, before publishing the image. +FROM harbor-core.harbor.svc.cluster.local/mirror/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973 USER root diff --git a/services/harbor/hermes-agent-base-mirror-job.yaml b/services/harbor/hermes-agent-base-mirror-job.yaml new file mode 100644 index 00000000..4c89ff38 --- /dev/null +++ b/services/harbor/hermes-agent-base-mirror-job.yaml @@ -0,0 +1,141 @@ +# services/harbor/hermes-agent-base-mirror-job.yaml +# +# One-shot mirror of the Hermes agent base image INDEX from docker.io into the +# in-cluster Harbor "mirror" project, so the reviewed image build +# (dockerfiles/Dockerfile.hermes-agent) pulls its FROM base internally with no +# docker.io fallback (which is IPv6-broken from build pods). +# +# Kept `suspend: true` like bootstrap-jobs/cassandra-registry-ensure-job.yaml: +# it needs egress to docker.io (registry-1.docker.io / *.pythonhosted is NOT +# involved here) and is only run deliberately, once per base-digest bump. To run +# it, an operator clears suspend (or `kubectl create job --from`) AFTER updating +# the digest in BOTH args below and in the Dockerfile FROM. Because the digest +# is content-addressed, `skopeo copy --all` reproduces the identical index and +# both arch leaves in Harbor -- the build stays digest-pinned and multi-arch. +apiVersion: batch/v1 +kind: Job +metadata: + name: harbor-hermes-agent-base-mirror-1 + namespace: harbor +spec: + suspend: true + backoffLimit: 2 + activeDeadlineSeconds: 1800 + template: + metadata: + annotations: + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-user: "65532" + vault.hashicorp.com/agent-run-as-group: "65532" + vault.hashicorp.com/role: harbor-policy-bootstrap + vault.hashicorp.com/agent-inject-secret-harbor-admin-password: kv/data/atlas/harbor/harbor-core + vault.hashicorp.com/agent-inject-template-harbor-admin-password: | + {{- with secret "kv/data/atlas/harbor/harbor-core" -}} + {{ .Data.data.harbor_admin_password }} + {{- end -}} + spec: + serviceAccountName: harbor-policy-bootstrap + enableServiceLinks: false + restartPolicy: Never + nodeSelector: + hardware: rpi5 + kubernetes.io/arch: arm64 + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/hostname + operator: NotIn + values: [titan-04, titan-14, titan-18, titan-19, titan-24] + securityContext: + fsGroup: 65532 + fsGroupChangePolicy: OnRootMismatch + seccompProfile: + type: RuntimeDefault + initContainers: + # Ensure the public "mirror" project exists before skopeo tries to push + # into it (Harbor only auto-creates repositories inside an existing + # project). Uses the same Vault-injected admin password as the sibling + # Harbor immutability jobs -- no new credential is introduced. + - name: ensure-project + image: docker.io/library/python@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 + imagePullPolicy: IfNotPresent + command: [python3, /scripts/harbor_mirror_project_ensure.py] + env: + - name: HARBOR_API_ORIGIN + value: http://harbor-core.harbor.svc.cluster.local/api/v2.0 + - name: HARBOR_ADMIN_PASSWORD_FILE + value: /vault/secrets/harbor-admin-password + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: scripts + mountPath: /scripts + readOnly: true + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {cpu: 250m, memory: 128Mi} + containers: + # skopeo copies the WHOLE multi-arch index (--all) straight from docker.io + # to the internal Harbor registry over HTTP (--dest-tls-verify=false, the + # same insecure in-cluster endpoint Kaniko already trusts). Source is the + # public upstream image, so no source credential is needed; the digest is + # asserted on both ends so a drifted upstream tag cannot be mirrored. + - name: mirror + image: quay.io/skopeo/stable@sha256:94f5c5e26997e2e78c234ec9abf19a391c234b39eb22e6d1210d0b527c97dcc8 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + - | + set -eu + src="docker://nousresearch/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973" + dst="docker://harbor-core.harbor.svc.cluster.local/mirror/hermes-agent@sha256:9c841866021c54c4596849f6135717e8a4d52ba510b7f52c50aef1de1a283973" + pw="$(cat /vault/secrets/harbor-admin-password)" + exec skopeo copy --all \ + --src-tls-verify=true \ + --dest-tls-verify=false \ + --dest-creds "admin:${pw}" \ + "${src}" "${dst}" + env: + - name: HOME + value: /tmp + - name: TMPDIR + value: /tmp + - name: XDG_RUNTIME_DIR + value: /tmp + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: tmp + mountPath: /tmp + resources: + requests: {cpu: 100m, memory: 256Mi} + limits: {cpu: "1", memory: 1Gi} + volumes: + - name: scripts + configMap: + name: harbor-hermes-agent-base-mirror-script + defaultMode: 0555 + - name: tmp + emptyDir: {} diff --git a/services/harbor/kustomization.yaml b/services/harbor/kustomization.yaml index b221025f..97b3fffc 100644 --- a/services/harbor/kustomization.yaml +++ b/services/harbor/kustomization.yaml @@ -16,6 +16,7 @@ resources: - hermes-agent-immutability-job.yaml - hermes-webui-immutability-job.yaml - hermes-chat-router-immutability-job.yaml + - hermes-agent-base-mirror-job.yaml - bootstrap-jobs/cassandra-registry-ensure-job.yaml - image.yaml configMapGenerator: @@ -32,3 +33,6 @@ configMapGenerator: files: - harbor_immutable_rule_ensure.py=scripts/harbor_immutable_rule_ensure.py - harbor_hermes_chat_router_immutability_ensure.py=scripts/harbor_hermes_chat_router_immutability_ensure.py + - name: harbor-hermes-agent-base-mirror-script + files: + - harbor_mirror_project_ensure.py=scripts/harbor_mirror_project_ensure.py diff --git a/services/harbor/scripts/harbor_mirror_project_ensure.py b/services/harbor/scripts/harbor_mirror_project_ensure.py new file mode 100644 index 00000000..50527577 --- /dev/null +++ b/services/harbor/scripts/harbor_mirror_project_ensure.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Idempotently ensure the public Harbor ``mirror`` project exists. + +The Hermes agent image base is pulled by Kaniko from an in-cluster Harbor +project instead of docker.io (see dockerfiles/Dockerfile.hermes-agent). This +helper creates that project so the sibling ``skopeo copy --all`` step has a +push target. It is deliberately tiny and fail-closed: any unexpected Harbor +response aborts with a non-zero exit so the mirror Job does not silently push +into a mis-scoped project. + +The project is created ``public`` on purpose: it only ever holds mirrored +copies of already-public upstream base images (content-addressed by digest), +and public read lets Kaniko pull it over the internal registry without needing +a Harbor pull credential in its build config. +""" + +from __future__ import annotations + +import base64 +import json +import os +import urllib.error +import urllib.request +from pathlib import Path + +PROJECT = "mirror" +MAX_RESPONSE = 1_048_576 + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """Never replay Basic credentials across an unexpected redirect.""" + + def redirect_request(self, *_args): # noqa: D401 - urllib signature + return None + + +def _admin_password() -> str: + path = os.environ.get( + "HARBOR_ADMIN_PASSWORD_FILE", "/vault/secrets/harbor-admin-password" + ) + password = Path(path).read_text(encoding="utf-8").strip() + if not password: + raise SystemExit(f"Harbor admin password at {path} is empty") + return password + + +def _api_origin() -> str: + origin = os.environ.get( + "HARBOR_API_ORIGIN", + "http://harbor-core.harbor.svc.cluster.local/api/v2.0", + ).rstrip("/") + return origin + + +def _request(opener, origin, auth, method, path, payload=None, ok=(200,)): + url = f"{origin}{path}" + data = None + headers = {"Authorization": f"Basic {auth}"} + if payload is not None: + data = json.dumps(payload).encode("utf-8") + headers["Content-Type"] = "application/json" + req = urllib.request.Request(url, data=data, method=method, headers=headers) + try: + with opener.open(req, timeout=30) as response: + body = response.read(MAX_RESPONSE + 1) + if len(body) > MAX_RESPONSE: + raise SystemExit(f"{method} {path} response exceeded cap") + if response.status not in ok: + raise SystemExit(f"{method} {path} returned {response.status}") + if not body: + return None + return json.loads(body.decode("utf-8")) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + if exc.code in ok: + return None + raise SystemExit(f"{method} {path} returned {exc.code}: {detail}") from exc + + +def main() -> None: + origin = _api_origin() + auth = base64.b64encode(f"admin:{_admin_password()}".encode("utf-8")).decode( + "ascii" + ) + opener = urllib.request.build_opener(NoRedirect()) + + existing = _request( + opener, origin, auth, "GET", f"/projects?name={PROJECT}", ok=(200,) + ) + if existing: + match = next( + ( + item + for item in existing + if item.get("name") == PROJECT + ), + None, + ) + if match is not None: + public = str( + (match.get("metadata") or {}).get("public", "") + ).lower() + if public not in ("true", "1"): + raise SystemExit( + f"Harbor project {PROJECT!r} exists but is not public; " + "refusing to push mirrored base image into it" + ) + print(f"Harbor project {PROJECT!r} already present and public") + return + + _request( + opener, + origin, + auth, + "POST", + "/projects", + { + "project_name": PROJECT, + "metadata": { + "public": "true", + "auto_scan": "false", + "enable_content_trust": "false", + "prevent_vul": "false", + "reuse_sys_cve_allowlist": "true", + }, + }, + ok=(201, 409), + ) + print(f"Ensured public Harbor project {PROJECT!r}") + + +if __name__ == "__main__": + main() diff --git a/testing/tests/test_hermes_mirror_project_ensure.py b/testing/tests/test_hermes_mirror_project_ensure.py new file mode 100644 index 00000000..19f6ee7e --- /dev/null +++ b/testing/tests/test_hermes_mirror_project_ensure.py @@ -0,0 +1,226 @@ +"""Boundary coverage for the Harbor ``mirror`` project bootstrap helper. + +The Hermes agent image build pulls its base from the in-cluster Harbor +``mirror`` project instead of docker.io; this exercises the tiny fail-closed +helper that guarantees the project exists (and is public) before the sibling +``skopeo copy --all`` step pushes into it. No network is touched. +""" + +from __future__ import annotations + +import importlib.util +import io +import json +import urllib.error +import urllib.request +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "services/harbor/scripts/harbor_mirror_project_ensure.py" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class Response(io.BytesIO): + """Context-managed urllib response used without network access.""" + + def __init__(self, status: int, body: bytes = b"") -> None: + super().__init__(body) + self.status = status + + def __enter__(self): + return self + + def __exit__(self, *_args): + self.close() + + +class FakeOpener: + """Ordered opener fake matching urllib's build_opener().open() contract.""" + + def __init__(self, results) -> None: + self.results = list(results) + self.requests = [] + + def open(self, req, timeout=None): # noqa: A003 - urllib signature + self.requests.append((req.get_method(), req.full_url, req.data)) + result = self.results.pop(0) + if isinstance(result, Exception): + raise result + return result + + +def _http_error(code: int, body: bytes = b"boom") -> urllib.error.HTTPError: + return urllib.error.HTTPError( + "http://harbor/api", code, "err", {}, io.BytesIO(body) + ) + + +def _password_file(tmp_path: Path, contents: str) -> Path: + path = tmp_path / "harbor-admin-password" + path.write_text(contents, encoding="utf-8") + return path + + +# --------------------------------------------------------------------------- # +# _admin_password / _api_origin +# --------------------------------------------------------------------------- # +def test_admin_password_reads_and_strips_the_injected_file( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load("mirror_pw_ok") + monkeypatch.setenv( + "HARBOR_ADMIN_PASSWORD_FILE", str(_password_file(tmp_path, " s3cret \n")) + ) + assert module._admin_password() == "s3cret" + + +def test_admin_password_rejects_empty_secret( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load("mirror_pw_empty") + monkeypatch.setenv( + "HARBOR_ADMIN_PASSWORD_FILE", str(_password_file(tmp_path, " \n")) + ) + with pytest.raises(SystemExit): + module._admin_password() + + +def test_api_origin_defaults_and_honours_override( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load("mirror_origin") + monkeypatch.delenv("HARBOR_API_ORIGIN", raising=False) + assert module._api_origin().endswith("harbor.svc.cluster.local/api/v2.0") + monkeypatch.setenv("HARBOR_API_ORIGIN", "http://example/api/v2.0/") + assert module._api_origin() == "http://example/api/v2.0" + + +# --------------------------------------------------------------------------- # +# _request +# --------------------------------------------------------------------------- # +def test_request_returns_parsed_body_and_records_method() -> None: + module = _load("mirror_req_ok") + opener = FakeOpener([Response(200, json.dumps([{"name": "mirror"}]).encode())]) + result = module._request(opener, "http://h", "auth", "GET", "/projects") + assert result == [{"name": "mirror"}] + assert opener.requests[0][0] == "GET" + + +def test_request_posts_payload_and_tolerates_empty_body() -> None: + module = _load("mirror_req_post") + opener = FakeOpener([Response(201, b"")]) + result = module._request( + opener, "http://h", "auth", "POST", "/projects", {"a": 1}, ok=(201, 409) + ) + assert result is None + method, _url, data = opener.requests[0] + assert method == "POST" and data == json.dumps({"a": 1}).encode() + + +def test_request_rejects_oversized_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + module = _load("mirror_req_big") + monkeypatch.setattr(module, "MAX_RESPONSE", 4) + opener = FakeOpener([Response(200, b"xxxxxxxx")]) + with pytest.raises(SystemExit): + module._request(opener, "http://h", "auth", "GET", "/projects") + + +def test_request_rejects_unexpected_status() -> None: + module = _load("mirror_req_status") + opener = FakeOpener([Response(500, b"nope")]) + with pytest.raises(SystemExit): + module._request(opener, "http://h", "auth", "GET", "/projects") + + +def test_request_maps_http_error_status_to_ok_or_failure() -> None: + module = _load("mirror_req_httperror") + tolerated = FakeOpener([_http_error(409)]) + assert ( + module._request( + tolerated, "http://h", "auth", "POST", "/projects", {"a": 1}, ok=(201, 409) + ) + is None + ) + fatal = FakeOpener([_http_error(403)]) + with pytest.raises(SystemExit): + module._request(fatal, "http://h", "auth", "POST", "/projects", {"a": 1}) + + +# --------------------------------------------------------------------------- # +# main +# --------------------------------------------------------------------------- # +def _prime(module, monkeypatch, tmp_path, results): + monkeypatch.setenv( + "HARBOR_ADMIN_PASSWORD_FILE", str(_password_file(tmp_path, "pw")) + ) + monkeypatch.setenv("HARBOR_API_ORIGIN", "http://harbor/api/v2.0") + opener = FakeOpener(results) + monkeypatch.setattr( + module.urllib.request, "build_opener", lambda *_a, **_k: opener + ) + return opener + + +def test_main_is_a_noop_when_public_project_exists( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load("mirror_main_present") + body = json.dumps( + [{"name": "mirror", "metadata": {"public": "true"}}] + ).encode() + opener = _prime(module, monkeypatch, tmp_path, [Response(200, body)]) + module.main() + assert [call[0] for call in opener.requests] == ["GET"] + + +def test_main_fails_closed_on_a_private_collision( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load("mirror_main_private") + body = json.dumps( + [{"name": "mirror", "metadata": {"public": "false"}}] + ).encode() + _prime(module, monkeypatch, tmp_path, [Response(200, body)]) + with pytest.raises(SystemExit): + module.main() + + +@pytest.mark.parametrize( + "listing", + [b"[]", json.dumps([{"name": "other", "metadata": {"public": "true"}}]).encode()], +) +def test_main_creates_the_project_when_absent( + listing: bytes, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load("mirror_main_create") + opener = _prime( + module, + monkeypatch, + tmp_path, + [Response(200, listing), Response(201, b"")], + ) + module.main() + methods = [call[0] for call in opener.requests] + assert methods == ["GET", "POST"] + assert opener.requests[1][1].endswith("/projects") + + +# --------------------------------------------------------------------------- # +# misc +# --------------------------------------------------------------------------- # +def test_no_redirect_handler_refuses_to_replay_credentials() -> None: + module = _load("mirror_noredirect") + handler = module.NoRedirect() + assert handler.redirect_request(None, None, 302, "m", {}, "http://elsewhere") is None