diff --git a/ariadne/app.py b/ariadne/app.py index 5e9ecb1..67492b0 100644 --- a/ariadne/app.py +++ b/ariadne/app.py @@ -389,6 +389,7 @@ def _startup() -> None: "metis_k3s_token_sync_cron": settings.metis_k3s_token_sync_cron, "platform_quality_suite_probe_cron": settings.platform_quality_suite_probe_cron, "jenkins_workspace_cleanup_cron": settings.jenkins_workspace_cleanup_cron, + "jenkins_workspace_cleanup_dry_run": settings.jenkins_workspace_cleanup_dry_run, "vault_k8s_auth_cron": settings.vault_k8s_auth_cron, "vault_oidc_cron": settings.vault_oidc_cron, "comms_guest_name_cron": settings.comms_guest_name_cron, diff --git a/ariadne/services/jenkins_workspace_cleanup.py b/ariadne/services/jenkins_workspace_cleanup.py index bf8d1b3..c731ab9 100644 --- a/ariadne/services/jenkins_workspace_cleanup.py +++ b/ariadne/services/jenkins_workspace_cleanup.py @@ -4,6 +4,8 @@ from dataclasses import dataclass from datetime import datetime, timedelta, timezone from typing import Any +from prometheus_client import Counter, Gauge + from ..k8s.client import delete_json, get_json from ..settings import settings from ..utils.logging import get_logger @@ -12,6 +14,48 @@ from ..utils.logging import get_logger logger = get_logger(__name__) +JENKINS_WORKSPACE_CLEANUP_RUNS_TOTAL = Counter( + "ariadne_jenkins_workspace_cleanup_runs_total", + "Jenkins workspace cleanup runs by status and mode", + ["status", "mode"], +) +JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL = Counter( + "ariadne_jenkins_workspace_cleanup_objects_total", + "Jenkins workspace cleanup objects by kind, action, and mode", + ["kind", "action", "mode"], +) +JENKINS_WORKSPACE_CLEANUP_LAST_RUN_TS = Gauge( + "ariadne_jenkins_workspace_cleanup_last_run_timestamp_seconds", + "Last Jenkins workspace cleanup run timestamp", +) +JENKINS_WORKSPACE_CLEANUP_LAST_SUCCESS_TS = Gauge( + "ariadne_jenkins_workspace_cleanup_last_success_timestamp_seconds", + "Last successful Jenkins workspace cleanup timestamp", +) +JENKINS_WORKSPACE_CLEANUP_LAST_FAILURE_TS = Gauge( + "ariadne_jenkins_workspace_cleanup_last_failure_timestamp_seconds", + "Last failed Jenkins workspace cleanup timestamp", +) +JENKINS_WORKSPACE_CLEANUP_LAST_DELETED = Gauge( + "ariadne_jenkins_workspace_cleanup_last_deleted_total", + "Last Jenkins workspace cleanup deleted object count", + ["kind"], +) +JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED = Gauge( + "ariadne_jenkins_workspace_cleanup_last_planned_total", + "Last Jenkins workspace cleanup planned object count", + ["kind"], +) +JENKINS_WORKSPACE_CLEANUP_LAST_SKIPPED = Gauge( + "ariadne_jenkins_workspace_cleanup_last_skipped_total", + "Last Jenkins workspace cleanup skipped object count", +) +JENKINS_WORKSPACE_CLEANUP_LAST_FAILURES = Gauge( + "ariadne_jenkins_workspace_cleanup_last_failures_total", + "Last Jenkins workspace cleanup failure count", +) + + @dataclass(frozen=True) class JenkinsWorkspaceCleanupSummary: """Summarize one Jenkins workspace-storage cleanup pass. @@ -20,11 +64,33 @@ class JenkinsWorkspaceCleanupSummary: Outputs: deterministic counters for operator logs and metrics. """ + pvs_planned: int + pvcs_planned: int + volumes_planned: int pvs_deleted: int pvcs_deleted: int volumes_deleted: int skipped: int failures: int + dry_run: bool + + @property + def planned(self) -> int: + return self.pvs_planned + self.pvcs_planned + self.volumes_planned + + @property + def deleted(self) -> int: + return self.pvs_deleted + self.pvcs_deleted + self.volumes_deleted + + +@dataclass(frozen=True) +class _CleanupCandidate: + name: str + kind: str + path: str + created_at: datetime | None + related_pvc: str | None = None + pv_name: str | None = None def _parse_timestamp(raw: str) -> datetime | None: @@ -37,19 +103,27 @@ def _parse_timestamp(raw: str) -> datetime | None: return None +def _created_at(metadata: dict[str, Any]) -> datetime | None: + raw = metadata.get("creationTimestamp") + if not isinstance(raw, str) or not raw: + return None + return _parse_timestamp(raw) + + def _is_old_enough(metadata: dict[str, Any]) -> bool: """Return true when an object age exceeds the configured cleanup threshold.""" - raw = metadata.get("creationTimestamp") - if not isinstance(raw, str) or not raw: - return True - created_at = _parse_timestamp(raw) + created_at = _created_at(metadata) if created_at is None: - return True + return False min_age = timedelta(hours=settings.jenkins_workspace_cleanup_min_age_hours) return datetime.now(timezone.utc) - created_at >= min_age +def _is_workspace_name(name: Any) -> bool: + return isinstance(name, str) and name.startswith(settings.jenkins_workspace_pvc_prefix) + + def _active_workspace_claims() -> set[str]: """Collect currently referenced Jenkins workspace PVC names from pods.""" @@ -61,6 +135,8 @@ def _active_workspace_claims() -> set[str]: for pod in items: if not isinstance(pod, dict): continue + metadata = pod.get("metadata") if isinstance(pod.get("metadata"), dict) else {} + annotations = metadata.get("annotations") if isinstance(metadata.get("annotations"), dict) else {} spec = pod.get("spec") if isinstance(pod.get("spec"), dict) else {} volumes = spec.get("volumes") if isinstance(spec.get("volumes"), list) else [] for volume in volumes: @@ -70,19 +146,22 @@ def _active_workspace_claims() -> set[str]: if not isinstance(claim, dict): continue claim_name = claim.get("claimName") - if isinstance(claim_name, str) and claim_name.startswith(prefix): + if _is_workspace_name(claim_name): active.add(claim_name) + claim_name = annotations.get("jenkins.io/workspace-pvc") + if _is_workspace_name(claim_name): + active.add(claim_name) return active -def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[dict[str, Any]], set[str]]: +def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[_CleanupCandidate], set[str]]: """Find releasable Jenkins workspace PVs and keep a set of all PV names.""" namespace = settings.jenkins_workspace_namespace prefix = settings.jenkins_workspace_pvc_prefix payload = get_json("/api/v1/persistentvolumes") items = payload.get("items") if isinstance(payload.get("items"), list) else [] - candidates: list[dict[str, Any]] = [] + candidates: list[_CleanupCandidate] = [] all_pv_names: set[str] = set() for pv in items: @@ -101,7 +180,7 @@ def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[dict[str, An phase = status.get("phase") if claim_namespace != namespace: continue - if not isinstance(claim_name, str) or not claim_name.startswith(prefix): + if not _is_workspace_name(claim_name): continue if claim_name in active_claims: continue @@ -109,18 +188,27 @@ def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[dict[str, An continue if not _is_old_enough(metadata): continue - candidates.append(pv) + if not isinstance(name, str) or not name: + continue + candidates.append( + _CleanupCandidate( + name=name, + kind="pv", + path=f"/api/v1/persistentvolumes/{name}", + created_at=_created_at(metadata), + related_pvc=claim_name if isinstance(claim_name, str) else None, + ) + ) return candidates, all_pv_names -def _workspace_pvc_candidates(active_claims: set[str]) -> list[dict[str, Any]]: +def _workspace_pvc_candidates(active_claims: set[str]) -> list[_CleanupCandidate]: """Find stale Jenkins workspace PVCs that are not actively referenced.""" namespace = settings.jenkins_workspace_namespace - prefix = settings.jenkins_workspace_pvc_prefix payload = get_json(f"/api/v1/namespaces/{namespace}/persistentvolumeclaims") items = payload.get("items") if isinstance(payload.get("items"), list) else [] - candidates: list[dict[str, Any]] = [] + candidates: list[_CleanupCandidate] = [] for pvc in items: if not isinstance(pvc, dict): @@ -129,7 +217,7 @@ def _workspace_pvc_candidates(active_claims: set[str]) -> list[dict[str, Any]]: status = pvc.get("status") if isinstance(pvc.get("status"), dict) else {} claim_name = metadata.get("name") phase = status.get("phase") - if not isinstance(claim_name, str) or not claim_name.startswith(prefix): + if not _is_workspace_name(claim_name): continue if claim_name in active_claims: continue @@ -137,96 +225,246 @@ def _workspace_pvc_candidates(active_claims: set[str]) -> list[dict[str, Any]]: continue if not _is_old_enough(metadata): continue - candidates.append(pvc) + if not isinstance(claim_name, str) or not claim_name: + continue + candidates.append( + _CleanupCandidate( + name=claim_name, + kind="pvc", + path=f"/api/v1/namespaces/{namespace}/persistentvolumeclaims/{claim_name}", + created_at=_created_at(metadata), + ) + ) return candidates +def _workspace_longhorn_candidates(all_pv_names: set[str], removed_pv_names: set[str]) -> list[_CleanupCandidate]: + namespace = "longhorn-system" + payload = get_json("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes") + items = payload.get("items") if isinstance(payload.get("items"), list) else [] + candidates: list[_CleanupCandidate] = [] + + for volume in items: + if not isinstance(volume, dict): + continue + metadata = volume.get("metadata") if isinstance(volume.get("metadata"), dict) else {} + status = volume.get("status") if isinstance(volume.get("status"), dict) else {} + spec = volume.get("spec") if isinstance(volume.get("spec"), dict) else {} + name = metadata.get("name") + if not isinstance(name, str) or not name: + continue + + labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {} + pvc_name = labels.get("kubernetes.io/created-for/pvc/name") + robust_state = status.get("robustness") + state = status.get("state") + attached = status.get("isAttached") + frontend = spec.get("frontend") + should_delete = False + if name in removed_pv_names: + should_delete = True + elif _is_workspace_name(pvc_name) and name not in all_pv_names: + should_delete = True + if not should_delete: + continue + if not _is_old_enough(metadata): + continue + if state not in {None, "detached", "faulted", "unknown"}: + continue + if attached is True: + continue + if robust_state not in {None, "unknown", "faulted", "degraded"}: + continue + if frontend not in {None, "", "blockdev"}: + continue + candidates.append( + _CleanupCandidate( + name=name, + kind="longhorn_volume", + path=f"/apis/longhorn.io/v1beta2/namespaces/{namespace}/volumes/{name}", + created_at=_created_at(metadata), + pv_name=name, + ) + ) + return candidates + + +def _record_metrics(summary: JenkinsWorkspaceCleanupSummary) -> None: + mode = "dry_run" if summary.dry_run else "delete" + status = "ok" if summary.failures == 0 else "error" + JENKINS_WORKSPACE_CLEANUP_RUNS_TOTAL.labels(status=status, mode=mode).inc() + if summary.failures: + JENKINS_WORKSPACE_CLEANUP_LAST_FAILURE_TS.set(datetime.now(timezone.utc).timestamp()) + else: + JENKINS_WORKSPACE_CLEANUP_LAST_SUCCESS_TS.set(datetime.now(timezone.utc).timestamp()) + JENKINS_WORKSPACE_CLEANUP_LAST_RUN_TS.set(datetime.now(timezone.utc).timestamp()) + JENKINS_WORKSPACE_CLEANUP_LAST_DELETED.labels(kind="pvc").set(summary.pvcs_deleted) + JENKINS_WORKSPACE_CLEANUP_LAST_DELETED.labels(kind="pv").set(summary.pvs_deleted) + JENKINS_WORKSPACE_CLEANUP_LAST_DELETED.labels(kind="longhorn_volume").set(summary.volumes_deleted) + JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED.labels(kind="pvc").set(summary.pvcs_planned) + JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED.labels(kind="pv").set(summary.pvs_planned) + JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED.labels(kind="longhorn_volume").set(summary.volumes_planned) + JENKINS_WORKSPACE_CLEANUP_LAST_SKIPPED.set(summary.skipped) + JENKINS_WORKSPACE_CLEANUP_LAST_FAILURES.set(summary.failures) + for kind, planned, deleted in ( + ("pvc", summary.pvcs_planned, summary.pvcs_deleted), + ("pv", summary.pvs_planned, summary.pvs_deleted), + ("longhorn_volume", summary.volumes_planned, summary.volumes_deleted), + ): + if planned: + JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels(kind=kind, action="planned", mode=mode).inc(planned) + if deleted: + JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels(kind=kind, action="deleted", mode=mode).inc(deleted) + if summary.skipped: + JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels( + kind="cleanup", + action="skipped", + mode=mode, + ).inc(summary.skipped) + if summary.failures: + JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels( + kind="cleanup", + action="failed", + mode=mode, + ).inc(summary.failures) + + def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary: """Delete stale Jenkins workspace PVC/PV artifacts and orphan Longhorn volumes.""" namespace = settings.jenkins_workspace_namespace - prefix = settings.jenkins_workspace_pvc_prefix + dry_run = settings.jenkins_workspace_cleanup_dry_run pvs_deleted = 0 pvcs_deleted = 0 volumes_deleted = 0 skipped = 0 failures = 0 + stale_pvs: list[_CleanupCandidate] = [] + stale_pvcs: list[_CleanupCandidate] = [] + stale_volumes: list[_CleanupCandidate] = [] - active_claims = _active_workspace_claims() - stale_pvs, all_pv_names = _workspace_pv_candidates(active_claims) - stale_pvcs = _workspace_pvc_candidates(active_claims) - removed_pv_names: set[str] = set() + summary: JenkinsWorkspaceCleanupSummary + try: + active_claims = _active_workspace_claims() + stale_pvs, all_pv_names = _workspace_pv_candidates(active_claims) + stale_pvcs = _workspace_pvc_candidates(active_claims) + removed_pv_names: set[str] = set() - for pvc in stale_pvcs: - metadata = pvc.get("metadata") if isinstance(pvc.get("metadata"), dict) else {} - claim_name = metadata.get("name") - if not isinstance(claim_name, str) or not claim_name: - skipped += 1 - continue - try: - delete_json(f"/api/v1/namespaces/{namespace}/persistentvolumeclaims/{claim_name}") - pvcs_deleted += 1 - except Exception as exc: - failures += 1 + if dry_run: logger.info( - "jenkins workspace pvc delete failed", - extra={"event": "jenkins_workspace_cleanup", "claim": claim_name, "detail": str(exc)}, + "jenkins workspace cleanup dry-run enabled", + extra={ + "event": "jenkins_workspace_cleanup", + "status": "dry_run", + "namespace": namespace, + "dry_run": True, + "planned_pvs": len(stale_pvs), + "planned_pvcs": len(stale_pvcs), + }, ) - for pv in stale_pvs: - metadata = pv.get("metadata") if isinstance(pv.get("metadata"), dict) else {} - pv_name = metadata.get("name") - if not isinstance(pv_name, str) or not pv_name: - skipped += 1 - continue - try: - delete_json(f"/api/v1/persistentvolumes/{pv_name}") - removed_pv_names.add(pv_name) - pvs_deleted += 1 - except Exception as exc: - failures += 1 - logger.info( - "jenkins workspace pv delete failed", - extra={"event": "jenkins_workspace_cleanup", "pv": pv_name, "detail": str(exc)}, - ) + for pvc in stale_pvcs: + claim_name = pvc.name + if not claim_name: + skipped += 1 + continue + if dry_run: + pvcs_deleted += 1 + continue + try: + delete_json(pvc.path) + pvcs_deleted += 1 + except Exception as exc: + failures += 1 + logger.info( + "jenkins workspace pvc delete failed", + extra={"event": "jenkins_workspace_cleanup", "claim": claim_name, "detail": str(exc)}, + ) - payload = get_json("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes") - items = payload.get("items") if isinstance(payload.get("items"), list) else [] - for volume in items: - if not isinstance(volume, dict): - continue - metadata = volume.get("metadata") if isinstance(volume.get("metadata"), dict) else {} - name = metadata.get("name") - if not isinstance(name, str) or not name: - skipped += 1 - continue - should_delete = name in removed_pv_names - if not should_delete: - labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {} - pvc_name = labels.get("kubernetes.io/created-for/pvc/name") - should_delete = ( - isinstance(pvc_name, str) - and pvc_name.startswith(prefix) - and name not in all_pv_names - ) - if not should_delete: - continue - if not _is_old_enough(metadata): - continue - try: - delete_json(f"/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/{name}") - volumes_deleted += 1 - except Exception as exc: - failures += 1 - logger.info( - "jenkins workspace longhorn volume delete failed", - extra={"event": "jenkins_workspace_cleanup", "volume": name, "detail": str(exc)}, - ) + for pv in stale_pvs: + pv_name = pv.name + if not pv_name: + skipped += 1 + continue + if dry_run: + pvs_deleted += 1 + removed_pv_names.add(pv_name) + continue + try: + delete_json(pv.path) + removed_pv_names.add(pv_name) + pvs_deleted += 1 + except Exception as exc: + failures += 1 + logger.info( + "jenkins workspace pv delete failed", + extra={"event": "jenkins_workspace_cleanup", "pv": pv_name, "detail": str(exc)}, + ) - return JenkinsWorkspaceCleanupSummary( - pvs_deleted=pvs_deleted, - pvcs_deleted=pvcs_deleted, - volumes_deleted=volumes_deleted, - skipped=skipped, - failures=failures, + stale_volumes = _workspace_longhorn_candidates(all_pv_names, removed_pv_names) + for volume in stale_volumes: + if not volume.name: + skipped += 1 + continue + if dry_run: + volumes_deleted += 1 + continue + try: + delete_json(volume.path) + volumes_deleted += 1 + except Exception as exc: + failures += 1 + logger.info( + "jenkins workspace longhorn volume delete failed", + extra={"event": "jenkins_workspace_cleanup", "volume": volume.name, "detail": str(exc)}, + ) + + summary = JenkinsWorkspaceCleanupSummary( + pvs_planned=len(stale_pvs), + pvcs_planned=len(stale_pvcs), + volumes_planned=len(stale_volumes), + pvs_deleted=pvs_deleted, + pvcs_deleted=pvcs_deleted, + volumes_deleted=volumes_deleted, + skipped=skipped, + failures=failures, + dry_run=dry_run, + ) + except Exception as exc: + failures += 1 + logger.exception( + "jenkins workspace cleanup failed", + extra={"event": "jenkins_workspace_cleanup", "status": "error", "namespace": namespace, "detail": str(exc)}, + ) + summary = JenkinsWorkspaceCleanupSummary( + pvs_planned=len(stale_pvs), + pvcs_planned=len(stale_pvcs), + volumes_planned=len(stale_volumes), + pvs_deleted=pvs_deleted, + pvcs_deleted=pvcs_deleted, + volumes_deleted=volumes_deleted, + skipped=skipped, + failures=failures, + dry_run=dry_run, + ) + _record_metrics(summary) + raise + _record_metrics(summary) + logger.info( + "jenkins workspace cleanup finished", + extra={ + "event": "jenkins_workspace_cleanup", + "status": "ok" if failures == 0 else "error", + "dry_run": dry_run, + "namespace": namespace, + "planned_pvs": summary.pvs_planned, + "planned_pvcs": summary.pvcs_planned, + "planned_volumes": summary.volumes_planned, + "deleted_pvs": pvs_deleted, + "deleted_pvcs": pvcs_deleted, + "deleted_volumes": volumes_deleted, + "skipped": skipped, + "failures": failures, + }, ) + return summary diff --git a/ariadne/settings.py b/ariadne/settings.py index 239837b..8d4712d 100644 --- a/ariadne/settings.py +++ b/ariadne/settings.py @@ -171,6 +171,7 @@ class Settings: jenkins_workspace_namespace: str jenkins_workspace_pvc_prefix: str jenkins_workspace_cleanup_min_age_hours: float + jenkins_workspace_cleanup_dry_run: bool vaultwarden_namespace: str vaultwarden_pod_label: str @@ -469,6 +470,7 @@ class Settings: "jenkins_workspace_namespace": _env("JENKINS_WORKSPACE_NAMESPACE", "jenkins"), "jenkins_workspace_pvc_prefix": _env("JENKINS_WORKSPACE_PVC_PREFIX", "pvc-workspace-"), "jenkins_workspace_cleanup_min_age_hours": _env_float("JENKINS_WORKSPACE_CLEANUP_MIN_AGE_HOURS", 12.0), + "jenkins_workspace_cleanup_dry_run": _env_bool("JENKINS_WORKSPACE_CLEANUP_DRY_RUN", "false"), } @classmethod diff --git a/testing/test_jenkins_workspace_cleanup.py b/testing/test_jenkins_workspace_cleanup.py index d6d521b..da109fb 100644 --- a/testing/test_jenkins_workspace_cleanup.py +++ b/testing/test_jenkins_workspace_cleanup.py @@ -3,101 +3,200 @@ from __future__ import annotations from datetime import datetime, timezone import types +from prometheus_client import REGISTRY + from ariadne.services import jenkins_workspace_cleanup as cleanup_module -def test_cleanup_jenkins_workspace_storage(monkeypatch) -> None: - dummy_settings = types.SimpleNamespace( +def _metric_value(name: str, labels: dict[str, str]) -> float: + value = REGISTRY.get_sample_value(name, labels) + return float(value) if value is not None else 0.0 + + +def _dummy_settings(*, dry_run: bool) -> types.SimpleNamespace: + return types.SimpleNamespace( jenkins_workspace_namespace="jenkins", jenkins_workspace_pvc_prefix="pvc-workspace-", jenkins_workspace_cleanup_min_age_hours=1.0, + jenkins_workspace_cleanup_dry_run=dry_run, ) - monkeypatch.setattr(cleanup_module, "settings", dummy_settings) + + +def _fake_payloads(now_iso: str, old_iso: str) -> dict[str, dict[str, object]]: + return { + "/api/v1/namespaces/jenkins/pods": { + "items": [ + { + "metadata": { + "annotations": { + "jenkins.io/workspace-pvc": "pvc-workspace-annotated-active", + } + }, + "spec": { + "volumes": [ + {"persistentVolumeClaim": {"claimName": "pvc-workspace-active"}}, + ] + }, + } + ] + }, + "/api/v1/namespaces/jenkins/persistentvolumeclaims": { + "items": [ + { + "metadata": {"name": "pvc-workspace-stale", "creationTimestamp": old_iso}, + "status": {"phase": "Lost"}, + }, + { + "metadata": {"name": "pvc-workspace-active", "creationTimestamp": old_iso}, + "status": {"phase": "Bound"}, + }, + { + "metadata": {"name": "pvc-workspace-annotated-active", "creationTimestamp": old_iso}, + "status": {"phase": "Lost"}, + }, + { + "metadata": {"name": "pvc-workspace-fresh", "creationTimestamp": now_iso}, + "status": {"phase": "Lost"}, + }, + ] + }, + "/api/v1/persistentvolumes": { + "items": [ + { + "metadata": {"name": "pvc-old", "creationTimestamp": old_iso}, + "status": {"phase": "Released"}, + "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-stale"}}, + }, + { + "metadata": {"name": "pvc-active", "creationTimestamp": old_iso}, + "status": {"phase": "Released"}, + "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-active"}}, + }, + { + "metadata": {"name": "pvc-annotated", "creationTimestamp": old_iso}, + "status": {"phase": "Released"}, + "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-annotated-active"}}, + }, + { + "metadata": {"name": "pvc-fresh", "creationTimestamp": now_iso}, + "status": {"phase": "Released"}, + "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-fresh"}}, + }, + ] + }, + "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes": { + "items": [ + {"metadata": {"name": "pvc-old", "creationTimestamp": old_iso}}, + { + "metadata": { + "name": "pvc-orphan", + "creationTimestamp": old_iso, + "labels": { + "kubernetes.io/created-for/pvc/name": "pvc-workspace-orphan", + }, + } + }, + { + "metadata": { + "name": "pvc-attached", + "creationTimestamp": old_iso, + "labels": { + "kubernetes.io/created-for/pvc/name": "pvc-workspace-annotated-active", + }, + }, + "status": {"state": "attached", "isAttached": True, "robustness": "healthy"}, + "spec": {"frontend": "blockdev"}, + }, + { + "metadata": { + "name": "pvc-orphan-fresh", + "creationTimestamp": now_iso, + "labels": { + "kubernetes.io/created-for/pvc/name": "pvc-workspace-fresh", + }, + } + }, + ] + }, + } + + +def test_cleanup_jenkins_workspace_storage_dry_run(monkeypatch) -> None: + monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=True)) now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") old_iso = "2020-01-01T00:00:00Z" + payloads = _fake_payloads(now_iso, old_iso) deleted_paths: list[str] = [] def fake_get_json(path: str): - if path == "/api/v1/namespaces/jenkins/pods": - return { - "items": [ - { - "spec": { - "volumes": [ - {"persistentVolumeClaim": {"claimName": "pvc-workspace-active"}}, - ] - } - } - ] - } - if path == "/api/v1/namespaces/jenkins/persistentvolumeclaims": - return { - "items": [ - { - "metadata": {"name": "pvc-workspace-stale", "creationTimestamp": old_iso}, - "status": {"phase": "Lost"}, - }, - { - "metadata": {"name": "pvc-workspace-active", "creationTimestamp": old_iso}, - "status": {"phase": "Bound"}, - }, - { - "metadata": {"name": "pvc-workspace-fresh", "creationTimestamp": now_iso}, - "status": {"phase": "Lost"}, - }, - ] - } - if path == "/api/v1/persistentvolumes": - return { - "items": [ - { - "metadata": {"name": "pvc-old", "creationTimestamp": old_iso}, - "status": {"phase": "Released"}, - "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-stale"}}, - }, - { - "metadata": {"name": "pvc-active", "creationTimestamp": old_iso}, - "status": {"phase": "Released"}, - "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-active"}}, - }, - { - "metadata": {"name": "pvc-fresh", "creationTimestamp": now_iso}, - "status": {"phase": "Released"}, - "spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-fresh"}}, - }, - ] - } - if path == "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes": - return { - "items": [ - {"metadata": {"name": "pvc-old", "creationTimestamp": old_iso}}, - { - "metadata": { - "name": "pvc-orphan", - "creationTimestamp": old_iso, - "labels": { - "kubernetes.io/created-for/pvc/name": "pvc-workspace-orphan", - }, - } - }, - { - "metadata": { - "name": "pvc-orphan-fresh", - "creationTimestamp": now_iso, - "labels": { - "kubernetes.io/created-for/pvc/name": "pvc-workspace-fresh", - }, - } - }, - ] - } + if path in payloads: + return payloads[path] raise AssertionError(f"unexpected path: {path}") def fake_delete_json(path: str): deleted_paths.append(path) return {"status": "Success"} + before_runs = _metric_value( + "ariadne_jenkins_workspace_cleanup_runs_total", + {"status": "ok", "mode": "dry_run"}, + ) + before_planned = _metric_value( + "ariadne_jenkins_workspace_cleanup_objects_total", + {"kind": "pvc", "action": "planned", "mode": "dry_run"}, + ) + + monkeypatch.setattr(cleanup_module, "get_json", fake_get_json) + monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json) + + summary = cleanup_module.cleanup_jenkins_workspace_storage() + + assert summary.dry_run is True + assert summary.pvcs_planned == 1 + assert summary.pvs_planned == 1 + assert summary.volumes_planned == 2 + assert summary.pvcs_deleted == 1 + assert summary.pvs_deleted == 1 + assert summary.volumes_deleted == 2 + assert summary.failures == 0 + assert deleted_paths == [] + assert _metric_value( + "ariadne_jenkins_workspace_cleanup_runs_total", + {"status": "ok", "mode": "dry_run"}, + ) == before_runs + 1 + assert _metric_value( + "ariadne_jenkins_workspace_cleanup_objects_total", + {"kind": "pvc", "action": "planned", "mode": "dry_run"}, + ) == before_planned + 1 + + +def test_cleanup_jenkins_workspace_storage(monkeypatch) -> None: + monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False)) + + now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + old_iso = "2020-01-01T00:00:00Z" + deleted_paths: list[str] = [] + payloads = _fake_payloads(now_iso, old_iso) + + def fake_get_json(path: str): + if path in payloads: + return payloads[path] + raise AssertionError(f"unexpected path: {path}") + + def fake_delete_json(path: str): + deleted_paths.append(path) + return {"status": "Success"} + + before_runs = _metric_value( + "ariadne_jenkins_workspace_cleanup_runs_total", + {"status": "ok", "mode": "delete"}, + ) + before_deleted = _metric_value( + "ariadne_jenkins_workspace_cleanup_objects_total", + {"kind": "longhorn_volume", "action": "deleted", "mode": "delete"}, + ) + monkeypatch.setattr(cleanup_module, "get_json", fake_get_json) monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json) @@ -111,15 +210,19 @@ def test_cleanup_jenkins_workspace_storage(monkeypatch) -> None: assert "/api/v1/persistentvolumes/pvc-old" in deleted_paths assert "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-old" in deleted_paths assert "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-orphan" in deleted_paths + assert "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-attached" not in deleted_paths + assert _metric_value( + "ariadne_jenkins_workspace_cleanup_runs_total", + {"status": "ok", "mode": "delete"}, + ) == before_runs + 1 + assert _metric_value( + "ariadne_jenkins_workspace_cleanup_objects_total", + {"kind": "longhorn_volume", "action": "deleted", "mode": "delete"}, + ) == before_deleted + 2 def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None: - dummy_settings = types.SimpleNamespace( - jenkins_workspace_namespace="jenkins", - jenkins_workspace_pvc_prefix="pvc-workspace-", - jenkins_workspace_cleanup_min_age_hours=1.0, - ) - monkeypatch.setattr(cleanup_module, "settings", dummy_settings) + monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False)) def fake_get_json(path: str): if path == "/api/v1/namespaces/jenkins/pods": @@ -142,9 +245,18 @@ def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None: def fake_delete_json(_path: str): raise RuntimeError("boom") + before_failures = _metric_value( + "ariadne_jenkins_workspace_cleanup_objects_total", + {"kind": "cleanup", "action": "failed", "mode": "delete"}, + ) + monkeypatch.setattr(cleanup_module, "get_json", fake_get_json) monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json) summary = cleanup_module.cleanup_jenkins_workspace_storage() assert summary.failures == 1 assert summary.pvcs_deleted == 0 + assert _metric_value( + "ariadne_jenkins_workspace_cleanup_objects_total", + {"kind": "cleanup", "action": "failed", "mode": "delete"}, + ) == before_failures + 1