cleanup(jenkins): detect longhorn orphan volumes via kubernetes status

ariadne/services/jenkins_workspace_cleanup.py

@@ -93,6 +93,13 @@ class _CleanupCandidate:
     pv_name: str | None = None
 
 
+@dataclass(frozen=True)
+class _LonghornBinding:
+    pvc_name: Any
+    pvc_namespace: Any
+    referenced_pv_name: Any
+
+
 def _parse_timestamp(raw: str) -> datetime | None:
     """Parse Kubernetes RFC3339 timestamps into timezone-aware datetimes."""
 
@@ -245,6 +252,47 @@ def _workspace_pvc_candidates(active_claims: set[str]) -> list[_CleanupCandidate
     return candidates
 
 
+def _workspace_binding_from_longhorn(
+    metadata: dict[str, Any],
+    status: dict[str, Any],
+) -> _LonghornBinding:
+    labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {}
+    kubernetes_status = status.get("kubernetesStatus") if isinstance(status.get("kubernetesStatus"), dict) else {}
+    pvc_name = labels.get("kubernetes.io/created-for/pvc/name")
+    if not isinstance(pvc_name, str) or not pvc_name:
+        pvc_name = kubernetes_status.get("pvcName")
+    pvc_namespace = labels.get("kubernetes.io/created-for/pvc/namespace")
+    if not isinstance(pvc_namespace, str) or not pvc_namespace:
+        pvc_namespace = kubernetes_status.get("namespace")
+    referenced_pv_name = kubernetes_status.get("pvName")
+    return _LonghornBinding(
+        pvc_name=pvc_name,
+        pvc_namespace=pvc_namespace,
+        referenced_pv_name=referenced_pv_name,
+    )
+
+
+def _should_delete_longhorn_volume(
+    name: str,
+    binding: _LonghornBinding,
+    all_pv_names: set[str],
+    removed_pv_names: set[str],
+) -> bool:
+    if name in removed_pv_names or binding.referenced_pv_name in removed_pv_names:
+        return True
+    if not _is_workspace_name(binding.pvc_name):
+        return False
+    if (
+        isinstance(binding.referenced_pv_name, str)
+        and binding.referenced_pv_name in all_pv_names
+    ) or name in all_pv_names:
+        return False
+    return (
+        binding.pvc_namespace in {None, ""}
+        or binding.pvc_namespace == settings.jenkins_workspace_namespace
+    )
+
+
 def _workspace_longhorn_candidates(all_pv_names: set[str], removed_pv_names: set[str]) -> list[_CleanupCandidate]:
     namespace = "longhorn-system"
     payload = get_json("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes")
@@ -261,26 +309,17 @@ def _workspace_longhorn_candidates(all_pv_names: set[str], removed_pv_names: set
         if not isinstance(name, str) or not name:
             continue
 
-        labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {}
+        binding = _workspace_binding_from_longhorn(metadata, status)
-        pvc_name = labels.get("kubernetes.io/created-for/pvc/name")
-        pvc_namespace = labels.get("kubernetes.io/created-for/pvc/namespace")
         robust_state = status.get("robustness")
         state = status.get("state")
         attached = status.get("isAttached")
         frontend = spec.get("frontend")
-        should_delete = False
+        if not _should_delete_longhorn_volume(
-        if name in removed_pv_names:
+            name,
-            should_delete = True
+            binding,
-        elif (
+            all_pv_names,
-            _is_workspace_name(pvc_name)
+            removed_pv_names,
-            and name not in all_pv_names
-            and (
-                pvc_namespace in {None, ""}
-                or pvc_namespace == settings.jenkins_workspace_namespace
-            )
         ):
-            should_delete = True
-        if not should_delete:
             continue
         if _is_deleting(metadata):
             continue

tests/test_jenkins_workspace_cleanup.py

@@ -305,6 +305,56 @@ def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None:
     ) == before_failures + 1
 
 
+def test_cleanup_jenkins_workspace_storage_uses_longhorn_kubernetes_status(monkeypatch) -> None:
+    monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False))
+    deleted_paths: list[str] = []
+
+    def fake_get_json(path: str):
+        if path == "/api/v1/namespaces/jenkins/pods":
+            return {"items": []}
+        if path == "/api/v1/namespaces/jenkins/persistentvolumeclaims":
+            return {"items": []}
+        if path == "/api/v1/persistentvolumes":
+            return {"items": []}
+        if path == "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes":
+            return {
+                "items": [
+                    {
+                        "metadata": {
+                            "name": "pvc-orphan-kstatus",
+                            "creationTimestamp": "2020-01-01T00:00:00Z",
+                        },
+                        "status": {
+                            "state": "detached",
+                            "isAttached": False,
+                            "robustness": "unknown",
+                            "kubernetesStatus": {
+                                "namespace": "jenkins",
+                                "pvcName": "pvc-workspace-kstatus",
+                                "pvName": "pvc-orphan-kstatus",
+                            },
+                        },
+                        "spec": {"frontend": "blockdev"},
+                    }
+                ]
+            }
+        raise AssertionError(f"unexpected path: {path}")
+
+    def fake_delete_json(path: str):
+        deleted_paths.append(path)
+        return {"status": "Success"}
+
+    monkeypatch.setattr(cleanup_module, "get_json", fake_get_json)
+    monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json)
+
+    summary = cleanup_module.cleanup_jenkins_workspace_storage()
+
+    assert summary.volumes_planned == 1
+    assert summary.volumes_deleted == 1
+    assert summary.failures == 0
+    assert deleted_paths == ["/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-orphan-kstatus"]
+
+
 def test_cleanup_jenkins_workspace_storage_guard_caps_mass_delete(monkeypatch) -> None:
     monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False, max_deletions=1))