cleanup(jenkins): cap per-run deletions without deadlock

ariadne/services/jenkins_workspace_cleanup.py

@@ -371,6 +371,7 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
         removed_pv_names: set[str] = set()
         stale_volumes = _workspace_longhorn_candidates(all_pv_names, removed_pv_names)
         planned_total = len(stale_pvs) + len(stale_pvcs) + len(stale_volumes)
+        deletion_budget: int | None = None
 
         if dry_run:
             logger.info(
@@ -386,13 +387,14 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
                     "max_deletions": max_deletions,
                 },
             )
-        elif planned_total > max_deletions:
+        else:
-            failures += 1
+            deletion_budget = max_deletions
+        if not dry_run and planned_total > max_deletions:
             logger.warning(
-                "jenkins workspace cleanup blocked by max deletions guard",
+                "jenkins workspace cleanup capped by max deletions guard",
                 extra={
                     "event": "jenkins_workspace_cleanup",
-                    "status": "guard_blocked",
+                    "status": "guard_capped",
                     "namespace": namespace,
                     "dry_run": False,
                     "planned_total": planned_total,
@@ -402,19 +404,6 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
                     "planned_volumes": len(stale_volumes),
                 },
             )
-            summary = JenkinsWorkspaceCleanupSummary(
-                pvs_planned=len(stale_pvs),
-                pvcs_planned=len(stale_pvcs),
-                volumes_planned=len(stale_volumes),
-                pvs_deleted=0,
-                pvcs_deleted=0,
-                volumes_deleted=0,
-                skipped=skipped,
-                failures=failures,
-                dry_run=dry_run,
-            )
-            _record_metrics(summary)
-            return summary
 
         for pvc in stale_pvcs:
             claim_name = pvc.name
@@ -423,6 +412,11 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
                 continue
             if dry_run:
                 continue
+            if deletion_budget is not None and deletion_budget <= 0:
+                skipped += 1
+                continue
+            if deletion_budget is not None:
+                deletion_budget -= 1
             try:
                 delete_json(pvc.path)
                 pvcs_deleted += 1
@@ -440,6 +434,11 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
                 continue
             if dry_run:
                 continue
+            if deletion_budget is not None and deletion_budget <= 0:
+                skipped += 1
+                continue
+            if deletion_budget is not None:
+                deletion_budget -= 1
             try:
                 delete_json(pv.path)
                 removed_pv_names.add(pv_name)
@@ -459,6 +458,11 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
                 continue
             if dry_run:
                 continue
+            if deletion_budget is not None and deletion_budget <= 0:
+                skipped += 1
+                continue
+            if deletion_budget is not None:
+                deletion_budget -= 1
             try:
                 delete_json(volume.path)
                 volumes_deleted += 1

testing/test_jenkins_workspace_cleanup.py

@@ -290,7 +290,7 @@ def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None:
     ) == before_failures + 1
 
 
-def test_cleanup_jenkins_workspace_storage_guard_blocks_mass_delete(monkeypatch) -> None:
+def test_cleanup_jenkins_workspace_storage_guard_caps_mass_delete(monkeypatch) -> None:
     monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False, max_deletions=1))
 
     now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@@ -312,11 +312,12 @@ def test_cleanup_jenkins_workspace_storage_guard_blocks_mass_delete(monkeypatch)
 
     summary = cleanup_module.cleanup_jenkins_workspace_storage()
 
-    assert summary.failures == 1
+    assert summary.failures == 0
     assert summary.pvcs_planned == 1
     assert summary.pvs_planned == 1
-    assert summary.volumes_planned == 2
+    assert summary.volumes_planned == 1
-    assert summary.pvcs_deleted == 0
+    assert summary.pvcs_deleted == 1
     assert summary.pvs_deleted == 0
     assert summary.volumes_deleted == 0
-    assert deleted_paths == []
+    assert summary.skipped == 2
+    assert deleted_paths == ["/api/v1/namespaces/jenkins/persistentvolumeclaims/pvc-workspace-stale"]