bstein/ariadne
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup(jenkins): cap per-run deletions without deadlock

Browse Source
This commit is contained in:
Brad Stein 2026-04-12 11:36:50 -03:00
parent c86948b4e5
commit 81a6dc629f
2 changed files with 27 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
ariadne/services/jenkins_workspace_cleanup.py
View File

@ -371,6 +371,7 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
removed_pv_names: set[str] = set()
stale_volumes = _workspace_longhorn_candidates(all_pv_names, removed_pv_names)
planned_total = len(stale_pvs) + len(stale_pvcs) + len(stale_volumes)
deletion_budget: int | None = None
if dry_run:
logger.info(
@ -386,13 +387,14 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
"max_deletions": max_deletions,
},
)
elif planned_total > max_deletions:
failures += 1
else:
deletion_budget = max_deletions
if not dry_run and planned_total > max_deletions:
logger.warning(
"jenkins workspace cleanup blocked by max deletions guard",
"jenkins workspace cleanup capped by max deletions guard",
extra={
"event": "jenkins_workspace_cleanup",
"status": "guard_blocked",
"status": "guard_capped",
"namespace": namespace,
"dry_run": False,
"planned_total": planned_total,
@ -402,19 +404,6 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
"planned_volumes": len(stale_volumes),
},
)
summary = JenkinsWorkspaceCleanupSummary(
pvs_planned=len(stale_pvs),
pvcs_planned=len(stale_pvcs),
volumes_planned=len(stale_volumes),
pvs_deleted=0,
pvcs_deleted=0,
volumes_deleted=0,
skipped=skipped,
failures=failures,
dry_run=dry_run,
)
_record_metrics(summary)
return summary
for pvc in stale_pvcs:
claim_name = pvc.name
@ -423,6 +412,11 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
continue
if dry_run:
continue
if deletion_budget is not None and deletion_budget <= 0:
skipped += 1
continue
if deletion_budget is not None:
deletion_budget -= 1
try:
delete_json(pvc.path)
pvcs_deleted += 1
@ -440,6 +434,11 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
continue
if dry_run:
continue
if deletion_budget is not None and deletion_budget <= 0:
skipped += 1
continue
if deletion_budget is not None:
deletion_budget -= 1
try:
delete_json(pv.path)
removed_pv_names.add(pv_name)
@ -459,6 +458,11 @@ def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
continue
if dry_run:
continue
if deletion_budget is not None and deletion_budget <= 0:
skipped += 1
continue
if deletion_budget is not None:
deletion_budget -= 1
try:
delete_json(volume.path)
volumes_deleted += 1

11
testing/test_jenkins_workspace_cleanup.py
View File

@ -290,7 +290,7 @@ def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None:
) == before_failures + 1
def test_cleanup_jenkins_workspace_storage_guard_blocks_mass_delete(monkeypatch) -> None:
def test_cleanup_jenkins_workspace_storage_guard_caps_mass_delete(monkeypatch) -> None:
monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False, max_deletions=1))
now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
@ -312,11 +312,12 @@ def test_cleanup_jenkins_workspace_storage_guard_blocks_mass_delete(monkeypatch)
summary = cleanup_module.cleanup_jenkins_workspace_storage()
assert summary.failures == 1
assert summary.failures == 0
assert summary.pvcs_planned == 1
assert summary.pvs_planned == 1
assert summary.volumes_planned == 2
assert summary.pvcs_deleted == 0
assert summary.volumes_planned == 1
assert summary.pvcs_deleted == 1
assert summary.pvs_deleted == 0
assert summary.volumes_deleted == 0
assert deleted_paths == []
assert summary.skipped == 2
assert deleted_paths == ["/api/v1/namespaces/jenkins/persistentvolumeclaims/pvc-workspace-stale"]