cleanup(jenkins): harden pvc cleanup with dry-run metrics
This commit is contained in:
parent
ad99a83a98
commit
2ff3686700
@ -389,6 +389,8 @@ def _startup() -> None:
|
||||
"metis_k3s_token_sync_cron": settings.metis_k3s_token_sync_cron,
|
||||
"platform_quality_suite_probe_cron": settings.platform_quality_suite_probe_cron,
|
||||
"jenkins_workspace_cleanup_cron": settings.jenkins_workspace_cleanup_cron,
|
||||
"jenkins_workspace_cleanup_dry_run": settings.jenkins_workspace_cleanup_dry_run,
|
||||
"jenkins_workspace_cleanup_max_deletions_per_run": settings.jenkins_workspace_cleanup_max_deletions_per_run,
|
||||
"vault_k8s_auth_cron": settings.vault_k8s_auth_cron,
|
||||
"vault_oidc_cron": settings.vault_oidc_cron,
|
||||
"comms_guest_name_cron": settings.comms_guest_name_cron,
|
||||
|
||||
@ -4,6 +4,8 @@ from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
from prometheus_client import Counter, Gauge
|
||||
|
||||
from ..k8s.client import delete_json, get_json
|
||||
from ..settings import settings
|
||||
from ..utils.logging import get_logger
|
||||
@ -12,6 +14,48 @@ from ..utils.logging import get_logger
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
JENKINS_WORKSPACE_CLEANUP_RUNS_TOTAL = Counter(
|
||||
"ariadne_jenkins_workspace_cleanup_runs_total",
|
||||
"Jenkins workspace cleanup runs by status and mode",
|
||||
["status", "mode"],
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL = Counter(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
"Jenkins workspace cleanup objects by kind, action, and mode",
|
||||
["kind", "action", "mode"],
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_RUN_TS = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_run_timestamp_seconds",
|
||||
"Last Jenkins workspace cleanup run timestamp",
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_SUCCESS_TS = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_success_timestamp_seconds",
|
||||
"Last successful Jenkins workspace cleanup timestamp",
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_FAILURE_TS = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_failure_timestamp_seconds",
|
||||
"Last failed Jenkins workspace cleanup timestamp",
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_DELETED = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_deleted_total",
|
||||
"Last Jenkins workspace cleanup deleted object count",
|
||||
["kind"],
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_planned_total",
|
||||
"Last Jenkins workspace cleanup planned object count",
|
||||
["kind"],
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_SKIPPED = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_skipped_total",
|
||||
"Last Jenkins workspace cleanup skipped object count",
|
||||
)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_FAILURES = Gauge(
|
||||
"ariadne_jenkins_workspace_cleanup_last_failures_total",
|
||||
"Last Jenkins workspace cleanup failure count",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JenkinsWorkspaceCleanupSummary:
|
||||
"""Summarize one Jenkins workspace-storage cleanup pass.
|
||||
@ -20,11 +64,33 @@ class JenkinsWorkspaceCleanupSummary:
|
||||
Outputs: deterministic counters for operator logs and metrics.
|
||||
"""
|
||||
|
||||
pvs_planned: int
|
||||
pvcs_planned: int
|
||||
volumes_planned: int
|
||||
pvs_deleted: int
|
||||
pvcs_deleted: int
|
||||
volumes_deleted: int
|
||||
skipped: int
|
||||
failures: int
|
||||
dry_run: bool
|
||||
|
||||
@property
|
||||
def planned(self) -> int:
|
||||
return self.pvs_planned + self.pvcs_planned + self.volumes_planned
|
||||
|
||||
@property
|
||||
def deleted(self) -> int:
|
||||
return self.pvs_deleted + self.pvcs_deleted + self.volumes_deleted
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _CleanupCandidate:
|
||||
name: str
|
||||
kind: str
|
||||
path: str
|
||||
created_at: datetime | None
|
||||
related_pvc: str | None = None
|
||||
pv_name: str | None = None
|
||||
|
||||
|
||||
def _parse_timestamp(raw: str) -> datetime | None:
|
||||
@ -37,30 +103,44 @@ def _parse_timestamp(raw: str) -> datetime | None:
|
||||
return None
|
||||
|
||||
|
||||
def _created_at(metadata: dict[str, Any]) -> datetime | None:
|
||||
raw = metadata.get("creationTimestamp")
|
||||
if not isinstance(raw, str) or not raw:
|
||||
return None
|
||||
return _parse_timestamp(raw)
|
||||
|
||||
|
||||
def _is_old_enough(metadata: dict[str, Any]) -> bool:
|
||||
"""Return true when an object age exceeds the configured cleanup threshold."""
|
||||
|
||||
raw = metadata.get("creationTimestamp")
|
||||
if not isinstance(raw, str) or not raw:
|
||||
return True
|
||||
created_at = _parse_timestamp(raw)
|
||||
created_at = _created_at(metadata)
|
||||
if created_at is None:
|
||||
return True
|
||||
return False
|
||||
min_age = timedelta(hours=settings.jenkins_workspace_cleanup_min_age_hours)
|
||||
return datetime.now(timezone.utc) - created_at >= min_age
|
||||
|
||||
|
||||
def _is_deleting(metadata: dict[str, Any]) -> bool:
|
||||
deletion_ts = metadata.get("deletionTimestamp")
|
||||
return isinstance(deletion_ts, str) and bool(deletion_ts.strip())
|
||||
|
||||
|
||||
def _is_workspace_name(name: Any) -> bool:
|
||||
return isinstance(name, str) and name.startswith(settings.jenkins_workspace_pvc_prefix)
|
||||
|
||||
|
||||
def _active_workspace_claims() -> set[str]:
|
||||
"""Collect currently referenced Jenkins workspace PVC names from pods."""
|
||||
|
||||
namespace = settings.jenkins_workspace_namespace
|
||||
prefix = settings.jenkins_workspace_pvc_prefix
|
||||
payload = get_json(f"/api/v1/namespaces/{namespace}/pods")
|
||||
items = payload.get("items") if isinstance(payload.get("items"), list) else []
|
||||
active: set[str] = set()
|
||||
for pod in items:
|
||||
if not isinstance(pod, dict):
|
||||
continue
|
||||
metadata = pod.get("metadata") if isinstance(pod.get("metadata"), dict) else {}
|
||||
annotations = metadata.get("annotations") if isinstance(metadata.get("annotations"), dict) else {}
|
||||
spec = pod.get("spec") if isinstance(pod.get("spec"), dict) else {}
|
||||
volumes = spec.get("volumes") if isinstance(spec.get("volumes"), list) else []
|
||||
for volume in volumes:
|
||||
@ -70,19 +150,21 @@ def _active_workspace_claims() -> set[str]:
|
||||
if not isinstance(claim, dict):
|
||||
continue
|
||||
claim_name = claim.get("claimName")
|
||||
if isinstance(claim_name, str) and claim_name.startswith(prefix):
|
||||
if _is_workspace_name(claim_name):
|
||||
active.add(claim_name)
|
||||
claim_name = annotations.get("jenkins.io/workspace-pvc")
|
||||
if _is_workspace_name(claim_name):
|
||||
active.add(claim_name)
|
||||
return active
|
||||
|
||||
|
||||
def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[dict[str, Any]], set[str]]:
|
||||
def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[_CleanupCandidate], set[str]]:
|
||||
"""Find releasable Jenkins workspace PVs and keep a set of all PV names."""
|
||||
|
||||
namespace = settings.jenkins_workspace_namespace
|
||||
prefix = settings.jenkins_workspace_pvc_prefix
|
||||
payload = get_json("/api/v1/persistentvolumes")
|
||||
items = payload.get("items") if isinstance(payload.get("items"), list) else []
|
||||
candidates: list[dict[str, Any]] = []
|
||||
candidates: list[_CleanupCandidate] = []
|
||||
all_pv_names: set[str] = set()
|
||||
|
||||
for pv in items:
|
||||
@ -101,7 +183,9 @@ def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[dict[str, An
|
||||
phase = status.get("phase")
|
||||
if claim_namespace != namespace:
|
||||
continue
|
||||
if not isinstance(claim_name, str) or not claim_name.startswith(prefix):
|
||||
if not _is_workspace_name(claim_name):
|
||||
continue
|
||||
if _is_deleting(metadata):
|
||||
continue
|
||||
if claim_name in active_claims:
|
||||
continue
|
||||
@ -109,18 +193,27 @@ def _workspace_pv_candidates(active_claims: set[str]) -> tuple[list[dict[str, An
|
||||
continue
|
||||
if not _is_old_enough(metadata):
|
||||
continue
|
||||
candidates.append(pv)
|
||||
if not isinstance(name, str) or not name:
|
||||
continue
|
||||
candidates.append(
|
||||
_CleanupCandidate(
|
||||
name=name,
|
||||
kind="pv",
|
||||
path=f"/api/v1/persistentvolumes/{name}",
|
||||
created_at=_created_at(metadata),
|
||||
related_pvc=claim_name if isinstance(claim_name, str) else None,
|
||||
)
|
||||
)
|
||||
return candidates, all_pv_names
|
||||
|
||||
|
||||
def _workspace_pvc_candidates(active_claims: set[str]) -> list[dict[str, Any]]:
|
||||
def _workspace_pvc_candidates(active_claims: set[str]) -> list[_CleanupCandidate]:
|
||||
"""Find stale Jenkins workspace PVCs that are not actively referenced."""
|
||||
|
||||
namespace = settings.jenkins_workspace_namespace
|
||||
prefix = settings.jenkins_workspace_pvc_prefix
|
||||
payload = get_json(f"/api/v1/namespaces/{namespace}/persistentvolumeclaims")
|
||||
items = payload.get("items") if isinstance(payload.get("items"), list) else []
|
||||
candidates: list[dict[str, Any]] = []
|
||||
candidates: list[_CleanupCandidate] = []
|
||||
|
||||
for pvc in items:
|
||||
if not isinstance(pvc, dict):
|
||||
@ -129,7 +222,9 @@ def _workspace_pvc_candidates(active_claims: set[str]) -> list[dict[str, Any]]:
|
||||
status = pvc.get("status") if isinstance(pvc.get("status"), dict) else {}
|
||||
claim_name = metadata.get("name")
|
||||
phase = status.get("phase")
|
||||
if not isinstance(claim_name, str) or not claim_name.startswith(prefix):
|
||||
if not _is_workspace_name(claim_name):
|
||||
continue
|
||||
if _is_deleting(metadata):
|
||||
continue
|
||||
if claim_name in active_claims:
|
||||
continue
|
||||
@ -137,134 +232,369 @@ def _workspace_pvc_candidates(active_claims: set[str]) -> list[dict[str, Any]]:
|
||||
continue
|
||||
if not _is_old_enough(metadata):
|
||||
continue
|
||||
candidates.append(pvc)
|
||||
if not isinstance(claim_name, str) or not claim_name:
|
||||
continue
|
||||
candidates.append(
|
||||
_CleanupCandidate(
|
||||
name=claim_name,
|
||||
kind="pvc",
|
||||
path=f"/api/v1/namespaces/{namespace}/persistentvolumeclaims/{claim_name}",
|
||||
created_at=_created_at(metadata),
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def _delete_stale_pvcs(stale_pvcs: list[dict[str, Any]], namespace: str) -> tuple[int, int, int]:
|
||||
"""Delete stale workspace PVCs and return deleted/skipped/failure counters."""
|
||||
|
||||
deleted = 0
|
||||
skipped = 0
|
||||
failures = 0
|
||||
for pvc in stale_pvcs:
|
||||
metadata = pvc.get("metadata") if isinstance(pvc.get("metadata"), dict) else {}
|
||||
claim_name = metadata.get("name")
|
||||
if not isinstance(claim_name, str) or not claim_name:
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
delete_json(f"/api/v1/namespaces/{namespace}/persistentvolumeclaims/{claim_name}")
|
||||
deleted += 1
|
||||
except Exception as exc:
|
||||
failures += 1
|
||||
logger.info(
|
||||
"jenkins workspace pvc delete failed",
|
||||
extra={"event": "jenkins_workspace_cleanup", "claim": claim_name, "detail": str(exc)},
|
||||
)
|
||||
return deleted, skipped, failures
|
||||
|
||||
|
||||
def _delete_stale_pvs(stale_pvs: list[dict[str, Any]]) -> tuple[int, int, int, set[str]]:
|
||||
"""Delete stale workspace PVs and return counters plus removed PV names."""
|
||||
|
||||
deleted = 0
|
||||
skipped = 0
|
||||
failures = 0
|
||||
removed_pv_names: set[str] = set()
|
||||
for pv in stale_pvs:
|
||||
metadata = pv.get("metadata") if isinstance(pv.get("metadata"), dict) else {}
|
||||
pv_name = metadata.get("name")
|
||||
if not isinstance(pv_name, str) or not pv_name:
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
delete_json(f"/api/v1/persistentvolumes/{pv_name}")
|
||||
removed_pv_names.add(pv_name)
|
||||
deleted += 1
|
||||
except Exception as exc:
|
||||
failures += 1
|
||||
logger.info(
|
||||
"jenkins workspace pv delete failed",
|
||||
extra={"event": "jenkins_workspace_cleanup", "pv": pv_name, "detail": str(exc)},
|
||||
)
|
||||
return deleted, skipped, failures, removed_pv_names
|
||||
|
||||
|
||||
def _volume_should_delete(
|
||||
metadata: dict[str, Any],
|
||||
removed_pv_names: set[str],
|
||||
all_pv_names: set[str],
|
||||
prefix: str,
|
||||
) -> bool:
|
||||
"""Return true when a Longhorn volume belongs to stale Jenkins workspace storage."""
|
||||
|
||||
name = metadata.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
return False
|
||||
if name in removed_pv_names:
|
||||
return True
|
||||
labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {}
|
||||
pvc_name = labels.get("kubernetes.io/created-for/pvc/name")
|
||||
return isinstance(pvc_name, str) and pvc_name.startswith(prefix) and name not in all_pv_names
|
||||
|
||||
|
||||
def _delete_stale_longhorn_volumes(
|
||||
removed_pv_names: set[str],
|
||||
all_pv_names: set[str],
|
||||
prefix: str,
|
||||
) -> tuple[int, int, int]:
|
||||
"""Delete stale Longhorn volumes tied to Jenkins workspace PVCs."""
|
||||
|
||||
deleted = 0
|
||||
skipped = 0
|
||||
failures = 0
|
||||
def _workspace_longhorn_candidates(all_pv_names: set[str], removed_pv_names: set[str]) -> list[_CleanupCandidate]:
|
||||
namespace = "longhorn-system"
|
||||
payload = get_json("/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes")
|
||||
items = payload.get("items") if isinstance(payload.get("items"), list) else []
|
||||
candidates: list[_CleanupCandidate] = []
|
||||
|
||||
for volume in items:
|
||||
if not isinstance(volume, dict):
|
||||
continue
|
||||
metadata = volume.get("metadata") if isinstance(volume.get("metadata"), dict) else {}
|
||||
status = volume.get("status") if isinstance(volume.get("status"), dict) else {}
|
||||
spec = volume.get("spec") if isinstance(volume.get("spec"), dict) else {}
|
||||
name = metadata.get("name")
|
||||
if not isinstance(name, str) or not name:
|
||||
skipped += 1
|
||||
continue
|
||||
if not _volume_should_delete(metadata, removed_pv_names, all_pv_names, prefix):
|
||||
|
||||
labels = metadata.get("labels") if isinstance(metadata.get("labels"), dict) else {}
|
||||
pvc_name = labels.get("kubernetes.io/created-for/pvc/name")
|
||||
robust_state = status.get("robustness")
|
||||
state = status.get("state")
|
||||
attached = status.get("isAttached")
|
||||
frontend = spec.get("frontend")
|
||||
should_delete = False
|
||||
if name in removed_pv_names:
|
||||
should_delete = True
|
||||
elif _is_workspace_name(pvc_name) and name not in all_pv_names:
|
||||
should_delete = True
|
||||
if not should_delete:
|
||||
continue
|
||||
if _is_deleting(metadata):
|
||||
continue
|
||||
if not _is_old_enough(metadata):
|
||||
continue
|
||||
if state not in {None, "detached", "faulted", "unknown"}:
|
||||
continue
|
||||
if attached is True:
|
||||
continue
|
||||
if robust_state not in {None, "unknown", "faulted", "degraded"}:
|
||||
continue
|
||||
if frontend not in {None, "", "blockdev"}:
|
||||
continue
|
||||
candidates.append(
|
||||
_CleanupCandidate(
|
||||
name=name,
|
||||
kind="longhorn_volume",
|
||||
path=f"/apis/longhorn.io/v1beta2/namespaces/{namespace}/volumes/{name}",
|
||||
created_at=_created_at(metadata),
|
||||
pv_name=name,
|
||||
)
|
||||
)
|
||||
return candidates
|
||||
|
||||
|
||||
def _validate_cleanup_settings() -> tuple[str, str, bool, int]:
|
||||
namespace = settings.jenkins_workspace_namespace
|
||||
prefix = settings.jenkins_workspace_pvc_prefix.strip()
|
||||
dry_run = settings.jenkins_workspace_cleanup_dry_run
|
||||
max_deletions = settings.jenkins_workspace_cleanup_max_deletions_per_run
|
||||
if not namespace.strip():
|
||||
raise ValueError("jenkins workspace cleanup namespace is empty")
|
||||
if not prefix:
|
||||
raise ValueError("jenkins workspace cleanup pvc prefix is empty")
|
||||
if settings.jenkins_workspace_cleanup_min_age_hours < 1.0:
|
||||
raise ValueError("jenkins workspace cleanup min age must be >= 1 hour")
|
||||
if max_deletions < 1:
|
||||
raise ValueError("jenkins workspace cleanup max deletions must be >= 1")
|
||||
return namespace, prefix, dry_run, max_deletions
|
||||
|
||||
|
||||
def _planned_removed_pv_names_dry_run(
|
||||
stale_pvcs: list[_CleanupCandidate],
|
||||
stale_pvs: list[_CleanupCandidate],
|
||||
max_deletions: int,
|
||||
) -> set[str]:
|
||||
remaining = max(max_deletions - len(stale_pvcs), 0)
|
||||
if remaining == 0:
|
||||
return set()
|
||||
names = [candidate.name for candidate in stale_pvs if candidate.name]
|
||||
return set(names[:remaining])
|
||||
|
||||
|
||||
def _delete_candidates(
|
||||
candidates: list[_CleanupCandidate],
|
||||
*,
|
||||
deletion_budget: int | None,
|
||||
failure_log: str,
|
||||
failure_field: str,
|
||||
removed_pv_names: set[str] | None = None,
|
||||
) -> tuple[int, int, int, int | None]:
|
||||
deleted = 0
|
||||
skipped = 0
|
||||
failures = 0
|
||||
budget = deletion_budget
|
||||
for candidate in candidates:
|
||||
if not candidate.name:
|
||||
skipped += 1
|
||||
continue
|
||||
if budget is not None and budget <= 0:
|
||||
skipped += 1
|
||||
continue
|
||||
if budget is not None:
|
||||
budget -= 1
|
||||
try:
|
||||
delete_json(f"/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/{name}")
|
||||
delete_json(candidate.path)
|
||||
deleted += 1
|
||||
if removed_pv_names is not None:
|
||||
removed_pv_names.add(candidate.name)
|
||||
except Exception as exc:
|
||||
failures += 1
|
||||
logger.info(
|
||||
"jenkins workspace longhorn volume delete failed",
|
||||
extra={"event": "jenkins_workspace_cleanup", "volume": name, "detail": str(exc)},
|
||||
failure_log,
|
||||
extra={"event": "jenkins_workspace_cleanup", failure_field: candidate.name, "detail": str(exc)},
|
||||
)
|
||||
return deleted, skipped, failures
|
||||
return deleted, skipped, failures, budget
|
||||
|
||||
|
||||
def _record_guard_cap(
|
||||
*,
|
||||
max_deletions: int,
|
||||
stale_pvcs: list[_CleanupCandidate],
|
||||
stale_pvs: list[_CleanupCandidate],
|
||||
stale_volumes: list[_CleanupCandidate],
|
||||
dry_run: bool,
|
||||
) -> None:
|
||||
planned_total = len(stale_pvcs) + len(stale_pvs) + len(stale_volumes)
|
||||
if planned_total <= max_deletions:
|
||||
return
|
||||
logger.warning(
|
||||
"jenkins workspace cleanup capped by max deletions guard",
|
||||
extra={
|
||||
"event": "jenkins_workspace_cleanup",
|
||||
"status": "guard_capped",
|
||||
"namespace": settings.jenkins_workspace_namespace,
|
||||
"dry_run": dry_run,
|
||||
"planned_total": planned_total,
|
||||
"max_deletions": max_deletions,
|
||||
"planned_pvs": len(stale_pvs),
|
||||
"planned_pvcs": len(stale_pvcs),
|
||||
"planned_volumes": len(stale_volumes),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _dry_run_summary(
|
||||
*,
|
||||
namespace: str,
|
||||
max_deletions: int,
|
||||
stale_pvcs: list[_CleanupCandidate],
|
||||
stale_pvs: list[_CleanupCandidate],
|
||||
all_pv_names: set[str],
|
||||
) -> JenkinsWorkspaceCleanupSummary:
|
||||
simulated_removed = _planned_removed_pv_names_dry_run(stale_pvcs, stale_pvs, max_deletions)
|
||||
stale_volumes = _workspace_longhorn_candidates(all_pv_names, simulated_removed)
|
||||
_record_guard_cap(
|
||||
max_deletions=max_deletions,
|
||||
stale_pvcs=stale_pvcs,
|
||||
stale_pvs=stale_pvs,
|
||||
stale_volumes=stale_volumes,
|
||||
dry_run=True,
|
||||
)
|
||||
logger.info(
|
||||
"jenkins workspace cleanup dry-run enabled",
|
||||
extra={
|
||||
"event": "jenkins_workspace_cleanup",
|
||||
"status": "dry_run",
|
||||
"namespace": namespace,
|
||||
"dry_run": True,
|
||||
"planned_pvs": len(stale_pvs),
|
||||
"planned_pvcs": len(stale_pvcs),
|
||||
"planned_volumes": len(stale_volumes),
|
||||
"max_deletions": max_deletions,
|
||||
},
|
||||
)
|
||||
return JenkinsWorkspaceCleanupSummary(
|
||||
pvs_planned=len(stale_pvs),
|
||||
pvcs_planned=len(stale_pvcs),
|
||||
volumes_planned=len(stale_volumes),
|
||||
pvs_deleted=0,
|
||||
pvcs_deleted=0,
|
||||
volumes_deleted=0,
|
||||
skipped=0,
|
||||
failures=0,
|
||||
dry_run=True,
|
||||
)
|
||||
|
||||
|
||||
def _delete_run_summary(
|
||||
*,
|
||||
namespace: str,
|
||||
max_deletions: int,
|
||||
stale_pvcs: list[_CleanupCandidate],
|
||||
stale_pvs: list[_CleanupCandidate],
|
||||
all_pv_names: set[str],
|
||||
) -> JenkinsWorkspaceCleanupSummary:
|
||||
removed_pv_names: set[str] = set()
|
||||
deletion_budget: int | None = max_deletions
|
||||
pvcs_deleted, pvc_skipped, pvc_failures, deletion_budget = _delete_candidates(
|
||||
stale_pvcs,
|
||||
deletion_budget=deletion_budget,
|
||||
failure_log="jenkins workspace pvc delete failed",
|
||||
failure_field="claim",
|
||||
)
|
||||
pvs_deleted, pv_skipped, pv_failures, deletion_budget = _delete_candidates(
|
||||
stale_pvs,
|
||||
deletion_budget=deletion_budget,
|
||||
failure_log="jenkins workspace pv delete failed",
|
||||
failure_field="pv",
|
||||
removed_pv_names=removed_pv_names,
|
||||
)
|
||||
stale_volumes = _workspace_longhorn_candidates(all_pv_names, removed_pv_names)
|
||||
_record_guard_cap(
|
||||
max_deletions=max_deletions,
|
||||
stale_pvcs=stale_pvcs,
|
||||
stale_pvs=stale_pvs,
|
||||
stale_volumes=stale_volumes,
|
||||
dry_run=False,
|
||||
)
|
||||
volumes_deleted, volume_skipped, volume_failures, _ = _delete_candidates(
|
||||
stale_volumes,
|
||||
deletion_budget=deletion_budget,
|
||||
failure_log="jenkins workspace longhorn volume delete failed",
|
||||
failure_field="volume",
|
||||
)
|
||||
return JenkinsWorkspaceCleanupSummary(
|
||||
pvs_planned=len(stale_pvs),
|
||||
pvcs_planned=len(stale_pvcs),
|
||||
volumes_planned=len(stale_volumes),
|
||||
pvs_deleted=pvs_deleted,
|
||||
pvcs_deleted=pvcs_deleted,
|
||||
volumes_deleted=volumes_deleted,
|
||||
skipped=pvc_skipped + pv_skipped + volume_skipped,
|
||||
failures=pvc_failures + pv_failures + volume_failures,
|
||||
dry_run=False,
|
||||
)
|
||||
|
||||
|
||||
def _record_metrics(summary: JenkinsWorkspaceCleanupSummary) -> None:
|
||||
mode = "dry_run" if summary.dry_run else "delete"
|
||||
status = "ok" if summary.failures == 0 else "error"
|
||||
JENKINS_WORKSPACE_CLEANUP_RUNS_TOTAL.labels(status=status, mode=mode).inc()
|
||||
if summary.failures:
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_FAILURE_TS.set(datetime.now(timezone.utc).timestamp())
|
||||
else:
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_SUCCESS_TS.set(datetime.now(timezone.utc).timestamp())
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_RUN_TS.set(datetime.now(timezone.utc).timestamp())
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_DELETED.labels(kind="pvc").set(summary.pvcs_deleted)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_DELETED.labels(kind="pv").set(summary.pvs_deleted)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_DELETED.labels(kind="longhorn_volume").set(summary.volumes_deleted)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED.labels(kind="pvc").set(summary.pvcs_planned)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED.labels(kind="pv").set(summary.pvs_planned)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_PLANNED.labels(kind="longhorn_volume").set(summary.volumes_planned)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_SKIPPED.set(summary.skipped)
|
||||
JENKINS_WORKSPACE_CLEANUP_LAST_FAILURES.set(summary.failures)
|
||||
for kind, planned, deleted in (
|
||||
("pvc", summary.pvcs_planned, summary.pvcs_deleted),
|
||||
("pv", summary.pvs_planned, summary.pvs_deleted),
|
||||
("longhorn_volume", summary.volumes_planned, summary.volumes_deleted),
|
||||
):
|
||||
if planned:
|
||||
JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels(kind=kind, action="planned", mode=mode).inc(planned)
|
||||
if deleted:
|
||||
JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels(kind=kind, action="deleted", mode=mode).inc(deleted)
|
||||
if summary.skipped:
|
||||
JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels(
|
||||
kind="cleanup",
|
||||
action="skipped",
|
||||
mode=mode,
|
||||
).inc(summary.skipped)
|
||||
if summary.failures:
|
||||
JENKINS_WORKSPACE_CLEANUP_OBJECTS_TOTAL.labels(
|
||||
kind="cleanup",
|
||||
action="failed",
|
||||
mode=mode,
|
||||
).inc(summary.failures)
|
||||
|
||||
|
||||
def cleanup_jenkins_workspace_storage() -> JenkinsWorkspaceCleanupSummary:
|
||||
"""Delete stale Jenkins workspace PVC/PV artifacts and orphan Longhorn volumes."""
|
||||
|
||||
namespace = settings.jenkins_workspace_namespace
|
||||
prefix = settings.jenkins_workspace_pvc_prefix
|
||||
active_claims = _active_workspace_claims()
|
||||
stale_pvs, all_pv_names = _workspace_pv_candidates(active_claims)
|
||||
stale_pvcs = _workspace_pvc_candidates(active_claims)
|
||||
pvcs_deleted, pvc_skipped, pvc_failures = _delete_stale_pvcs(stale_pvcs, namespace)
|
||||
pvs_deleted, pv_skipped, pv_failures, removed_pv_names = _delete_stale_pvs(stale_pvs)
|
||||
volumes_deleted, volume_skipped, volume_failures = _delete_stale_longhorn_volumes(
|
||||
removed_pv_names, all_pv_names, prefix
|
||||
summary = JenkinsWorkspaceCleanupSummary(
|
||||
pvs_planned=0,
|
||||
pvcs_planned=0,
|
||||
volumes_planned=0,
|
||||
pvs_deleted=0,
|
||||
pvcs_deleted=0,
|
||||
volumes_deleted=0,
|
||||
skipped=0,
|
||||
failures=0,
|
||||
dry_run=settings.jenkins_workspace_cleanup_dry_run,
|
||||
)
|
||||
skipped = pvc_skipped + pv_skipped + volume_skipped
|
||||
failures = pvc_failures + pv_failures + volume_failures
|
||||
|
||||
return JenkinsWorkspaceCleanupSummary(
|
||||
pvs_deleted=pvs_deleted,
|
||||
pvcs_deleted=pvcs_deleted,
|
||||
volumes_deleted=volumes_deleted,
|
||||
skipped=skipped,
|
||||
failures=failures,
|
||||
try:
|
||||
namespace, _prefix, dry_run, max_deletions = _validate_cleanup_settings()
|
||||
active_claims = _active_workspace_claims()
|
||||
stale_pvs, all_pv_names = _workspace_pv_candidates(active_claims)
|
||||
stale_pvcs = _workspace_pvc_candidates(active_claims)
|
||||
if dry_run:
|
||||
summary = _dry_run_summary(
|
||||
namespace=namespace,
|
||||
max_deletions=max_deletions,
|
||||
stale_pvcs=stale_pvcs,
|
||||
stale_pvs=stale_pvs,
|
||||
all_pv_names=all_pv_names,
|
||||
)
|
||||
else:
|
||||
summary = _delete_run_summary(
|
||||
namespace=namespace,
|
||||
max_deletions=max_deletions,
|
||||
stale_pvcs=stale_pvcs,
|
||||
stale_pvs=stale_pvs,
|
||||
all_pv_names=all_pv_names,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception(
|
||||
"jenkins workspace cleanup failed",
|
||||
extra={
|
||||
"event": "jenkins_workspace_cleanup",
|
||||
"status": "error",
|
||||
"namespace": settings.jenkins_workspace_namespace,
|
||||
"detail": str(exc),
|
||||
},
|
||||
)
|
||||
summary = JenkinsWorkspaceCleanupSummary(
|
||||
pvs_planned=summary.pvs_planned,
|
||||
pvcs_planned=summary.pvcs_planned,
|
||||
volumes_planned=summary.volumes_planned,
|
||||
pvs_deleted=summary.pvs_deleted,
|
||||
pvcs_deleted=summary.pvcs_deleted,
|
||||
volumes_deleted=summary.volumes_deleted,
|
||||
skipped=summary.skipped,
|
||||
failures=summary.failures + 1,
|
||||
dry_run=summary.dry_run,
|
||||
)
|
||||
_record_metrics(summary)
|
||||
raise
|
||||
_record_metrics(summary)
|
||||
logger.info(
|
||||
"jenkins workspace cleanup finished",
|
||||
extra={
|
||||
"event": "jenkins_workspace_cleanup",
|
||||
"status": "ok" if summary.failures == 0 else "error",
|
||||
"dry_run": summary.dry_run,
|
||||
"namespace": namespace,
|
||||
"planned_pvs": summary.pvs_planned,
|
||||
"planned_pvcs": summary.pvcs_planned,
|
||||
"planned_volumes": summary.volumes_planned,
|
||||
"deleted_pvs": summary.pvs_deleted,
|
||||
"deleted_pvcs": summary.pvcs_deleted,
|
||||
"deleted_volumes": summary.volumes_deleted,
|
||||
"skipped": summary.skipped,
|
||||
"failures": summary.failures,
|
||||
},
|
||||
)
|
||||
return summary
|
||||
|
||||
@ -171,6 +171,8 @@ class Settings:
|
||||
jenkins_workspace_namespace: str
|
||||
jenkins_workspace_pvc_prefix: str
|
||||
jenkins_workspace_cleanup_min_age_hours: float
|
||||
jenkins_workspace_cleanup_dry_run: bool
|
||||
jenkins_workspace_cleanup_max_deletions_per_run: int
|
||||
|
||||
vaultwarden_namespace: str
|
||||
vaultwarden_pod_label: str
|
||||
@ -469,6 +471,11 @@ class Settings:
|
||||
"jenkins_workspace_namespace": _env("JENKINS_WORKSPACE_NAMESPACE", "jenkins"),
|
||||
"jenkins_workspace_pvc_prefix": _env("JENKINS_WORKSPACE_PVC_PREFIX", "pvc-workspace-"),
|
||||
"jenkins_workspace_cleanup_min_age_hours": _env_float("JENKINS_WORKSPACE_CLEANUP_MIN_AGE_HOURS", 12.0),
|
||||
"jenkins_workspace_cleanup_dry_run": _env_bool("JENKINS_WORKSPACE_CLEANUP_DRY_RUN", "false"),
|
||||
"jenkins_workspace_cleanup_max_deletions_per_run": _env_int(
|
||||
"JENKINS_WORKSPACE_CLEANUP_MAX_DELETIONS_PER_RUN",
|
||||
20,
|
||||
),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
|
||||
@ -3,101 +3,228 @@ from __future__ import annotations
|
||||
from datetime import datetime, timezone
|
||||
import types
|
||||
|
||||
from prometheus_client import REGISTRY
|
||||
|
||||
from ariadne.services import jenkins_workspace_cleanup as cleanup_module
|
||||
|
||||
|
||||
def test_cleanup_jenkins_workspace_storage(monkeypatch) -> None:
|
||||
dummy_settings = types.SimpleNamespace(
|
||||
def _metric_value(name: str, labels: dict[str, str]) -> float:
|
||||
value = REGISTRY.get_sample_value(name, labels)
|
||||
return float(value) if value is not None else 0.0
|
||||
|
||||
|
||||
def _dummy_settings(*, dry_run: bool, max_deletions: int = 20) -> types.SimpleNamespace:
|
||||
return types.SimpleNamespace(
|
||||
jenkins_workspace_namespace="jenkins",
|
||||
jenkins_workspace_pvc_prefix="pvc-workspace-",
|
||||
jenkins_workspace_cleanup_min_age_hours=1.0,
|
||||
jenkins_workspace_cleanup_dry_run=dry_run,
|
||||
jenkins_workspace_cleanup_max_deletions_per_run=max_deletions,
|
||||
)
|
||||
monkeypatch.setattr(cleanup_module, "settings", dummy_settings)
|
||||
|
||||
|
||||
def _fake_payloads(now_iso: str, old_iso: str) -> dict[str, dict[str, object]]:
|
||||
return {
|
||||
"/api/v1/namespaces/jenkins/pods": {
|
||||
"items": [
|
||||
{
|
||||
"metadata": {
|
||||
"annotations": {
|
||||
"jenkins.io/workspace-pvc": "pvc-workspace-annotated-active",
|
||||
}
|
||||
},
|
||||
"spec": {
|
||||
"volumes": [
|
||||
{"persistentVolumeClaim": {"claimName": "pvc-workspace-active"}},
|
||||
]
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"/api/v1/namespaces/jenkins/persistentvolumeclaims": {
|
||||
"items": [
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-stale", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Lost"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-active", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Bound"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-annotated-active", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Lost"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-fresh", "creationTimestamp": now_iso},
|
||||
"status": {"phase": "Lost"},
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-workspace-deleting",
|
||||
"creationTimestamp": old_iso,
|
||||
"deletionTimestamp": old_iso,
|
||||
},
|
||||
"status": {"phase": "Lost"},
|
||||
},
|
||||
]
|
||||
},
|
||||
"/api/v1/persistentvolumes": {
|
||||
"items": [
|
||||
{
|
||||
"metadata": {"name": "pvc-old", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-stale"}},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-active", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-active"}},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-annotated", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-annotated-active"}},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-fresh", "creationTimestamp": now_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-fresh"}},
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-deleting",
|
||||
"creationTimestamp": old_iso,
|
||||
"deletionTimestamp": old_iso,
|
||||
},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-deleting"}},
|
||||
},
|
||||
]
|
||||
},
|
||||
"/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes": {
|
||||
"items": [
|
||||
{"metadata": {"name": "pvc-old", "creationTimestamp": old_iso}},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-orphan",
|
||||
"creationTimestamp": old_iso,
|
||||
"labels": {
|
||||
"kubernetes.io/created-for/pvc/name": "pvc-workspace-orphan",
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-attached",
|
||||
"creationTimestamp": old_iso,
|
||||
"labels": {
|
||||
"kubernetes.io/created-for/pvc/name": "pvc-workspace-annotated-active",
|
||||
},
|
||||
},
|
||||
"status": {"state": "attached", "isAttached": True, "robustness": "healthy"},
|
||||
"spec": {"frontend": "blockdev"},
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-orphan-fresh",
|
||||
"creationTimestamp": now_iso,
|
||||
"labels": {
|
||||
"kubernetes.io/created-for/pvc/name": "pvc-workspace-fresh",
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-vol-deleting",
|
||||
"creationTimestamp": old_iso,
|
||||
"deletionTimestamp": old_iso,
|
||||
"labels": {
|
||||
"kubernetes.io/created-for/pvc/name": "pvc-workspace-orphan",
|
||||
},
|
||||
}
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def test_cleanup_jenkins_workspace_storage_dry_run(monkeypatch) -> None:
|
||||
monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=True))
|
||||
|
||||
now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
old_iso = "2020-01-01T00:00:00Z"
|
||||
payloads = _fake_payloads(now_iso, old_iso)
|
||||
deleted_paths: list[str] = []
|
||||
|
||||
def fake_get_json(path: str):
|
||||
if path == "/api/v1/namespaces/jenkins/pods":
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"spec": {
|
||||
"volumes": [
|
||||
{"persistentVolumeClaim": {"claimName": "pvc-workspace-active"}},
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
if path == "/api/v1/namespaces/jenkins/persistentvolumeclaims":
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-stale", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Lost"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-active", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Bound"},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-workspace-fresh", "creationTimestamp": now_iso},
|
||||
"status": {"phase": "Lost"},
|
||||
},
|
||||
]
|
||||
}
|
||||
if path == "/api/v1/persistentvolumes":
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"metadata": {"name": "pvc-old", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-stale"}},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-active", "creationTimestamp": old_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-active"}},
|
||||
},
|
||||
{
|
||||
"metadata": {"name": "pvc-fresh", "creationTimestamp": now_iso},
|
||||
"status": {"phase": "Released"},
|
||||
"spec": {"claimRef": {"namespace": "jenkins", "name": "pvc-workspace-fresh"}},
|
||||
},
|
||||
]
|
||||
}
|
||||
if path == "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes":
|
||||
return {
|
||||
"items": [
|
||||
{"metadata": {"name": "pvc-old", "creationTimestamp": old_iso}},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-orphan",
|
||||
"creationTimestamp": old_iso,
|
||||
"labels": {
|
||||
"kubernetes.io/created-for/pvc/name": "pvc-workspace-orphan",
|
||||
},
|
||||
}
|
||||
},
|
||||
{
|
||||
"metadata": {
|
||||
"name": "pvc-orphan-fresh",
|
||||
"creationTimestamp": now_iso,
|
||||
"labels": {
|
||||
"kubernetes.io/created-for/pvc/name": "pvc-workspace-fresh",
|
||||
},
|
||||
}
|
||||
},
|
||||
]
|
||||
}
|
||||
if path in payloads:
|
||||
return payloads[path]
|
||||
raise AssertionError(f"unexpected path: {path}")
|
||||
|
||||
def fake_delete_json(path: str):
|
||||
deleted_paths.append(path)
|
||||
return {"status": "Success"}
|
||||
|
||||
before_runs = _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_runs_total",
|
||||
{"status": "ok", "mode": "dry_run"},
|
||||
)
|
||||
before_planned = _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
{"kind": "pvc", "action": "planned", "mode": "dry_run"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(cleanup_module, "get_json", fake_get_json)
|
||||
monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json)
|
||||
|
||||
summary = cleanup_module.cleanup_jenkins_workspace_storage()
|
||||
|
||||
assert summary.dry_run is True
|
||||
assert summary.pvcs_planned == 1
|
||||
assert summary.pvs_planned == 1
|
||||
assert summary.volumes_planned == 2
|
||||
assert summary.pvcs_deleted == 0
|
||||
assert summary.pvs_deleted == 0
|
||||
assert summary.volumes_deleted == 0
|
||||
assert summary.failures == 0
|
||||
assert deleted_paths == []
|
||||
assert _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_runs_total",
|
||||
{"status": "ok", "mode": "dry_run"},
|
||||
) == before_runs + 1
|
||||
assert _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
{"kind": "pvc", "action": "planned", "mode": "dry_run"},
|
||||
) == before_planned + 1
|
||||
|
||||
|
||||
def test_cleanup_jenkins_workspace_storage(monkeypatch) -> None:
|
||||
monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False))
|
||||
|
||||
now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
old_iso = "2020-01-01T00:00:00Z"
|
||||
deleted_paths: list[str] = []
|
||||
payloads = _fake_payloads(now_iso, old_iso)
|
||||
|
||||
def fake_get_json(path: str):
|
||||
if path in payloads:
|
||||
return payloads[path]
|
||||
raise AssertionError(f"unexpected path: {path}")
|
||||
|
||||
def fake_delete_json(path: str):
|
||||
deleted_paths.append(path)
|
||||
return {"status": "Success"}
|
||||
|
||||
before_runs = _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_runs_total",
|
||||
{"status": "ok", "mode": "delete"},
|
||||
)
|
||||
before_deleted = _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
{"kind": "longhorn_volume", "action": "deleted", "mode": "delete"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(cleanup_module, "get_json", fake_get_json)
|
||||
monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json)
|
||||
|
||||
@ -111,15 +238,19 @@ def test_cleanup_jenkins_workspace_storage(monkeypatch) -> None:
|
||||
assert "/api/v1/persistentvolumes/pvc-old" in deleted_paths
|
||||
assert "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-old" in deleted_paths
|
||||
assert "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-orphan" in deleted_paths
|
||||
assert "/apis/longhorn.io/v1beta2/namespaces/longhorn-system/volumes/pvc-attached" not in deleted_paths
|
||||
assert _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_runs_total",
|
||||
{"status": "ok", "mode": "delete"},
|
||||
) == before_runs + 1
|
||||
assert _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
{"kind": "longhorn_volume", "action": "deleted", "mode": "delete"},
|
||||
) == before_deleted + 2
|
||||
|
||||
|
||||
def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None:
|
||||
dummy_settings = types.SimpleNamespace(
|
||||
jenkins_workspace_namespace="jenkins",
|
||||
jenkins_workspace_pvc_prefix="pvc-workspace-",
|
||||
jenkins_workspace_cleanup_min_age_hours=1.0,
|
||||
)
|
||||
monkeypatch.setattr(cleanup_module, "settings", dummy_settings)
|
||||
monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False))
|
||||
|
||||
def fake_get_json(path: str):
|
||||
if path == "/api/v1/namespaces/jenkins/pods":
|
||||
@ -142,9 +273,51 @@ def test_cleanup_jenkins_workspace_storage_failure(monkeypatch) -> None:
|
||||
def fake_delete_json(_path: str):
|
||||
raise RuntimeError("boom")
|
||||
|
||||
before_failures = _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
{"kind": "cleanup", "action": "failed", "mode": "delete"},
|
||||
)
|
||||
|
||||
monkeypatch.setattr(cleanup_module, "get_json", fake_get_json)
|
||||
monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json)
|
||||
|
||||
summary = cleanup_module.cleanup_jenkins_workspace_storage()
|
||||
assert summary.failures == 1
|
||||
assert summary.pvcs_deleted == 0
|
||||
assert _metric_value(
|
||||
"ariadne_jenkins_workspace_cleanup_objects_total",
|
||||
{"kind": "cleanup", "action": "failed", "mode": "delete"},
|
||||
) == before_failures + 1
|
||||
|
||||
|
||||
def test_cleanup_jenkins_workspace_storage_guard_caps_mass_delete(monkeypatch) -> None:
|
||||
monkeypatch.setattr(cleanup_module, "settings", _dummy_settings(dry_run=False, max_deletions=1))
|
||||
|
||||
now_iso = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
old_iso = "2020-01-01T00:00:00Z"
|
||||
payloads = _fake_payloads(now_iso, old_iso)
|
||||
deleted_paths: list[str] = []
|
||||
|
||||
def fake_get_json(path: str):
|
||||
if path in payloads:
|
||||
return payloads[path]
|
||||
raise AssertionError(f"unexpected path: {path}")
|
||||
|
||||
def fake_delete_json(path: str):
|
||||
deleted_paths.append(path)
|
||||
return {"status": "Success"}
|
||||
|
||||
monkeypatch.setattr(cleanup_module, "get_json", fake_get_json)
|
||||
monkeypatch.setattr(cleanup_module, "delete_json", fake_delete_json)
|
||||
|
||||
summary = cleanup_module.cleanup_jenkins_workspace_storage()
|
||||
|
||||
assert summary.failures == 0
|
||||
assert summary.pvcs_planned == 1
|
||||
assert summary.pvs_planned == 1
|
||||
assert summary.volumes_planned == 1
|
||||
assert summary.pvcs_deleted == 1
|
||||
assert summary.pvs_deleted == 0
|
||||
assert summary.volumes_deleted == 0
|
||||
assert summary.skipped == 2
|
||||
assert deleted_paths == ["/api/v1/namespaces/jenkins/persistentvolumeclaims/pvc-workspace-stale"]
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user