109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
"""Glue checks for Ariadne schedules exported to VictoriaMetrics."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
import requests
|
|
import yaml
|
|
|
|
|
|
CONFIG_PATH = Path(__file__).with_name("config.yaml")
|
|
|
|
|
|
def _load_config() -> dict:
|
|
with CONFIG_PATH.open("r", encoding="utf-8") as handle:
|
|
return yaml.safe_load(handle) or {}
|
|
|
|
|
|
def _query(promql: str) -> list[dict]:
|
|
vm_url = os.environ.get("VM_URL", "http://victoria-metrics-single-server:8428").rstrip("/")
|
|
response = requests.get(f"{vm_url}/api/v1/query", params={"query": promql}, timeout=10)
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
return payload.get("data", {}).get("result", [])
|
|
|
|
|
|
def _expected_tasks() -> list[dict]:
|
|
cfg = _load_config()
|
|
tasks = [
|
|
_normalize_task(item, cfg)
|
|
for item in cfg.get("ariadne_schedule_tasks", [])
|
|
]
|
|
assert tasks, "No Ariadne schedule tasks configured"
|
|
return tasks
|
|
|
|
|
|
def _normalize_task(item: object, cfg: dict) -> dict:
|
|
if isinstance(item, str):
|
|
return {
|
|
"task": item,
|
|
"check_last_success": True,
|
|
"max_success_age_hours": cfg.get("max_success_age_hours", 48),
|
|
}
|
|
if isinstance(item, dict):
|
|
normalized = dict(item)
|
|
normalized.setdefault("check_last_success", True)
|
|
normalized.setdefault("max_success_age_hours", cfg.get("max_success_age_hours", 48))
|
|
return normalized
|
|
raise TypeError(f"Unsupported Ariadne schedule task config entry: {item!r}")
|
|
|
|
|
|
def _tracked_tasks(tasks: list[dict]) -> list[dict]:
|
|
tracked = [item for item in tasks if item.get("check_last_success")]
|
|
assert tracked, "No Ariadne schedule tasks are marked for success tracking"
|
|
return tracked
|
|
|
|
|
|
def _task_regex(tasks: list[dict]) -> str:
|
|
return "|".join(item["task"] for item in tasks)
|
|
|
|
|
|
def test_ariadne_schedule_series_exist():
|
|
tasks = _expected_tasks()
|
|
selector = _task_regex(tasks)
|
|
series = _query(f'ariadne_schedule_next_run_timestamp_seconds{{task=~"{selector}"}}')
|
|
seen = {item.get("metric", {}).get("task") for item in series}
|
|
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
|
assert not missing, f"Missing next-run metrics for: {', '.join(missing)}"
|
|
|
|
|
|
def test_ariadne_schedule_recent_success():
|
|
tasks = _tracked_tasks(_expected_tasks())
|
|
selector = _task_regex(tasks)
|
|
series = _query(f'ariadne_schedule_last_success_timestamp_seconds{{task=~"{selector}"}}')
|
|
seen = {item.get("metric", {}).get("task") for item in series}
|
|
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
|
assert not missing, f"Missing last-success metrics for: {', '.join(missing)}"
|
|
|
|
now = datetime.now(timezone.utc)
|
|
age_by_task = {
|
|
item.get("metric", {}).get("task"): (now - datetime.fromtimestamp(float(item["value"][1]), tz=timezone.utc)).total_seconds() / 3600
|
|
for item in series
|
|
}
|
|
too_old = [
|
|
f"{task} ({age_by_task[task]:.1f}h > {item['max_success_age_hours']}h)"
|
|
for item in tasks
|
|
if (task := item["task"]) in age_by_task and age_by_task[task] > float(item["max_success_age_hours"])
|
|
]
|
|
assert not too_old, "Ariadne schedules are stale: " + ", ".join(too_old)
|
|
|
|
|
|
def test_ariadne_schedule_last_status_present_and_boolean():
|
|
tasks = _tracked_tasks(_expected_tasks())
|
|
selector = _task_regex(tasks)
|
|
series = _query(f'ariadne_schedule_last_status{{task=~"{selector}"}}')
|
|
seen = {item.get("metric", {}).get("task") for item in series}
|
|
missing = [item["task"] for item in tasks if item["task"] not in seen]
|
|
assert not missing, f"Missing last-status metrics for: {', '.join(missing)}"
|
|
|
|
invalid = []
|
|
for item in series:
|
|
task = item.get("metric", {}).get("task")
|
|
value = float(item["value"][1])
|
|
if value not in (0.0, 1.0):
|
|
invalid.append(f"{task}={value}")
|
|
assert not invalid, f"Unexpected Ariadne last-status values: {', '.join(invalid)}"
|