titan-iac/scripts/dashboards_render_atlas.py

#!/usr/bin/env python3
"""Generate Atlas Grafana dashboards and render them into ConfigMaps.

Usage:
  scripts/dashboards_render_atlas.py --build   # rebuild JSON + ConfigMaps
  scripts/dashboards_render_atlas.py           # re-render ConfigMaps from JSON
"""

import argparse
import json
import textwrap
import urllib.parse
from pathlib import Path

# ---------------------------------------------------------------------------
# Paths, folders, and shared metadata
# ---------------------------------------------------------------------------

ROOT = Path(__file__).resolve().parents[1]
DASHBOARD_DIR = ROOT / "services" / "monitoring" / "dashboards"
CONFIG_TEMPLATE = textwrap.dedent(
    """# {relative_path}
apiVersion: v1
kind: ConfigMap
metadata:
  name: {name}
  labels:
    grafana_dashboard: "1"
data:
  {key}: |
{payload}
"""
)

PROM_DS = {"type": "prometheus", "uid": "atlas-vm"}
PUBLIC_FOLDER = "overview"
PRIVATE_FOLDER = "atlas-internal"
ASTRAIOS_MOUNTPOINT = "/mnt/astraios"

PERCENT_THRESHOLDS = {
    "mode": "absolute",
    "steps": [
        {"color": "green", "value": None},
        {"color": "yellow", "value": 50},
        {"color": "orange", "value": 75},
        {"color": "red", "value": 91.5},
    ],
}

NAMESPACE_CPU_WINDOW = "1m"

# ---------------------------------------------------------------------------
# Cluster metadata
# ---------------------------------------------------------------------------

CONTROL_PLANE_NODES = ["titan-0a", "titan-0b", "titan-0c"]
CONTROL_DEPENDENCIES = ["titan-db", "titan-jh"]
CONTROL_ALL = CONTROL_PLANE_NODES + CONTROL_DEPENDENCIES
WORKER_NODES = [
    "titan-04",
    "titan-05",
    "titan-06",
    "titan-07",
    "titan-08",
    "titan-09",
    "titan-10",
    "titan-11",
    "titan-20",
    "titan-21",
    "titan-12",
    "titan-13",
    "titan-14",
    "titan-15",
    "titan-16",
    "titan-17",
    "titan-18",
    "titan-19",
    "titan-22",
    "titan-24",
]

CONTROL_REGEX = "|".join(CONTROL_PLANE_NODES)
CONTROL_ALL_REGEX = "|".join(CONTROL_ALL)
WORKER_REGEX = "|".join(WORKER_NODES)
CONTROL_TOTAL = len(CONTROL_PLANE_NODES)
WORKER_TOTAL = len(WORKER_NODES)
CONTROL_SUFFIX = f"/{CONTROL_TOTAL}"
WORKER_SUFFIX = f"/{WORKER_TOTAL}"
# Namespaces considered infrastructure (excluded from workload counts)
INFRA_PATTERNS = [
    "kube-.*",
    ".*-system",
    "traefik",
    "monitoring",
    "logging",
    "cert-manager",
    "maintenance",
    "postgres",
]
INFRA_REGEX = f"^({'|'.join(INFRA_PATTERNS)})$"
# Namespaces allowed on control plane without counting as workloads
CP_ALLOWED_NS = INFRA_REGEX
LONGHORN_NODE_REGEX = "titan-1[2-9]|titan-2[24]"
GAUGE_WIDTHS = [4, 3, 3, 4, 3, 3, 4]
CONTROL_WORKLOADS_EXPR = (
    f'sum(kube_pod_info{{node=~"{CONTROL_REGEX}",namespace!~"{CP_ALLOWED_NS}"}}) or on() vector(0)'
)

# ---------------------------------------------------------------------------
# PromQL helpers
# ---------------------------------------------------------------------------

NODE_INFO = 'label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)")'


def node_filter(regex):
    """Return a selector that evaluates to 1 for nodes matching the regex."""
    return (
        f'label_replace(node_uname_info{{nodename=~"{regex}"}}, '
        '"node", "$1", "nodename", "(.*)")'
    )


def scoped_node_expr(base, scope=""):
    """Attach nodename metadata and optionally filter to a scope regex."""
    expr = f"avg by (node) (({base}) * on(instance) group_left(node) {NODE_INFO})"
    if scope:
        expr = f"({expr}) * on(node) group_left() {node_filter(scope)}"
    return expr


def node_cpu_expr(scope=""):
    idle = 'avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))'
    base = f"(1 - {idle}) * 100"
    return scoped_node_expr(base, scope)


def node_mem_expr(scope=""):
    usage = (
        "avg by (instance) ("
        "(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) "
        "/ node_memory_MemTotal_bytes * 100)"
    )
    return scoped_node_expr(usage, scope)


def filesystem_usage_expr(mount, scope=""):
    base = (
        f'avg by (instance) ('
        f'(1 - (node_filesystem_avail_bytes{{mountpoint="{mount}",fstype!~"tmpfs|overlay"}} '
        f'/ node_filesystem_size_bytes{{mountpoint="{mount}",fstype!~"tmpfs|overlay"}})) * 100)'
    )
    return scoped_node_expr(base, scope)


def root_usage_expr(scope=""):
    return filesystem_usage_expr("/", scope)


def astraios_usage_expr(scope=""):
    return filesystem_usage_expr(ASTRAIOS_MOUNTPOINT, scope)


def astreae_usage_expr(mount):
    return (
        f"100 - (sum(node_filesystem_avail_bytes{{mountpoint=\"{mount}\",fstype!~\"tmpfs|overlay\"}}) / "
        f"sum(node_filesystem_size_bytes{{mountpoint=\"{mount}\",fstype!~\"tmpfs|overlay\"}}) * 100)"
    )


def astreae_free_expr(mount):
    return f"sum(node_filesystem_avail_bytes{{mountpoint=\"{mount}\",fstype!~\"tmpfs|overlay\"}})"


def topk_with_node(expr):
    return f'label_replace(topk(1, {expr}), "__name__", "$1", "node", "(.*)")'


def node_net_expr(scope=""):
    base = (
        'sum by (instance) ('
        'rate(node_network_receive_bytes_total{device!~"lo"}[5m]) '
        '+ rate(node_network_transmit_bytes_total{device!~"lo"}[5m]))'
    )
    return scoped_node_expr(base, scope)


def node_io_expr(scope=""):
    base = (
        "sum by (instance) (rate(node_disk_read_bytes_total[5m]) "
        "+ rate(node_disk_written_bytes_total[5m]))"
    )
    return scoped_node_expr(base, scope)


def namespace_selector(scope_var):
    return f'namespace!="",pod!="",container!="",container!="POD",{scope_var}'


def namespace_gpu_selector(scope_var):
    return f'namespace!="",pod!="",{scope_var}'


def namespace_cpu_raw(scope_var):
    return (
        "sum(rate(container_cpu_usage_seconds_total"
        f"{{{namespace_selector(scope_var)}}}[{NAMESPACE_CPU_WINDOW}])) by (namespace)"
    )


def namespace_ram_raw(scope_var):
    return f"sum(container_memory_working_set_bytes{{{namespace_selector(scope_var)}}}) by (namespace)"


def namespace_gpu_usage_instant(scope_var):
    return gpu_usage_by_namespace(scope_var)


def jetson_gpu_util_by_node():
    return 'max by (node) (jetson_gr3d_freq_percent{node!=""})'


def dcgm_gpu_util_by_node():
    dcgm_pod = 'label_replace(DCGM_FI_DEV_GPU_UTIL, "pod", "$1", "Hostname", "(.*)")'
    dcgm_ns = 'label_replace(' + dcgm_pod + ', "namespace", "monitoring", "", "")'
    return (
        "avg by (node) ("
        f"{dcgm_ns} * on(namespace,pod) group_left(node) "
        'kube_pod_info{namespace="monitoring"}'
        ")"
    )


def gpu_util_by_node():
    return f"{dcgm_gpu_util_by_node()} or {jetson_gpu_util_by_node()}"


def gpu_util_by_hostname():
    return 'label_replace(' + gpu_util_by_node() + ', "Hostname", "$1", "node", "(.*)")'


def gpu_node_labels():
    return 'kube_node_labels{label_accelerator=~".+"} or kube_node_labels{label_jetson="true"}'


def gpu_requests_by_namespace_node(scope_var):
    return (
        "sum by (namespace,node) ("
        f'kube_pod_container_resource_requests{{resource=~"nvidia.com/gpu.*",{scope_var}}} '
        "* on(namespace,pod) group_left(node) kube_pod_info "
        f"* on(node) group_left() ({gpu_node_labels()})"
        ")"
    )


def gpu_usage_by_namespace(scope_var):
    requests_by_ns = gpu_requests_by_namespace_node(scope_var)
    total_by_node = f"sum by (node) ({requests_by_ns})"
    return (
        "sum by (namespace) ("
        f"({requests_by_ns}) / clamp_min({total_by_node}, 1) "
        f"* on(node) group_left() ({gpu_util_by_node()})"
        ")"
    )


def jetson_gpu_usage_by_namespace(scope_var):
    requests_by_ns = jetson_gpu_requests(scope_var)
    total_by_node = f"sum by (node) ({requests_by_ns})"
    return (
        "sum by (namespace) ("
        f"({requests_by_ns}) / clamp_min({total_by_node}, 1) "
        f"* on(node) group_left() {jetson_gpu_util_by_node()}"
        ")"
    )


def namespace_share_expr(resource_expr):
    total = f"clamp_min(sum( {resource_expr} ), 1)"
    return f"100 * ( {resource_expr} ) / {total}"


def namespace_cpu_share_expr(scope_var):
    return namespace_share_expr(namespace_cpu_raw(scope_var))


def namespace_ram_share_expr(scope_var):
    return namespace_share_expr(namespace_ram_raw(scope_var))


def namespace_gpu_share_expr(scope_var):
    usage = namespace_gpu_usage_instant(scope_var)
    total = f"(sum({usage}) or on() vector(0))"
    share = f"100 * ({usage}) / clamp_min({total}, 1)"
    idle = 'label_replace(vector(100), "namespace", "idle", "", "") * scalar(' + total + " == bool 0)"
    return f"({share}) or ({idle})"


PROBLEM_PODS_EXPR = (
    'sum(max by (namespace,pod) (kube_pod_status_phase{phase!~"Running|Succeeded"})) '
    "or on() vector(0)"
)
CRASHLOOP_EXPR = (
    'sum(max by (namespace,pod) (kube_pod_container_status_waiting_reason'
    '{reason=~"CrashLoopBackOff|ImagePullBackOff"})) '
    "or on() vector(0)"
)
STUCK_TERMINATING_EXPR = (
    'sum(max by (namespace,pod) ('
    '((time() - kube_pod_deletion_timestamp{pod!=""}) > bool 600)'
    ' and on(namespace,pod) (kube_pod_deletion_timestamp{pod!=""} > bool 0)'
    ')) '
    "or on() vector(0)"
)
UPTIME_WINDOW = "365d"
# Keep the subquery step coarse so we don't request an excessive number of points.
UPTIME_STEP = "1h"
TRAEFIK_READY_EXPR = (
    "("
    'sum(kube_deployment_status_replicas_available{namespace=~"traefik|kube-system",deployment="traefik"})'
    " / clamp_min("
    'sum(kube_deployment_spec_replicas{namespace=~"traefik|kube-system",deployment="traefik"}), 1)'
    ")"
)
CONTROL_READY_FRACTION_EXPR = (
    f"(sum(kube_node_status_condition{{condition=\"Ready\",status=\"true\",node=~\"{CONTROL_REGEX}\"}})"
    f" / {CONTROL_TOTAL})"
)
UPTIME_AVAIL_EXPR = (
    f"min(({CONTROL_READY_FRACTION_EXPR}), ({TRAEFIK_READY_EXPR}))"
)

# Tie-breaker to deterministically pick one node per namespace when shares tie.
NODE_TIEBREAKER = " + ".join(
    f"({node_filter(node)}) * 1e-6 * {idx}"
    for idx, node in enumerate(CONTROL_ALL + WORKER_NODES, start=1)
)
UPTIME_AVG_EXPR = f"avg_over_time(({UPTIME_AVAIL_EXPR})[{UPTIME_WINDOW}:{UPTIME_STEP}])"
UPTIME_PERCENT_EXPR = UPTIME_AVG_EXPR
UPTIME_NINES_EXPR = f"-log10(1 - clamp_max({UPTIME_AVG_EXPR}, 0.999999999))"
UPTIME_THRESHOLDS = {
    "mode": "absolute",
    "steps": [
        {"color": "red", "value": None},
        {"color": "orange", "value": 2},
        {"color": "yellow", "value": 3},
        {"color": "green", "value": 3.5},
    ],
}
UPTIME_PERCENT_THRESHOLDS = {
    "mode": "absolute",
    "steps": [
        {"color": "red", "value": None},
        {"color": "orange", "value": 0.99},
        {"color": "yellow", "value": 0.999},
        {"color": "green", "value": 0.9999},
        {"color": "blue", "value": 0.99999},
    ],
}
PROBLEM_TABLE_EXPR = (
    "(time() - kube_pod_created{pod!=\"\"}) "
    "* on(namespace,pod) group_left(node) kube_pod_info "
    "* on(namespace,pod) group_left(phase) "
    "max by (namespace,pod,phase) (kube_pod_status_phase{phase!~\"Running|Succeeded\"})"
)
CRASHLOOP_TABLE_EXPR = (
    "(time() - kube_pod_created{pod!=\"\"}) "
    "* on(namespace,pod) group_left(node) kube_pod_info "
    "* on(namespace,pod,container) group_left(reason) "
    "max by (namespace,pod,container,reason) "
    "(kube_pod_container_status_waiting_reason{reason=~\"CrashLoopBackOff|ImagePullBackOff\"})"
)
STUCK_TABLE_EXPR = (
    "("
    "((time() - kube_pod_deletion_timestamp{pod!=\"\"}) "
    "and on(namespace,pod) (kube_pod_deletion_timestamp{pod!=\"\"} > bool 0)) "
    "* on(namespace,pod) group_left(node) kube_pod_info"
    ")"
)

NAMESPACE_SCOPE_WORKLOAD = f'namespace!~"{INFRA_REGEX}"'
NAMESPACE_SCOPE_ALL = 'namespace=~".*"'
NAMESPACE_SCOPE_INFRA = f'namespace=~"{INFRA_REGEX}"'
NAMESPACE_SCOPE_VARS = ["namespace_scope_cpu", "namespace_scope_gpu", "namespace_scope_ram"]
GLUE_LABEL = 'label_atlas_bstein_dev_glue="true"'
GLUE_JOBS = f"kube_cronjob_labels{{{GLUE_LABEL}}}"
GLUE_FILTER = f"and on(namespace,cronjob) {GLUE_JOBS}"
GLUE_LAST_SUCCESS = f"(kube_cronjob_status_last_successful_time {GLUE_FILTER})"
GLUE_LAST_SCHEDULE = f"(kube_cronjob_status_last_schedule_time {GLUE_FILTER})"
GLUE_SUSPENDED = f"(kube_cronjob_spec_suspend {GLUE_FILTER}) == 1"
GLUE_ACTIVE = f"(kube_cronjob_status_active {GLUE_FILTER})"
GLUE_LAST_SUCCESS_AGE = f"(time() - {GLUE_LAST_SUCCESS})"
GLUE_LAST_SCHEDULE_AGE = f"(time() - {GLUE_LAST_SCHEDULE})"
GLUE_LAST_SUCCESS_AGE_HOURS = f"({GLUE_LAST_SUCCESS_AGE}) / 3600"
GLUE_LAST_SCHEDULE_AGE_HOURS = f"({GLUE_LAST_SCHEDULE_AGE}) / 3600"
GLUE_STALE_WINDOW_SEC = 36 * 3600
GLUE_STALE = f"({GLUE_LAST_SUCCESS_AGE} > bool {GLUE_STALE_WINDOW_SEC})"
GLUE_MISSING = f"({GLUE_JOBS} unless on(namespace,cronjob) kube_cronjob_status_last_successful_time)"
GLUE_STALE_ACTIVE = f"({GLUE_STALE} unless on(namespace,cronjob) {GLUE_SUSPENDED})"
GLUE_MISSING_ACTIVE = f"({GLUE_MISSING} unless on(namespace,cronjob) {GLUE_SUSPENDED})"
GLUE_STALE_COUNT = f"(sum({GLUE_STALE_ACTIVE}) + count({GLUE_MISSING_ACTIVE})) or on() vector(0)"
GLUE_MISSING_COUNT = f"count({GLUE_MISSING_ACTIVE}) or on() vector(0)"
GLUE_SUSPENDED_COUNT = f"sum({GLUE_SUSPENDED}) or on() vector(0)"
ARIADNE_TASK_ERRORS_RANGE = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[$__range]))'
ARIADNE_TASK_ERRORS_24H = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[24h]))'
ARIADNE_TASK_ERRORS_1H = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[1h]))'
ARIADNE_TASK_ERRORS_30D = 'sum by (task) (increase(ariadne_task_runs_total{status="error"}[30d]))'
ARIADNE_TASK_SUCCESS_24H = 'sum by (task) (increase(ariadne_task_runs_total{status="ok"}[24h]))'
ARIADNE_TASK_RUNS_BY_STATUS_1H = 'sum by (status) (increase(ariadne_task_runs_total[1h]))'
ARIADNE_TASK_ERRORS_1H_TOTAL = 'sum(increase(ariadne_task_runs_total{status="error"}[1h]))'
ARIADNE_TASK_ERRORS_24H_TOTAL = 'sum(increase(ariadne_task_runs_total{status="error"}[24h]))'
ARIADNE_TASK_RUNS_1H_TOTAL = 'sum(increase(ariadne_task_runs_total[1h]))'
ARIADNE_TASK_ATTEMPTS_SERIES = 'sum(increase(ariadne_task_runs_total[$__interval]))'
ARIADNE_TASK_FAILURES_SERIES = 'sum(increase(ariadne_task_runs_total{status="error"}[$__interval]))'
ARIADNE_TASK_WARNINGS_SERIES = (
    'sum(increase(ariadne_task_runs_total{status!~"ok|error"}[$__interval])) or on() vector(0)'
)
ARIADNE_SCHEDULE_TASK_FILTER = 'task=~"^schedule\\..+$"'
ARIADNE_SCHEDULE_LAST_SUCCESS_HOURS = (
    f"(time() - ariadne_schedule_last_success_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}) / 3600"
)
ARIADNE_SCHEDULE_LAST_ERROR_HOURS = (
    f"(time() - ariadne_schedule_last_error_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}) / 3600"
)
ARIADNE_SCHEDULE_LAST_SUCCESS_RANGE_HOURS = (
    f"(time() - max_over_time(ariadne_schedule_last_success_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}[$__range])) / 3600"
)
ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS = (
    f"(time() - max_over_time(ariadne_schedule_last_error_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}[$__range])) / 3600"
)
ARIADNE_SCHEDULE_NEXT_RUN_HOURS = (
    f"((ariadne_schedule_next_run_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}} - time()) / 3600)"
)
ARIADNE_SCHEDULE_TASK_INDEX = f"ariadne_schedule_next_run_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}"
ARIADNE_SCHEDULE_LAST_STATUS = f"ariadne_schedule_last_status{{{ARIADNE_SCHEDULE_TASK_FILTER}}}"
ARIADNE_SCHEDULE_SIGNAL_COUNT = (
    f"count(ariadne_schedule_last_success_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}) or on() vector(0)"
)
ARIADNE_SCHEDULE_STALE_WINDOW_SEC = 36 * 3600
ARIADNE_SCHEDULE_STALE_COUNT = (
    f"sum(((time() - ariadne_schedule_last_success_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}}) > bool {ARIADNE_SCHEDULE_STALE_WINDOW_SEC}))"
    " or on() vector(0)"
)
ARIADNE_SCHEDULE_MISSING_SUCCESS_COUNT = (
    f"count((ariadne_schedule_next_run_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}} unless on(task) "
    f"ariadne_schedule_last_success_timestamp_seconds{{{ARIADNE_SCHEDULE_TASK_FILTER}}})) or on() vector(0)"
)
ARIADNE_SCHEDULE_FAILED_LAST_COUNT = (
    f"sum(((1 - ariadne_schedule_last_status{{{ARIADNE_SCHEDULE_TASK_FILTER}}}) > bool 0)) or on() vector(0)"
)
ARIADNE_SCHEDULE_RUNS_RANGE = (
    f'sum by (task) (increase(ariadne_task_runs_total{{{ARIADNE_SCHEDULE_TASK_FILTER}}}[$__range]))'
)
ARIADNE_SCHEDULE_ERRORS_RANGE = (
    f'sum by (task) (increase(ariadne_task_runs_total{{status="error",{ARIADNE_SCHEDULE_TASK_FILTER}}}[$__range]))'
)
ARIADNE_SCHEDULE_LAST_SUCCESS_RANGE_HOURS_FALLBACK = (
    f"({ARIADNE_SCHEDULE_LAST_SUCCESS_RANGE_HOURS}) or on(task) (0 * {ARIADNE_SCHEDULE_TASK_INDEX} + 999)"
)
ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS_FALLBACK = (
    f"({ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS}) or on(task) (0 * {ARIADNE_SCHEDULE_TASK_INDEX} + 999)"
)
ARIADNE_SCHEDULE_LAST_STATUS_FALLBACK = (
    f"({ARIADNE_SCHEDULE_LAST_STATUS}) or on(task) (0 * {ARIADNE_SCHEDULE_TASK_INDEX} - 1)"
)
ARIADNE_SCHEDULE_RUNS_RANGE_FALLBACK = (
    f"({ARIADNE_SCHEDULE_RUNS_RANGE}) or on(task) (0 * {ARIADNE_SCHEDULE_TASK_INDEX})"
)
ARIADNE_SCHEDULE_ERRORS_RANGE_FALLBACK = (
    f"({ARIADNE_SCHEDULE_ERRORS_RANGE}) or on(task) (0 * {ARIADNE_SCHEDULE_TASK_INDEX})"
)
JENKINS_CLEANUP_SIGNAL_COUNT = (
    "count(ariadne_jenkins_workspace_cleanup_last_run_timestamp_seconds) or on() vector(0)"
)
JENKINS_CLEANUP_RUNS_RANGE = (
    "sum by (mode, status) (increase(ariadne_jenkins_workspace_cleanup_runs_total[$__range]))"
)
JENKINS_CLEANUP_OBJECTS_RANGE = (
    "sum by (kind, action, mode) (increase(ariadne_jenkins_workspace_cleanup_objects_total[$__range]))"
)
JENKINS_CLEANUP_LAST_RUN_AGE_HOURS = (
    "((time() - ariadne_jenkins_workspace_cleanup_last_run_timestamp_seconds) / 3600) or on() vector(999)"
)
JENKINS_CLEANUP_LAST_SUCCESS_AGE_HOURS = (
    "((time() - ariadne_jenkins_workspace_cleanup_last_success_timestamp_seconds) / 3600) or on() vector(999)"
)
JENKINS_CLEANUP_LAST_DELETED = "ariadne_jenkins_workspace_cleanup_last_deleted_total or on() vector(0)"
JENKINS_CLEANUP_LAST_PLANNED = "ariadne_jenkins_workspace_cleanup_last_planned_total or on() vector(0)"
JENKINS_BUILD_WEATHER_LAST_STATUS = "ariadne_jenkins_build_weather_job_last_status"
JENKINS_BUILD_WEATHER_LAST_RUN_AGE_HOURS = (
    "(time() - ariadne_jenkins_build_weather_job_last_run_timestamp_seconds) / 3600"
)
JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS = (
    "(time() - ariadne_jenkins_build_weather_job_last_success_timestamp_seconds) / 3600"
)
JENKINS_BUILD_WEATHER_LAST_FAILURE_AGE_HOURS = (
    "(time() - ariadne_jenkins_build_weather_job_last_failure_timestamp_seconds) / 3600"
)
JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES = (
    "ariadne_jenkins_build_weather_job_last_duration_seconds / 60"
)
JENKINS_WORKSPACE_PV_STALE_COUNT = (
    'sum((kube_persistentvolume_status_phase{phase=~"Released|Failed"} > bool 0) '
    '* on(persistentvolume) group_left(claim_namespace,name) '
    'kube_persistentvolume_claim_ref{claim_namespace="jenkins",name=~"pvc-workspace-.*"}) or on() vector(0)'
)
JENKINS_WORKSPACE_PV_STALE_AGE_HOURS = (
    '((time() - kube_persistentvolume_created) / 3600) '
    '* on(persistentvolume) group_left(claim_namespace,name) '
    'kube_persistentvolume_claim_ref{claim_namespace="jenkins",name=~"pvc-workspace-.*"} '
    '* on(persistentvolume) group_left() (kube_persistentvolume_status_phase{phase=~"Released|Failed"} > bool 0)'
)
ARIADNE_ACCESS_REQUESTS = "ariadne_access_requests_total"
PLATFORM_TEST_SUITE_NAMES = [
    "ariadne",
    "metis",
    "ananke",
    "atlasbot",
    "lesavka",
    "pegasus",
    "soteria",
    "titan-iac",
    "bstein-home",
    "arcanagon",
    "data-prepper",
]
PLATFORM_TEST_SUITE_MATCHER = "|".join(PLATFORM_TEST_SUITE_NAMES)
PLATFORM_TEST_SUCCESS_EVENTS_30D = (
    f'(sum(increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status=~"ok|passed|success"}}[30d])) or on() vector(0))'
)
PLATFORM_TEST_TOTAL_EVENTS_30D = (
    f'(sum(increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[30d])) or on() vector(0))'
)
TEST_SUCCESS_RATE = (
    f"100 * ({PLATFORM_TEST_SUCCESS_EVENTS_30D}) / clamp_min(({PLATFORM_TEST_TOTAL_EVENTS_30D}), 1)"
)
TEST_FAILURES_24H_TOTAL = (
    f'(sum(increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status!~"ok|passed|success"}}[24h])) or on() vector(0))'
)
PLATFORM_TEST_FAILURES_24H_BY_SUITE = (
    f'sort_desc(sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status!~"ok|passed|success"}}[24h])))'
)
PLATFORM_TEST_ACTIVITY_30D = (
    f'sum by (suite, status) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[30d]))'
)
PLATFORM_TEST_POINT_WINDOW = "1h"
PLATFORM_TEST_SUCCESS_RATE_SUITE_TARGETS = [
    {
        "refId": chr(ord("A") + index),
        "expr": (
            f'(100 * (sum(increase(platform_quality_gate_runs_total{{suite="{suite}",status=~"ok|passed|success"}}'
            f'[{PLATFORM_TEST_POINT_WINDOW}]))) / '
            f'clamp_min((sum(increase(platform_quality_gate_runs_total{{suite="{suite}"}}[{PLATFORM_TEST_POINT_WINDOW}]))), 1)) '
            f'and on() ((sum(increase(platform_quality_gate_runs_total{{suite="{suite}"}}[{PLATFORM_TEST_POINT_WINDOW}]))) > 0)'
        ),
        "legendFormat": suite,
    }
    for index, suite in enumerate(PLATFORM_TEST_SUITE_NAMES)
]
PLATFORM_TEST_SUCCESS_RATE_24H_BY_SUITE = (
    f'sort_desc((100 * (sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}",status=~"ok|passed|success"}}[24h]))) '
    f'/ clamp_min((sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[24h]))), 1)) '
    f'and on(suite) ((sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[24h]))) > 0))'
)
QUALITY_GATE_SUITE_INDEX_30D = (
    f'sum by (suite) (increase(platform_quality_gate_runs_total{{suite=~"{PLATFORM_TEST_SUITE_MATCHER}"}}[30d]))'
)
QUALITY_GATE_COVERAGE_BY_SUITE = (
    '(max by (suite) ({__name__=~".*_quality_gate_coverage_percent"})) '
    'or on(suite) (max by (suite) (platform_quality_gate_workspace_line_coverage_percent))'
)
QUALITY_GATE_COVERAGE_BY_SUITE_WITH_MISSING = (
    f"({QUALITY_GATE_COVERAGE_BY_SUITE}) or on(suite) (0 * ({QUALITY_GATE_SUITE_INDEX_30D}) - 1)"
)
QUALITY_GATE_COVERAGE_GAP_BY_SUITE = (
    f"clamp_min(95 - ({QUALITY_GATE_COVERAGE_BY_SUITE}), 0)"
)
QUALITY_GATE_SMELL_INFRACTIONS_BY_SUITE = (
    "max by (suite) (platform_quality_gate_source_lines_over_500_total)"
)
QUALITY_GATE_SMELL_INFRACTIONS_BY_SUITE_WITH_MISSING = (
    f"({QUALITY_GATE_SMELL_INFRACTIONS_BY_SUITE}) or on(suite) (0 * ({QUALITY_GATE_SUITE_INDEX_30D}) - 1)"
)
PVC_BACKUP_AGE_HOURS_BY_PVC = (
    'sort_desc(max by (namespace, pvc) (((time() - pvc_backup_last_success_timestamp_seconds{driver="restic"}) / 3600) '
    'or on(namespace,pvc,volume,driver) ((((pvc_backup_health_reason{driver="restic",reason=~"missing|no_completed|lookup_failed|unknown_timestamp"} > 0) '
    '* (pvc_backup_count{driver="restic"} > bool 0)) * 999))))'
)
ANANKE_SELECTOR = 'job="ananke-power"'
ANANKE_UPS_DB_NAME = "Pyrphoros"
ANANKE_UPS_DB_NODE = "titan-db"
ANANKE_UPS_TETHYS_NAME = "Statera"
ANANKE_UPS_TETHYS_NODE = "titan-24"
ANANKE_UPS_DB_SELECTOR = f'{ANANKE_SELECTOR},source="{ANANKE_UPS_DB_NAME}"'
ANANKE_UPS_TETHYS_SELECTOR = f'{ANANKE_SELECTOR},source="{ANANKE_UPS_TETHYS_NAME}"'
ANANKE_UPS_ON_BATTERY = f"sum(ananke_ups_on_battery{{{ANANKE_SELECTOR}}}) or on() vector(0)"
ANANKE_UPS_LOW_BATTERY = f"sum(ananke_ups_low_battery{{{ANANKE_SELECTOR}}}) or on() vector(0)"
ANANKE_UPS_RUNTIME_MIN = f"min(ananke_ups_runtime_seconds{{{ANANKE_SELECTOR}}}) or on() vector(0)"
ANANKE_UPS_RUNTIME_HEADROOM_PERCENT = (
    f"100 * min(ananke_ups_runtime_seconds{{{ANANKE_SELECTOR}}}) / "
    f"clamp_min(max(ananke_ups_threshold_seconds{{{ANANKE_SELECTOR}}}), 1)"
)
ANANKE_UPS_TRIGGER_COUNT_1D = f"increase(ananke_shutdown_triggers_total{{{ANANKE_SELECTOR}}}[1d]) or on() vector(0)"
ANANKE_UPS_RUNTIME_DB = (
    f'max(ananke_ups_runtime_seconds{{{ANANKE_UPS_DB_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_RUNTIME_TETHYS = (
    f'max(ananke_ups_runtime_seconds{{{ANANKE_UPS_TETHYS_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_ON_BATTERY_DB = (
    f'max(ananke_ups_on_battery{{{ANANKE_UPS_DB_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_ON_BATTERY_TETHYS = (
    f'max(ananke_ups_on_battery{{{ANANKE_UPS_TETHYS_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_THRESHOLD_DB = (
    f'clamp_min(max(ananke_ups_threshold_seconds{{{ANANKE_UPS_DB_SELECTOR}}}), 1)'
)
ANANKE_UPS_THRESHOLD_TETHYS = (
    f'clamp_min(max(ananke_ups_threshold_seconds{{{ANANKE_UPS_TETHYS_SELECTOR}}}), 1)'
)
ANANKE_UPS_BATTERY_CHARGE_DB = (
    f'max(ananke_ups_battery_charge_percent{{{ANANKE_UPS_DB_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_BATTERY_CHARGE_TETHYS = (
    f'max(ananke_ups_battery_charge_percent{{{ANANKE_UPS_TETHYS_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_LOAD_DB = (
    f'max(ananke_ups_load_percent{{{ANANKE_UPS_DB_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_LOAD_TETHYS = (
    f'max(ananke_ups_load_percent{{{ANANKE_UPS_TETHYS_SELECTOR}}}) or on() vector(0)'
)
ANANKE_UPS_DRAW_WATTS_DB = (
    f'max((ananke_ups_load_percent{{{ANANKE_UPS_DB_SELECTOR}}} '
    f'* ananke_ups_power_nominal_watts{{{ANANKE_UPS_DB_SELECTOR}}}) / 100) or on() vector(0)'
)
ANANKE_UPS_DRAW_WATTS_TETHYS = (
    f'max((ananke_ups_load_percent{{{ANANKE_UPS_TETHYS_SELECTOR}}} '
    f'* ananke_ups_power_nominal_watts{{{ANANKE_UPS_TETHYS_SELECTOR}}}) / 100) or on() vector(0)'
)
ANANKE_UPS_DRAW_WATTS_TOTAL = (
    f'sum((ananke_ups_load_percent{{{ANANKE_SELECTOR}}} * ananke_ups_power_nominal_watts{{{ANANKE_SELECTOR}}}) / 100) '
    "or on() vector(0)"
)
ANANKE_UPS_DRAW_WATTS_DB_SERIES = (
    f'((ananke_ups_load_percent{{{ANANKE_UPS_DB_SELECTOR}}} '
    f'* ananke_ups_power_nominal_watts{{{ANANKE_UPS_DB_SELECTOR}}}) / 100)'
)
ANANKE_UPS_DRAW_WATTS_TETHYS_SERIES = (
    f'((ananke_ups_load_percent{{{ANANKE_UPS_TETHYS_SELECTOR}}} '
    f'* ananke_ups_power_nominal_watts{{{ANANKE_UPS_TETHYS_SELECTOR}}}) / 100)'
)
ANANKE_UPS_DRAW_WATTS_TOTAL_SERIES = (
    f'sum((ananke_ups_load_percent{{{ANANKE_SELECTOR}}} * ananke_ups_power_nominal_watts{{{ANANKE_SELECTOR}}}) / 100)'
)
ANANKE_UPS_RUNTIME_BY_SOURCE = f"ananke_ups_runtime_seconds{{{ANANKE_SELECTOR}}}"
ANANKE_UPS_LOAD_BY_SOURCE = f"ananke_ups_load_percent{{{ANANKE_SELECTOR}}}"
ANANKE_UPS_CHARGE_BY_SOURCE = f"ananke_ups_battery_charge_percent{{{ANANKE_SELECTOR}}}"
ANANKE_UPS_TRIGGER_BY_SOURCE = f"ananke_ups_trigger_active{{{ANANKE_SELECTOR}}}"


def ups_discharge_risk_expr(on_battery_expr, runtime_expr, shutdown_threshold_expr):
    return (
        f"((({on_battery_expr}) > bool 0) * ("
        f"1 + (({runtime_expr}) < bool (3 * ({shutdown_threshold_expr}))) + "
        f"(({runtime_expr}) < bool (2 * ({shutdown_threshold_expr})))"
        f")) or on() vector(0)"
    )


ANANKE_UPS_DISCHARGE_RISK_DB = ups_discharge_risk_expr(
    ANANKE_UPS_ON_BATTERY_DB,
    ANANKE_UPS_RUNTIME_DB,
    ANANKE_UPS_THRESHOLD_DB,
)
ANANKE_UPS_DISCHARGE_RISK_TETHYS = ups_discharge_risk_expr(
    ANANKE_UPS_ON_BATTERY_TETHYS,
    ANANKE_UPS_RUNTIME_TETHYS,
    ANANKE_UPS_THRESHOLD_TETHYS,
)
CLIMATE_SENSOR_COUNT = "count(typhon_temperature_celsius) or on() vector(0)"
CLIMATE_DEDUP_LABELS = "job,instance,pod,service,endpoint,namespace"
CLIMATE_TEMP_SERIES = (
    f"max without ({CLIMATE_DEDUP_LABELS}) (typhon_temperature_celsius != 0)"
)
CLIMATE_TEMP_FAHRENHEIT_SERIES = f"({CLIMATE_TEMP_SERIES}) * 9 / 5 + 32"
CLIMATE_PRESSURE_SERIES = (
    f"max without ({CLIMATE_DEDUP_LABELS}) (typhon_vpd_kpa != 0)"
)
CLIMATE_HUMIDITY_SERIES = (
    f"max without ({CLIMATE_DEDUP_LABELS}) (typhon_relative_humidity_percent != 0)"
)
CLIMATE_TEMP_MAX = f"max({CLIMATE_TEMP_SERIES}) or on() vector(0)"
CLIMATE_TEMP_FAHRENHEIT_MAX = f"max({CLIMATE_TEMP_FAHRENHEIT_SERIES}) or on() vector(0)"
CLIMATE_PRESSURE_CURRENT = f"max({CLIMATE_PRESSURE_SERIES}) or on() vector(0)"
CLIMATE_HUMIDITY_MAX = f"max({CLIMATE_HUMIDITY_SERIES}) or on() vector(0)"
CLIMATE_TEMP_MIN_BOUND_SERIES = f"(min_over_time({CLIMATE_TEMP_SERIES}[$__range]) - 0.08)"
CLIMATE_TEMP_MAX_BOUND_SERIES = f"(max_over_time({CLIMATE_TEMP_SERIES}[$__range]) + 0.08)"
CLIMATE_HUMIDITY_MIN_BOUND_SERIES = (
    f"clamp_min((min_over_time({CLIMATE_HUMIDITY_SERIES}[$__range]) - 0.35), 0)"
)
CLIMATE_HUMIDITY_MAX_BOUND_SERIES = (
    f"clamp_max((max_over_time({CLIMATE_HUMIDITY_SERIES}[$__range]) + 0.35), 100)"
)
CLIMATE_PRESSURE_MIN_BOUND_SERIES = (
    f"clamp_min((min_over_time({CLIMATE_PRESSURE_SERIES}[$__range]) - 0.03), 0)"
)
CLIMATE_PRESSURE_MAX_BOUND_SERIES = (
    f"(max_over_time({CLIMATE_PRESSURE_SERIES}[$__range]) + 0.03)"
)
CLIMATE_FAN_OUTLET_CURRENT = (
    f'max(max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group="outlet"}})) or on() vector(0)'
)
CLIMATE_FAN_INSIDE_INLET_CURRENT = (
    f'max(max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group="inside_inlet"}})) or on() vector(0)'
)
CLIMATE_FAN_OUTSIDE_INLET_CURRENT = (
    f'max(max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group="outside_inlet"}})) or on() vector(0)'
)
CLIMATE_FAN_INTERIOR_CURRENT = (
    f'max(max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group=~"interior|unknown"}})) or on() vector(0)'
)
CLIMATE_FAN_OUTLET_SERIES = (
    f'max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group="outlet"}})'
)
CLIMATE_FAN_INSIDE_INLET_SERIES = (
    f'max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group="inside_inlet"}})'
)
CLIMATE_FAN_OUTSIDE_INLET_SERIES = (
    f'max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group="outside_inlet"}})'
)
CLIMATE_FAN_INTERIOR_SERIES = (
    f'max without ({CLIMATE_DEDUP_LABELS}) (typhon_fan_speed_level{{fan_group=~"interior|unknown"}})'
)


def with_metric_label(expr, metric):
    return f'label_replace(({expr}), "metric", "{metric}", "__name__", ".*")'


def with_ups_metric_labels(expr, ups, metric):
    return (
        f'label_replace('
        f'label_replace(({expr}), "ups", "{ups}", "__name__", ".*"), '
        f'"metric", "{metric}", "__name__", ".*"'
        f')'
    )


CLIMATE_CURRENT_ROW_EXPR = " or ".join(
    [
        with_metric_label(CLIMATE_TEMP_MAX, "Temp °C"),
        with_metric_label(CLIMATE_TEMP_FAHRENHEIT_MAX, "Temp °F"),
        with_metric_label(CLIMATE_HUMIDITY_MAX, "Humidity"),
        with_metric_label(CLIMATE_PRESSURE_CURRENT, "Pressure"),
    ]
)
CLIMATE_FAN_CURRENT_ROW_EXPR = " or ".join(
    [
        with_metric_label(f"round({CLIMATE_FAN_OUTLET_CURRENT})", "Outlet"),
        with_metric_label(f"round({CLIMATE_FAN_INSIDE_INLET_CURRENT})", "Inlet In"),
        with_metric_label(f"round({CLIMATE_FAN_OUTSIDE_INLET_CURRENT})", "Inlet Out"),
        with_metric_label(f"round({CLIMATE_FAN_INTERIOR_CURRENT})", "Interior"),
    ]
)
UPS_CURRENT_ROW_EXPR = " or ".join(
    [
        with_ups_metric_labels(ANANKE_UPS_DRAW_WATTS_DB, ANANKE_UPS_DB_NAME, "Draw"),
        with_ups_metric_labels(ANANKE_UPS_RUNTIME_DB, ANANKE_UPS_DB_NAME, "Runtime"),
        with_ups_metric_labels(ANANKE_UPS_DRAW_WATTS_TETHYS, ANANKE_UPS_TETHYS_NAME, "Draw"),
        with_ups_metric_labels(ANANKE_UPS_RUNTIME_TETHYS, ANANKE_UPS_TETHYS_NAME, "Runtime"),
    ]
)
POSTGRES_CONN_USED = (
    'label_replace(sum(pg_stat_activity_count), "conn", "used", "__name__", ".*") '
    'or label_replace(max(pg_settings_max_connections), "conn", "max", "__name__", ".*")'
)
POSTGRES_CONN_HOTTEST = 'topk(1, sum by (datname) (pg_stat_activity_count))'
ONEOFF_JOB_OWNER = (
    'label_replace(kube_job_owner{owner_kind="CronJob"}, "owner_name", "$1", "job_name", "(.*)")'
)
ONEOFF_JOB_PODS = f'(kube_pod_owner{{owner_kind="Job"}} unless on(namespace, owner_name) {ONEOFF_JOB_OWNER})'
ONEOFF_JOB_POD_AGE_HOURS = (
    '((time() - kube_pod_start_time{pod!=""}) / 3600) '
    f'* on(namespace,pod) group_left(owner_name) {ONEOFF_JOB_PODS} '
    '* on(namespace,pod) group_left(phase) '
    'max by (namespace,pod,phase) (kube_pod_status_phase{phase=~"Running|Succeeded"})'
)
GLUE_LAST_SUCCESS_RANGE_HOURS = f"(time() - max_over_time({GLUE_LAST_SUCCESS}[$__range])) / 3600"
GLUE_LAST_SCHEDULE_RANGE_HOURS = f"(time() - max_over_time({GLUE_LAST_SCHEDULE}[$__range])) / 3600"
GPU_NODES = ["titan-20", "titan-21", "titan-22", "titan-24"]
GPU_NODE_REGEX = "|".join(GPU_NODES)
TRAEFIK_ROUTER_EXPR = "sum by (router) (rate(traefik_router_requests_total[5m]))"
TRAEFIK_NET_INGRESS = (
    'sum(rate(container_network_receive_bytes_total{namespace="traefik",pod=~"traefik-.*"}[5m]))'
    " or on() vector(0)"
)
TRAEFIK_NET_EGRESS = (
    'sum(rate(container_network_transmit_bytes_total{namespace="traefik",pod=~"traefik-.*"}[5m]))'
    " or on() vector(0)"
)
NET_CLUSTER_RX = (
    'sum(rate(container_network_receive_bytes_total{namespace!="",pod!="",container!=""}[5m]))'
    " or on() vector(0)"
)
NET_CLUSTER_TX = (
    'sum(rate(container_network_transmit_bytes_total{namespace!="",pod!="",container!=""}[5m]))'
    " or on() vector(0)"
)
PHYSICAL_NET_FILTER = 'device!~"lo|cni.*|veth.*|flannel.*|docker.*|virbr.*|vxlan.*|wg.*"'
NET_NODE_RX_PHYS = (
    f'sum(rate(node_network_receive_bytes_total{{{PHYSICAL_NET_FILTER}}}[5m])) or on() vector(0)'
)
NET_NODE_TX_PHYS = (
    f'sum(rate(node_network_transmit_bytes_total{{{PHYSICAL_NET_FILTER}}}[5m])) or on() vector(0)'
)
NET_TOTAL_EXPR = NET_NODE_TX_PHYS
NET_INGRESS_EXPR = NET_NODE_RX_PHYS
NET_EGRESS_EXPR = NET_NODE_TX_PHYS
NET_INTERNAL_EXPR = (
    'sum(rate(container_network_receive_bytes_total{namespace!="traefik",pod!=""}[5m]) '
    '+ rate(container_network_transmit_bytes_total{namespace!="traefik",pod!=""}[5m]))'
    ' or on() vector(0)'
)
APISERVER_5XX_RATE = 'sum(rate(apiserver_request_total{code=~"5.."}[5m]))'
APISERVER_P99_LATENCY_MS = (
    "histogram_quantile(0.99, sum by (le) (rate(apiserver_request_duration_seconds_bucket[5m]))) * 1000"
)
ETCD_P99_LATENCY_MS = (
    "histogram_quantile(0.99, sum by (le) (rate(etcd_request_duration_seconds_bucket[5m]))) * 1000"
)
TRAEFIK_TOTAL_5M = "sum(rate(traefik_entrypoint_requests_total[5m]))"
TRAEFIK_SUCCESS_5M = 'sum(rate(traefik_entrypoint_requests_total{code!~"5.."}[5m]))'
TRAEFIK_SLI_5M = f"({TRAEFIK_SUCCESS_5M}) / clamp_min({TRAEFIK_TOTAL_5M}, 1)"
TRAEFIK_P99_LATENCY_MS = (
    "histogram_quantile(0.99, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket[5m]))) * 1000"
)
TRAEFIK_P95_LATENCY_MS = (
    "histogram_quantile(0.95, sum by (le) (rate(traefik_entrypoint_request_duration_seconds_bucket[5m]))) * 1000"
)
SLO_AVAILABILITY = 0.999


def traefik_sli(window):
    total = f'sum(rate(traefik_entrypoint_requests_total[{window}]))'
    success = f'sum(rate(traefik_entrypoint_requests_total{{code!~"5.."}}[{window}]))'
    return f"({success}) / clamp_min({total}, 1)"


def traefik_burn(window):
    sli = traefik_sli(window)
    return f"(1 - ({sli})) / {1 - SLO_AVAILABILITY}"

# ---------------------------------------------------------------------------
# Panel factories
# ---------------------------------------------------------------------------


def stat_panel(
    panel_id,
    title,
    expr,
    grid,
    *,
    unit="none",
    decimals=None,
    thresholds=None,
    text_mode="value",
    legend=None,
    instant=False,
    value_suffix=None,
    links=None,
    targets=None,
    field_overrides=None,
    description=None,
    orientation=None,
    wide_layout=None,
    graph_mode="area",
    justify_mode="center",
    title_size=None,
    value_size=None,
):
    """Return a Grafana stat panel definition."""
    defaults = {
        "color": {"mode": "thresholds"},
        "mappings": [],
        "thresholds": thresholds
        or {
            "mode": "absolute",
            "steps": [
                {"color": "rgba(115, 115, 115, 1)", "value": None},
                {"color": "green", "value": 1},
            ],
        },
        "unit": unit,
        "custom": {"displayMode": "auto"},
    }
    if value_suffix:
        defaults["custom"]["valueSuffix"] = value_suffix
    if decimals is not None:
        defaults["decimals"] = decimals
    target_list = targets if targets is not None else [{"expr": expr, "refId": "A"}]

    panel = {
        "id": panel_id,
        "type": "stat",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": target_list,
        "fieldConfig": {"defaults": defaults, "overrides": field_overrides or []},
        "options": {
            "colorMode": "value",
            "graphMode": graph_mode,
            "justifyMode": justify_mode,
            "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
            "textMode": text_mode,
        },
    }
    if orientation:
        panel["options"]["orientation"] = orientation
    if wide_layout is not None:
        panel["options"]["wideLayout"] = wide_layout
    if title_size is not None or value_size is not None:
        panel["options"]["text"] = {}
        if title_size is not None:
            panel["options"]["text"]["titleSize"] = title_size
        if value_size is not None:
            panel["options"]["text"]["valueSize"] = value_size
    if legend and len(panel["targets"]) == 1:
        panel["targets"][0]["legendFormat"] = legend
    if instant:
        for t in panel["targets"]:
            t.setdefault("instant", True)
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def gauge_panel(
    panel_id,
    title,
    expr,
    grid,
    *,
    min_value=0,
    max_value=1,
    thresholds=None,
    links=None,
):
    return {
        "id": panel_id,
        "type": "gauge",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": [{"expr": expr, "refId": "A"}],
        "fieldConfig": {
            "defaults": {
                "min": min_value,
                "max": max_value,
                "thresholds": thresholds
                or {
                    "mode": "absolute",
                    "steps": [
                        {"color": "green", "value": None},
                        {"color": "red", "value": max_value},
                    ],
                },
            },
            "overrides": [],
        },
        "options": {
            "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
            "orientation": "auto",
            "showThresholdMarkers": False,
            "showThresholdLabels": False,
        },
        **({"links": links} if links else {}),
    }


def timeseries_panel(
    panel_id,
    title,
    expr,
    grid,
    *,
    unit="none",
    max_value=None,
    legend=None,
    legend_display="table",
    legend_placement="bottom",
    legend_calcs=None,
    time_from=None,
    links=None,
    targets=None,
    field_overrides=None,
    description=None,
):
    """Return a Grafana time-series panel definition."""
    target_list = targets if targets is not None else [{"expr": expr, "refId": "A"}]
    panel = {
        "id": panel_id,
        "type": "timeseries",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": target_list,
        "fieldConfig": {"defaults": {"unit": unit}, "overrides": field_overrides or []},
        "options": {
            "legend": {
                "displayMode": legend_display,
                "placement": legend_placement,
            },
            "tooltip": {"mode": "multi"},
        },
    }
    if max_value is not None:
        panel["fieldConfig"]["defaults"]["max"] = max_value
    if legend and len(panel["targets"]) == 1:
        panel["targets"][0]["legendFormat"] = legend
    if legend_calcs:
        panel["options"]["legend"]["calcs"] = legend_calcs
    if time_from:
        panel["timeFrom"] = time_from
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def canvas_metric_grid_panel(
    panel_id,
    title,
    grid,
    *,
    targets,
    field_overrides=None,
    links=None,
    description=None,
    metric_size=26,
    label_size=12,
    color_fields=None,
    thresholds=None,
):
    """Return a canvas panel with a deterministic 2x2 metric layout."""

    if color_fields is None:
        color_fields = [targets[0]["legendFormat"], targets[1]["legendFormat"], targets[2]["legendFormat"], targets[3]["legendFormat"]]

    def text_element(name, text, left, top):
        return {
            "type": "text",
            "name": name,
            "constraint": {"horizontal": "left", "vertical": "top"},
            "placement": {
                "left": left,
                "top": top,
                "width": 146,
                "height": 14,
            },
            "background": {"color": {"fixed": "transparent"}},
            "border": {"color": {"fixed": "transparent"}},
            "config": {
                "align": "center",
                "valign": "middle",
                "size": label_size,
                "color": {"fixed": "text"},
                "text": {
                    "fixed": text,
                },
            },
            "links": [],
        }

    def metric_element(name, field, left, top):
        return {
            "type": "metric-value",
            "name": name,
            "constraint": {"horizontal": "left", "vertical": "top"},
            "placement": {
                "left": left,
                "top": top,
                "width": 146,
                "height": 42,
            },
            "background": {"color": {"fixed": "transparent"}},
            "border": {"color": {"fixed": "transparent"}},
            "config": {
                "align": "center",
                "valign": "middle",
                "size": metric_size,
                "color": {"field": color_fields[int(name.split()[-1]) - 1], "fixed": "text"},
                "text": {
                    "mode": "field",
                    "field": field,
                    "fixed": "",
                },
            },
            "links": [],
        }

    panel = {
        "id": panel_id,
        "type": "canvas",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": targets,
        "fieldConfig": {
            "defaults": {
                "mappings": [],
                "thresholds": thresholds
                or {
                    "mode": "absolute",
                    "steps": [
                        {"color": "rgba(115, 115, 115, 1)", "value": None},
                        {"color": "green", "value": 1},
                    ],
                },
                "color": {"mode": "thresholds"},
            },
            "overrides": field_overrides or [],
        },
        "options": {
            "inlineEditing": False,
            "showAdvancedTypes": True,
            "panZoom": False,
            "infinitePan": False,
            "root": {
                "type": "frame",
                "name": f"{title} frame",
                "elements": [
                    text_element("Cell 1 label", targets[0]["legendFormat"], 12, 18),
                    metric_element("Cell 1", targets[0]["legendFormat"], 12, 32),
                    text_element("Cell 2 label", targets[1]["legendFormat"], 168, 18),
                    metric_element("Cell 2", targets[1]["legendFormat"], 168, 32),
                    text_element("Cell 3 label", targets[2]["legendFormat"], 12, 76),
                    metric_element("Cell 3", targets[2]["legendFormat"], 12, 90),
                    text_element("Cell 4 label", targets[3]["legendFormat"], 168, 76),
                    metric_element("Cell 4", targets[3]["legendFormat"], 168, 90),
                ],
                "background": {"color": {"fixed": "transparent"}},
                "border": {"color": {"fixed": "transparent"}},
            },
        },
    }
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def canvas_two_metric_row_panel(
    panel_id,
    title,
    grid,
    *,
    targets,
    field_overrides=None,
    links=None,
    description=None,
    metric_size=30,
    metric_top=10,
    metric_height=42,
    color_fields=None,
    thresholds=None,
):
    """Return a canvas panel with two values rendered side-by-side and no labels."""

    if color_fields is None:
        color_fields = [targets[0]["legendFormat"], targets[1]["legendFormat"]]

    def metric_element(name, field, left, top, color_field):
        return {
            "type": "metric-value",
            "name": name,
            "constraint": {"horizontal": "left", "vertical": "top"},
            "placement": {
                "left": left,
                "top": top,
                "width": 146,
                "height": metric_height,
            },
            "background": {"color": {"fixed": "transparent"}},
            "border": {"color": {"fixed": "transparent"}},
            "config": {
                "align": "center",
                "valign": "middle",
                "size": metric_size,
                "color": {"field": color_field, "fixed": "text"},
                "text": {
                    "mode": "field",
                    "field": field,
                    "fixed": "",
                },
            },
            "links": [],
        }

    panel = {
        "id": panel_id,
        "type": "canvas",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": targets,
        "fieldConfig": {
            "defaults": {
                "mappings": [],
                "thresholds": thresholds
                or {
                    "mode": "absolute",
                    "steps": [
                        {"color": "rgba(115, 115, 115, 1)", "value": None},
                        {"color": "green", "value": 1},
                    ],
                },
                "color": {"mode": "thresholds"},
            },
            "overrides": field_overrides or [],
        },
        "options": {
            "inlineEditing": False,
            "showAdvancedTypes": True,
            "panZoom": False,
            "infinitePan": False,
            "root": {
                "type": "frame",
                "name": f"{title} frame",
                "elements": [
                    metric_element("Cell 1", targets[0]["legendFormat"], 12, metric_top, color_fields[0]),
                    metric_element("Cell 2", targets[1]["legendFormat"], 168, metric_top, color_fields[1]),
                ],
                "background": {"color": {"fixed": "transparent"}},
                "border": {"color": {"fixed": "transparent"}},
            },
        },
    }
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def table_panel(
    panel_id,
    title,
    expr,
    grid,
    *,
    unit="none",
    transformations=None,
    instant=False,
    options=None,
    filterable=True,
    footer=None,
    format=None,
    targets=None,
    field_overrides=None,
    links=None,
    description=None,
):
    """Return a Grafana table panel definition."""
    # Optional PromQL subquery helpers in expr: share(), etc.
    panel_options = {"showHeader": True, "columnFilters": False}
    if options:
        panel_options.update(options)
    if footer is not None:
        panel_options["footer"] = footer
    field_defaults = {"unit": unit, "custom": {"filterable": filterable}}
    target_list = targets if targets is not None else [{"expr": expr, "refId": "A"}]
    if instant:
        for target in target_list:
            target.setdefault("instant", True)
    if format:
        for target in target_list:
            target.setdefault("format", format)
    panel = {
        "id": panel_id,
        "type": "table",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": target_list,
        "fieldConfig": {"defaults": field_defaults, "overrides": field_overrides or []},
        "options": panel_options,
    }
    if transformations:
        panel["transformations"] = transformations
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def pie_panel(panel_id, title, expr, grid, *, links=None, description=None):
    """Return a pie chart panel with readable namespace labels."""
    panel = {
        "id": panel_id,
        "type": "piechart",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": [{"expr": expr, "refId": "A", "legendFormat": "{{namespace}}"}],
        "fieldConfig": {
            "defaults": {
                "unit": "percent",
                "color": {"mode": "palette-classic"},
            },
            "overrides": [],
        },
        "options": {
            "legend": {"displayMode": "list", "placement": "right"},
            "pieType": "pie",
            "displayLabels": [],
            "tooltip": {"mode": "single"},
            "colorScheme": "interpolateSpectral",
            "colorBy": "value",
            "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
        },
    }
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def namespace_scope_variable(var_name, label):
    options = [
        {
            "text": "workload namespaces only",
            "value": NAMESPACE_SCOPE_WORKLOAD,
            "selected": True,
        },
        {"text": "all namespaces", "value": NAMESPACE_SCOPE_ALL, "selected": False},
        {
            "text": "infrastructure namespaces only",
            "value": NAMESPACE_SCOPE_INFRA,
            "selected": False,
        },
    ]
    query = (
        "workload namespaces only : "
        + NAMESPACE_SCOPE_WORKLOAD
        + ",all namespaces : "
        + NAMESPACE_SCOPE_ALL
        + ",infrastructure namespaces only : "
        + NAMESPACE_SCOPE_INFRA
    )
    return {
        "name": var_name,
        "label": label,
        "type": "custom",
        "query": query,
        "current": {"text": options[0]["text"], "value": options[0]["value"], "selected": True},
        "options": options,
        "hide": 2,
        "multi": False,
        "includeAll": False,
        "refresh": 1,
        "sort": 0,
        "skipUrlSync": False,
    }


def namespace_scope_links(var_name):
    def with_value(value):
        encoded = urllib.parse.quote(value, safe="")
        params = []
        for other in NAMESPACE_SCOPE_VARS:
            if other == var_name:
                params.append(f"var-{other}={encoded}")
            else:
                params.append(f"var-{other}=${{{other}}}")
        return "?" + "&".join(params)

    return [
        {"title": "Workload namespaces only", "url": with_value(NAMESPACE_SCOPE_WORKLOAD), "targetBlank": False},
        {"title": "All namespaces", "url": with_value(NAMESPACE_SCOPE_ALL), "targetBlank": False},
        {
            "title": "Infrastructure namespaces only",
            "url": with_value(NAMESPACE_SCOPE_INFRA),
            "targetBlank": False,
        },
    ]


def bargauge_panel(
    panel_id,
    title,
    expr,
    grid,
    *,
    unit="none",
    legend=None,
    links=None,
    limit=None,
    sort_order="desc",
    thresholds=None,
    decimals=None,
    instant=False,
    overrides=None,
):
    """Return a bar gauge panel with label-aware reduction."""
    cleaned_expr = expr.strip()
    if not cleaned_expr.startswith(("sort(", "sort_desc(")):
        if sort_order == "desc":
            expr = f"sort_desc({expr})"
        elif sort_order == "asc":
            expr = f"sort({expr})"
    panel = {
        "id": panel_id,
        "type": "bargauge",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": [
            {
                "expr": expr,
                "refId": "A",
                "legendFormat": legend or "{{node}}",
                **({"instant": True} if instant else {}),
            }
        ],
        "fieldConfig": {
            "defaults": {
                "unit": unit,
                "min": 0,
                "max": 100 if unit == "percent" else None,
                "thresholds": thresholds
                or {
                    "mode": "absolute",
                    "steps": [
                        {"color": "green", "value": None},
                        {"color": "yellow", "value": 50},
                        {"color": "orange", "value": 70},
                        {"color": "red", "value": 85},
                    ],
                },
            },
            "overrides": [],
        },
        "options": {
            "displayMode": "gradient",
            "orientation": "horizontal",
            "reduceOptions": {
                "calcs": ["lastNotNull"],
                "fields": "",
                "values": False,
            },
        },
    }
    if overrides:
        panel["fieldConfig"]["overrides"].extend(overrides)
    if decimals is not None:
        panel["fieldConfig"]["defaults"]["decimals"] = decimals
    if links:
        panel["links"] = links
    # Keep bars ordered by value descending for readability.
    panel["transformations"] = [
        {
            "id": "sortBy",
            "options": {"fields": ["Value"], "order": sort_order},
        }
    ]
    if limit:
        panel["transformations"].append({"id": "limit", "options": {"limit": limit}})
    return panel


def _jenkins_weather_status_expr(base_expr, comparator):
    return (
        f"({base_expr}) and on(exported_job,job_url,weather_icon) "
        f"({JENKINS_BUILD_WEATHER_LAST_STATUS} {comparator})"
    )


def jenkins_weather_bargauge_panel(
    panel_id,
    title,
    expr,
    grid,
    *,
    unit="h",
    decimals=2,
    sort_order="asc",
    limit=12,
    thresholds=None,
    links=None,
    description=None,
):
    panel = {
        "id": panel_id,
        "type": "bargauge",
        "title": title,
        "datasource": PROM_DS,
        "gridPos": grid,
        "targets": [
            {
                "refId": "A",
                "expr": _jenkins_weather_status_expr(expr, "== 1"),
                "legendFormat": "{{weather_icon}} {{exported_job}}",
                "instant": True,
            },
            {
                "refId": "B",
                "expr": _jenkins_weather_status_expr(expr, "== 0"),
                "legendFormat": "{{weather_icon}} {{exported_job}}",
                "instant": True,
            },
            {
                "refId": "C",
                "expr": _jenkins_weather_status_expr(expr, "== 2"),
                "legendFormat": "{{weather_icon}} {{exported_job}}",
                "instant": True,
            },
            {
                "refId": "D",
                "expr": _jenkins_weather_status_expr(expr, "< 0"),
                "legendFormat": "{{weather_icon}} {{exported_job}}",
                "instant": True,
            },
        ],
        "fieldConfig": {
            "defaults": {
                "unit": unit,
                "min": 0,
                "thresholds": thresholds
                or {
                    "mode": "absolute",
                    "steps": [
                        {"color": "green", "value": None},
                        {"color": "yellow", "value": 6},
                        {"color": "orange", "value": 24},
                        {"color": "red", "value": 72},
                    ],
                },
                "links": [
                    {
                        "title": "Open Jenkins job",
                        "url": "${__field.labels.job_url}",
                        "targetBlank": True,
                    }
                ],
            },
            "overrides": [
                {
                    "matcher": {"id": "byFrameRefID", "options": "A"},
                    "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}],
                },
                {
                    "matcher": {"id": "byFrameRefID", "options": "B"},
                    "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}],
                },
                {
                    "matcher": {"id": "byFrameRefID", "options": "C"},
                    "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "yellow"}}],
                },
                {
                    "matcher": {"id": "byFrameRefID", "options": "D"},
                    "properties": [{"id": "color", "value": {"mode": "fixed", "fixedColor": "gray"}}],
                },
            ],
        },
        "options": {
            "displayMode": "basic",
            "orientation": "horizontal",
            "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
        },
        "transformations": [
            {"id": "sortBy", "options": {"fields": ["Value"], "order": sort_order}},
        ],
    }
    if decimals is not None:
        panel["fieldConfig"]["defaults"]["decimals"] = decimals
    if limit:
        panel["transformations"].append({"id": "limit", "options": {"limit": limit}})
    if links:
        panel["links"] = links
    if description:
        panel["description"] = description
    return panel


def text_panel(panel_id, title, content, grid):
    return {
        "id": panel_id,
        "type": "text",
        "title": title,
        "gridPos": grid,
        "datasource": None,
        "options": {"mode": "markdown", "content": content},
    }


def link_to(uid):
    return [{"title": f"Open {uid} dashboard", "url": f"/d/{uid}", "targetBlank": True}]


# ---------------------------------------------------------------------------
# Dashboard builders
# ---------------------------------------------------------------------------


def build_overview():
    panels = []

    count_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 1},
            {"color": "orange", "value": 2},
            {"color": "red", "value": 3},
        ],
    }
    age_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 6},
            {"color": "orange", "value": 24},
            {"color": "red", "value": 48},
        ],
    }

    row1_stats = [
        {
            "id": 2,
            "title": "Control Plane Ready",
            "expr": f'sum(kube_node_status_condition{{condition="Ready",status="true",node=~"{CONTROL_REGEX}"}})',
            "kind": "gauge",
            "max_value": CONTROL_TOTAL,
            "thresholds": {
                "mode": "absolute",
                "steps": [
                    {"color": "red", "value": None},
                    {"color": "green", "value": CONTROL_TOTAL},
                ],
            },
        },
        {
            "id": 3,
            "title": "Control Plane Workloads",
            "expr": CONTROL_WORKLOADS_EXPR,
            "kind": "stat",
            "thresholds": {
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 2},
                    {"color": "red", "value": 3},
                ],
            },
            "links": link_to("atlas-pods"),
        },
        {
            "id": 5,
            "title": "Stuck Terminating",
            "expr": STUCK_TERMINATING_EXPR,
            "kind": "stat",
            "thresholds": {
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 2},
                    {"color": "red", "value": 3},
                ],
            },
            "links": link_to("atlas-pods"),
        },
        {
            "id": 27,
            "title": "Atlas Availability",
            "expr": UPTIME_PERCENT_EXPR,
            "kind": "stat",
            "thresholds": UPTIME_PERCENT_THRESHOLDS,
            "unit": "percentunit",
            "decimals": 4,
            "text_mode": "value",
        },
        {
            "id": 4,
            "title": "Problem Pods",
            "expr": PROBLEM_PODS_EXPR,
            "kind": "stat",
            "thresholds": {
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 2},
                    {"color": "red", "value": 3},
                ],
            },
            "links": link_to("atlas-pods"),
        },
        {
            "id": 6,
            "title": "CrashLoop / ImagePull",
            "expr": CRASHLOOP_EXPR,
            "kind": "stat",
            "thresholds": {
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 2},
                    {"color": "red", "value": 3},
                ],
            },
            "links": link_to("atlas-pods"),
        },
        {
            "id": 1,
            "title": "Workers Ready",
            "expr": f'sum(kube_node_status_condition{{condition="Ready",status="true",node=~"{WORKER_REGEX}"}})',
            "kind": "gauge",
            "max_value": WORKER_TOTAL,
            "thresholds": {
                "mode": "absolute",
                "steps": [
                    {"color": "red", "value": None},
                    {"color": "orange", "value": WORKER_TOTAL - 2},
                    {"color": "yellow", "value": WORKER_TOTAL - 1},
                    {"color": "green", "value": WORKER_TOTAL},
                ],
            },
        },
    ]

    def gauge_grid(idx):
        width = GAUGE_WIDTHS[idx] if idx < len(GAUGE_WIDTHS) else 4
        x = sum(GAUGE_WIDTHS[:idx])
        return width, x

    for idx, item in enumerate(row1_stats):
        panel_id = item["id"]
        width, x = gauge_grid(idx)
        grid = {"h": 5, "w": width, "x": x, "y": 0}
        kind = item.get("kind", "gauge")
        if kind == "stat":
            panels.append(
                stat_panel(
                    panel_id,
                    item["title"],
                    item["expr"],
                    grid,
                    thresholds=item.get("thresholds"),
            legend=None,
            links=item.get("links"),
            text_mode=item.get("text_mode", "value"),
            value_suffix=item.get("value_suffix"),
            unit=item.get("unit", "none"),
            decimals=item.get("decimals"),
        )
    )
        else:
            panels.append(
                gauge_panel(
                    panel_id,
                    item["title"],
                    item["expr"],
                    grid,
                    min_value=0,
                    max_value=item.get("max_value", 5),
                    thresholds=item.get("thresholds"),
                    links=item.get("links"),
                )
            )

    top_health_panels = [
        (7, "Hottest node: CPU", topk_with_node(node_cpu_expr()), "percent"),
        (8, "Hottest node: RAM", topk_with_node(node_mem_expr()), "percent"),
        (9, "Hottest node: NET (rx+tx)", topk_with_node(node_net_expr()), "Bps"),
        (10, "Hottest node: I/O (r+w)", topk_with_node(node_io_expr()), "Bps"),
        (23, "Astreae Usage", astreae_usage_expr("/mnt/astreae"), "percent"),
        (24, "Asteria Usage", astreae_usage_expr("/mnt/asteria"), "percent"),
        (25, "Astreae Free", astreae_free_expr("/mnt/astreae"), "decbytes"),
        (26, "Asteria Free", astreae_free_expr("/mnt/asteria"), "decbytes"),
    ]
    for idx, (panel_id, title, expr, unit) in enumerate(top_health_panels):
        is_hottest_panel = panel_id in {7, 8, 9, 10}
        panels.append(
            stat_panel(
                panel_id,
                title,
                f"{expr}",
                {"h": 2, "w": 3, "x": 3 * idx, "y": 5},
                unit=unit,
                thresholds=PERCENT_THRESHOLDS if unit == "percent" else None,
                text_mode="name_and_value" if is_hottest_panel else "value",
                legend="{{node}}" if is_hottest_panel else None,
                instant=is_hottest_panel,
                links=link_to("atlas-storage" if panel_id in {23, 24, 25, 26} else "atlas-nodes"),
            )
        )

    mail_bounce_rate_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 5},
            {"color": "orange", "value": 8},
            {"color": "red", "value": 10},
        ],
    }
    mail_limit_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 70},
            {"color": "orange", "value": 85},
            {"color": "red", "value": 95},
        ],
    }
    mail_success_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "red", "value": None},
            {"color": "orange", "value": 90},
            {"color": "yellow", "value": 95},
            {"color": "green", "value": 98},
        ],
    }
    panels.append(
        canvas_two_metric_row_panel(
            40,
            f"{ANANKE_UPS_DB_NAME} UPS Current",
            {"h": 2, "w": 6, "x": 0, "y": 7},
            metric_size=24,
            metric_top=8,
            metric_height=34,
            targets=[
                {"refId": "A", "expr": ANANKE_UPS_DRAW_WATTS_DB, "legendFormat": "Draw", "instant": True},
                {"refId": "B", "expr": ANANKE_UPS_RUNTIME_DB, "legendFormat": "Runtime", "instant": True},
            ],
            field_overrides=[
                {"matcher": {"id": "byName", "options": "Draw"}, "properties": [{"id": "unit", "value": "watt"}]},
                {"matcher": {"id": "byName", "options": "Runtime"}, "properties": [{"id": "unit", "value": "s"}]},
            ],
            links=link_to("atlas-power"),
        )
    )
    panels.append(
        canvas_two_metric_row_panel(
            144,
            f"{ANANKE_UPS_TETHYS_NAME} UPS Current",
            {"h": 3, "w": 6, "x": 0, "y": 9},
            metric_size=30,
            targets=[
                {"refId": "A", "expr": ANANKE_UPS_DRAW_WATTS_TETHYS, "legendFormat": "Draw", "instant": True},
                {"refId": "B", "expr": ANANKE_UPS_RUNTIME_TETHYS, "legendFormat": "Runtime", "instant": True},
            ],
            field_overrides=[
                {"matcher": {"id": "byName", "options": "Draw"}, "properties": [{"id": "unit", "value": "watt"}]},
                {"matcher": {"id": "byName", "options": "Runtime"}, "properties": [{"id": "unit", "value": "s"}]},
            ],
            links=link_to("atlas-power"),
        )
    )
    panels.append(
        timeseries_panel(
            41,
            "UPS History (Power Draw)",
            None,
            {"h": 5, "w": 6, "x": 6, "y": 7},
            unit="watt",
            targets=[
                {"refId": "A", "expr": ANANKE_UPS_DRAW_WATTS_DB_SERIES, "legendFormat": ANANKE_UPS_DB_NAME},
                {"refId": "B", "expr": ANANKE_UPS_DRAW_WATTS_TETHYS_SERIES, "legendFormat": ANANKE_UPS_TETHYS_NAME},
                {"refId": "C", "expr": ANANKE_UPS_DRAW_WATTS_TOTAL_SERIES, "legendFormat": "combined"},
            ],
            legend_display="table",
            legend_placement="right",
            links=link_to("atlas-power"),
        )
    )
    panels.append(
        canvas_two_metric_row_panel(
            42,
            "Current Enclosure Temperature",
            {"h": 3, "w": 6, "x": 0, "y": 12},
            metric_size=30,
            targets=[
                {"refId": "A", "expr": CLIMATE_TEMP_MAX, "legendFormat": "°C", "instant": True},
                {"refId": "B", "expr": CLIMATE_TEMP_FAHRENHEIT_MAX, "legendFormat": "°F", "instant": True},
            ],
            field_overrides=[
                {"matcher": {"id": "byName", "options": "°C"}, "properties": [{"id": "unit", "value": "celsius"}]},
                {"matcher": {"id": "byName", "options": "°F"}, "properties": [{"id": "unit", "value": "fahrenheit"}]},
            ],
            links=link_to("atlas-power"),
        )
    )
    panels.append(
        canvas_two_metric_row_panel(
            143,
            "Current Enclosure Climate",
            {"h": 3, "w": 6, "x": 0, "y": 15},
            metric_size=30,
            targets=[
                {"refId": "A", "expr": CLIMATE_HUMIDITY_MAX, "legendFormat": "%RH", "instant": True},
                {"refId": "B", "expr": CLIMATE_PRESSURE_CURRENT, "legendFormat": "kPa", "instant": True},
            ],
            field_overrides=[
                {"matcher": {"id": "byName", "options": "%RH"}, "properties": [{"id": "unit", "value": "suffix:%RH"}]},
                {"matcher": {"id": "byName", "options": "kPa"}, "properties": [{"id": "unit", "value": "suffix:kPa"}]},
            ],
            links=link_to("atlas-power"),
        )
    )
    climate_history_panel = timeseries_panel(
            43,
            "Enclosure Climate History",
            None,
            {"h": 6, "w": 6, "x": 6, "y": 12},
            unit="none",
            targets=[
                {"refId": "A", "expr": CLIMATE_TEMP_SERIES, "legendFormat": "C"},
                {"refId": "B", "expr": CLIMATE_HUMIDITY_SERIES, "legendFormat": "RH"},
                {"refId": "C", "expr": CLIMATE_PRESSURE_SERIES, "legendFormat": "P"},
                {"refId": "D", "expr": CLIMATE_TEMP_MIN_BOUND_SERIES, "legendFormat": "C bound min"},
                {"refId": "E", "expr": CLIMATE_TEMP_MAX_BOUND_SERIES, "legendFormat": "C bound max"},
                {"refId": "F", "expr": CLIMATE_HUMIDITY_MIN_BOUND_SERIES, "legendFormat": "RH bound min"},
                {"refId": "G", "expr": CLIMATE_HUMIDITY_MAX_BOUND_SERIES, "legendFormat": "RH bound max"},
                {"refId": "H", "expr": CLIMATE_PRESSURE_MIN_BOUND_SERIES, "legendFormat": "P bound min"},
                {"refId": "I", "expr": CLIMATE_PRESSURE_MAX_BOUND_SERIES, "legendFormat": "P bound max"},
            ],
            field_overrides=[
                {
                    "matcher": {"id": "byName", "options": "C"},
                    "properties": [
                        {"id": "unit", "value": "suffix:°C"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisPlacement", "value": "left"},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                },
                {
                    "matcher": {"id": "byRegexp", "options": "C bound .*"},
                    "properties": [
                        {"id": "unit", "value": "suffix:°C"},
                        {"id": "custom.axisPlacement", "value": "left"},
                        {"id": "custom.axisCenteredZero", "value": False},
                        {"id": "custom.hideFrom", "value": {"legend": True, "tooltip": True, "viz": False}},
                        {"id": "custom.lineWidth", "value": 0},
                        {"id": "custom.fillOpacity", "value": 0},
                        {"id": "custom.showPoints", "value": "never"},
                        {"id": "color", "value": {"mode": "fixed", "fixedColor": "transparent"}},
                    ],
                },
                {
                    "matcher": {"id": "byName", "options": "RH"},
                    "properties": [
                        {"id": "unit", "value": "suffix:%"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisPlacement", "value": "right"},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                },
                {
                    "matcher": {"id": "byRegexp", "options": "RH bound .*"},
                    "properties": [
                        {"id": "unit", "value": "suffix:%"},
                        {"id": "custom.axisPlacement", "value": "right"},
                        {"id": "custom.axisCenteredZero", "value": False},
                        {"id": "custom.hideFrom", "value": {"legend": True, "tooltip": True, "viz": False}},
                        {"id": "custom.lineWidth", "value": 0},
                        {"id": "custom.fillOpacity", "value": 0},
                        {"id": "custom.showPoints", "value": "never"},
                        {"id": "color", "value": {"mode": "fixed", "fixedColor": "transparent"}},
                    ],
                },
                {
                    "matcher": {"id": "byName", "options": "P"},
                    "properties": [
                        {"id": "unit", "value": "suffix:kPa"},
                        {"id": "custom.axisPlacement", "value": "right"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                },
                {
                    "matcher": {"id": "byRegexp", "options": "P bound .*"},
                    "properties": [
                        {"id": "unit", "value": "suffix:kPa"},
                        {"id": "custom.axisPlacement", "value": "right"},
                        {"id": "custom.axisCenteredZero", "value": False},
                        {"id": "custom.hideFrom", "value": {"legend": True, "tooltip": True, "viz": False}},
                        {"id": "custom.lineWidth", "value": 0},
                        {"id": "custom.fillOpacity", "value": 0},
                        {"id": "custom.showPoints", "value": "never"},
                        {"id": "color", "value": {"mode": "fixed", "fixedColor": "transparent"}},
                    ],
                },
            ],
            legend_display="list",
            legend_placement="bottom",
            links=link_to("atlas-power"),
            description="Temperature on left axis, humidity and pressure on right axis with dynamic bound series so small swings remain visible.",
        )
    climate_history_panel["fieldConfig"]["defaults"]["custom"] = {
        "drawStyle": "line",
        "lineInterpolation": "linear",
        "lineWidth": 2,
        "fillOpacity": 10,
        "showPoints": "never",
        "spanNulls": True,
    }
    panels.append(climate_history_panel)
    panels.append(
        stat_panel(
            140,
            "Fan Activity",
            CLIMATE_FAN_CURRENT_ROW_EXPR,
            {"h": 6, "w": 6, "x": 12, "y": 12},
            unit="none",
            decimals=0,
            text_mode="name_and_value",
            legend="{{metric}}",
            instant=True,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 7},
                    {"color": "red", "value": 9},
                ],
            },
            field_overrides=[
                {"matcher": {"id": "byName", "options": "Outlet"}, "properties": [{"id": "decimals", "value": 0}]},
                {"matcher": {"id": "byName", "options": "Inlet In"}, "properties": [{"id": "decimals", "value": 0}]},
                {"matcher": {"id": "byName", "options": "Inlet Out"}, "properties": [{"id": "decimals", "value": 0}]},
                {"matcher": {"id": "byName", "options": "Interior"}, "properties": [{"id": "decimals", "value": 0}]},
            ],
            links=link_to("atlas-power"),
            orientation="vertical",
            wide_layout=False,
        )
    )
    panels.append(
        timeseries_panel(
            141,
            "Fan History (0-10)",
            None,
            {"h": 6, "w": 6, "x": 18, "y": 12},
            unit="none",
            max_value=10,
            targets=[
                {"refId": "A", "expr": CLIMATE_FAN_OUTLET_SERIES, "legendFormat": "Inside Outlet"},
                {"refId": "B", "expr": CLIMATE_FAN_INSIDE_INLET_SERIES, "legendFormat": "Inside Inlet"},
                {"refId": "C", "expr": CLIMATE_FAN_OUTSIDE_INLET_SERIES, "legendFormat": "Outside Inlet"},
                {"refId": "D", "expr": CLIMATE_FAN_INTERIOR_SERIES, "legendFormat": "Interior Fans"},
            ],
            legend_display="table",
            legend_placement="right",
            links=link_to("atlas-power"),
        )
    )

    panels.append(
        bargauge_panel(
            44,
            "One-off Job Pods (age hours)",
            ONEOFF_JOB_POD_AGE_HOURS,
            {"h": 5, "w": 8, "x": 0, "y": 32},
            unit="h",
            instant=True,
            legend="{{namespace}}/{{pod}}",
            thresholds=age_thresholds,
            limit=12,
            decimals=2,
            links=link_to("atlas-jobs"),
        )
    )
    panels.append(
        {
            "id": 45,
            "type": "timeseries",
            "title": "Ariadne Attempts / Failures",
            "datasource": PROM_DS,
            "gridPos": {"h": 5, "w": 6, "x": 12, "y": 7},
            "targets": [
                {"expr": ARIADNE_TASK_ATTEMPTS_SERIES, "refId": "A", "legendFormat": "Attempts"},
                {"expr": ARIADNE_TASK_FAILURES_SERIES, "refId": "B", "legendFormat": "Failures"},
            ],
            "fieldConfig": {
                "defaults": {"unit": "none"},
                "overrides": [
                    {
                        "matcher": {"id": "byName", "options": "Attempts"},
                        "properties": [
                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}
                        ],
                    },
                    {
                        "matcher": {"id": "byName", "options": "Failures"},
                        "properties": [
                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}
                        ],
                    },
                ],
            },
            "options": {
                "legend": {"displayMode": "table", "placement": "right"},
                "tooltip": {"mode": "multi"},
            },
            "links": link_to("atlas-jobs"),
        }
    )
    test_success = timeseries_panel(
        46,
        "Platform Test Success Rate",
        None,
        {"h": 5, "w": 6, "x": 18, "y": 7},
        unit="percent",
        targets=PLATFORM_TEST_SUCCESS_RATE_SUITE_TARGETS,
        legend_display="table",
        legend_placement="right",
        legend_calcs=["lastNotNull"],
        links=link_to("atlas-testing"),
    )
    test_success["fieldConfig"]["defaults"]["min"] = 0
    test_success["fieldConfig"]["defaults"]["max"] = 100
    test_success["fieldConfig"]["defaults"]["custom"] = {
        "drawStyle": "line",
        "lineInterpolation": "linear",
        "lineWidth": 2,
        "fillOpacity": 10,
        "showPoints": "always",
        "pointSize": 4,
        "spanNulls": True,
    }
    test_success["timeFrom"] = "7d"
    test_success["description"] = (
        "Per-run interval pass points (0-100) for each software suite over the last 7 days. Points are connected to show trend; missing-run intervals are ignored."
    )
    panels.append(test_success)
    panels.append(
        bargauge_panel(
            47,
            "PVC Backup Health / Age",
            PVC_BACKUP_AGE_HOURS_BY_PVC,
            {"h": 5, "w": 24, "x": 0, "y": 56},
            unit="h",
            instant=True,
            legend="{{namespace}}/{{pvc}}",
            sort_order="desc",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 6},
                    {"color": "orange", "value": 12},
                    {"color": "red", "value": 24},
                ],
            },
        )
    )
    panels[-1]["links"] = link_to("atlas-storage")
    panels[-1]["description"] = (
        "Backup age in hours computed from last-success timestamps for restic-managed PVCs. "
        "PVCs that have backup history but currently no successful backup (missing/no_completed/error) are pinned to 999h for visibility."
    )
    panels.append(
        jenkins_weather_bargauge_panel(
            142,
            "Jenkins Last Run (hours ago, newest first)",
            JENKINS_BUILD_WEATHER_LAST_RUN_AGE_HOURS,
            {"h": 5, "w": 5, "x": 8, "y": 32},
            unit="h",
            decimals=2,
            sort_order="asc",
            limit=8,
            thresholds=age_thresholds,
            links=link_to("atlas-jobs"),
            description=(
                "Jenkins weather list from Ariadne: icon + name with status color and age since the most recent run."
            ),
        )
    )
    panels.append(
        jenkins_weather_bargauge_panel(
            243,
            "Jenkins Last Success (hours ago)",
            JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS,
            {"h": 5, "w": 5, "x": 13, "y": 32},
            unit="h",
            decimals=2,
            sort_order="asc",
            limit=8,
            thresholds=age_thresholds,
            links=link_to("atlas-jobs"),
            description="Per-job age since the most recent successful run.",
        )
    )
    panels.append(
        jenkins_weather_bargauge_panel(
            244,
            "Jenkins Last Duration (minutes)",
            JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES,
            {"h": 5, "w": 6, "x": 18, "y": 32},
            unit="m",
            decimals=2,
            sort_order="desc",
            limit=8,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 5},
                    {"color": "orange", "value": 15},
                    {"color": "red", "value": 30},
                ],
            },
            links=link_to("atlas-jobs"),
            description="Most recent completed build duration per Jenkins job.",
        )
    )

    panels.append(
        stat_panel(
            30,
            "Mail Sent (1d)",
            'max(postmark_outbound_sent{window="1d"})',
            {"h": 2, "w": 4, "x": 0, "y": 18},
            unit="none",
            links=link_to("atlas-mail"),
        )
    )
    panels.append(
        {
            "id": 31,
            "type": "stat",
            "title": "Mail Bounces (1d)",
            "datasource": PROM_DS,
            "gridPos": {"h": 2, "w": 4, "x": 8, "y": 18},
            "targets": [
                {
                    "expr": 'max(postmark_outbound_bounce_rate{window="1d"})',
                    "refId": "A",
                    "legendFormat": "Rate",
                },
                {
                    "expr": 'max(postmark_outbound_bounced{window="1d"})',
                    "refId": "B",
                    "legendFormat": "Count",
                },
            ],
            "fieldConfig": {
                "defaults": {
                    "color": {"mode": "thresholds"},
                    "custom": {"displayMode": "auto"},
                    "thresholds": mail_bounce_rate_thresholds,
                    "unit": "none",
                },
                "overrides": [
                    {
                        "matcher": {"id": "byName", "options": "Rate"},
                        "properties": [{"id": "unit", "value": "percent"}],
                    },
                    {
                        "matcher": {"id": "byName", "options": "Count"},
                        "properties": [{"id": "unit", "value": "none"}],
                    },
                ],
            },
            "options": {
                "colorMode": "value",
                "graphMode": "area",
                "justifyMode": "center",
                "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
                "textMode": "name_and_value",
            },
            "links": link_to("atlas-mail"),
        }
    )
    panels.append(
        stat_panel(
            32,
            "Mail Success Rate (1d)",
            'clamp_min(100 - max(postmark_outbound_bounce_rate{window="1d"}), 0)',
            {"h": 2, "w": 4, "x": 4, "y": 18},
            unit="percent",
            thresholds=mail_success_thresholds,
            decimals=1,
            links=link_to("atlas-mail"),
        )
    )
    panels.append(
        stat_panel(
            33,
            "Mail Limit Used (30d)",
            "max(postmark_sending_limit_used_percent)",
            {"h": 2, "w": 4, "x": 12, "y": 18},
            unit="percent",
            thresholds=mail_limit_thresholds,
            decimals=1,
            links=link_to("atlas-mail"),
        )
    )
    panels.append(
        stat_panel(
            34,
            "Postgres Connections Used",
            POSTGRES_CONN_USED,
            {"h": 2, "w": 4, "x": 16, "y": 18},
            decimals=0,
            text_mode="name_and_value",
            legend="{{conn}}",
            instant=True,
        )
    )
    panels.append(
        stat_panel(
            35,
            "Postgres Hottest Connections",
            POSTGRES_CONN_HOTTEST,
            {"h": 2, "w": 4, "x": 20, "y": 18},
            unit="none",
            decimals=0,
            text_mode="name_and_value",
            legend="{{datname}}",
            instant=True,
        )
    )

    cpu_scope = "$namespace_scope_cpu"
    gpu_scope = "$namespace_scope_gpu"
    ram_scope = "$namespace_scope_ram"

    panels.append(
        pie_panel(
            11,
            "Namespace CPU Share",
            namespace_cpu_share_expr(cpu_scope),
            {"h": 9, "w": 8, "x": 0, "y": 23},
            links=namespace_scope_links("namespace_scope_cpu"),
            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
        )
    )
    panels.append(
        pie_panel(
            12,
            "Namespace GPU Share",
            namespace_gpu_share_expr(gpu_scope),
            {"h": 9, "w": 8, "x": 8, "y": 23},
            links=namespace_scope_links("namespace_scope_gpu"),
            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
        )
    )
    panels.append(
        pie_panel(
            13,
            "Namespace RAM Share",
            namespace_ram_share_expr(ram_scope),
            {"h": 9, "w": 8, "x": 16, "y": 23},
            links=namespace_scope_links("namespace_scope_ram"),
            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
        )
    )

    worker_filter = f"{WORKER_REGEX}"
    panels.append(
        timeseries_panel(
            14,
            "Worker Node CPU",
            node_cpu_expr(worker_filter),
            {"h": 12, "w": 12, "x": 0, "y": 44},
            unit="percent",
            legend="{{node}}",
            legend_calcs=["last"],
            legend_display="table",
            legend_placement="right",
            links=link_to("atlas-nodes"),
        )
    )
    panels.append(
        timeseries_panel(
            15,
            "Worker Node RAM",
            node_mem_expr(worker_filter),
            {"h": 12, "w": 12, "x": 12, "y": 44},
            unit="percent",
            legend="{{node}}",
            legend_calcs=["last"],
            legend_display="table",
            legend_placement="right",
            links=link_to("atlas-nodes"),
        )
    )

    panels.append(
        timeseries_panel(
            16,
            "Control plane CPU",
            node_cpu_expr(CONTROL_ALL_REGEX),
            {"h": 10, "w": 12, "x": 0, "y": 56},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            17,
            "Control plane RAM",
            node_mem_expr(CONTROL_ALL_REGEX),
            {"h": 10, "w": 12, "x": 12, "y": 56},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
        )
    )

    panels.append(
        pie_panel(
            28,
            "Node Pod Share",
            '(sum(kube_pod_info{pod!="" , node!=""}) by (node) / clamp_min(sum(kube_pod_info{pod!="" , node!=""}), 1)) * 100',
            {"h": 10, "w": 12, "x": 0, "y": 66},
        )
    )
    panels.append(
        bargauge_panel(
            29,
            "Top Nodes by Pod Count",
            'topk(12, sum(kube_pod_info{pod!="" , node!=""}) by (node))',
            {"h": 10, "w": 12, "x": 12, "y": 66},
            unit="none",
            limit=12,
            decimals=0,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 50},
                    {"color": "orange", "value": 75},
                    {"color": "red", "value": 100},
                ],
            },
            instant=True,
        )
    )

    panels.append(
        timeseries_panel(
            18,
            "Cluster Ingress Throughput",
            NET_INGRESS_EXPR,
            {"h": 7, "w": 8, "x": 0, "y": 37},
            unit="Bps",
            legend="Ingress (Traefik)",
            legend_display="list",
            legend_placement="bottom",
            links=link_to("atlas-network"),
        )
    )
    panels.append(
        timeseries_panel(
            19,
            "Cluster Egress Throughput",
            NET_EGRESS_EXPR,
            {"h": 7, "w": 8, "x": 8, "y": 37},
            unit="Bps",
            legend="Egress (Traefik)",
            legend_display="list",
            legend_placement="bottom",
            links=link_to("atlas-network"),
        )
    )
    panels.append(
        timeseries_panel(
            20,
            "Intra-Cluster Throughput",
            NET_INTERNAL_EXPR,
            {"h": 7, "w": 8, "x": 16, "y": 37},
            unit="Bps",
            legend="Internal traffic",
            legend_display="list",
            legend_placement="bottom",
            links=link_to("atlas-network"),
        )
    )

    panels.append(
        timeseries_panel(
            21,
            "Root Filesystem Usage",
            root_usage_expr(),
            {"h": 16, "w": 12, "x": 0, "y": 76},
            unit="percent",
            legend="{{node}}",
            legend_calcs=["last"],
            legend_display="table",
            legend_placement="right",
            time_from="30d",
            links=link_to("atlas-storage"),
        )
    )
    panels.append(
        timeseries_panel(
            22,
            "Nodes Closest to Full Astraios Disks",
            astraios_usage_expr(),
            {"h": 16, "w": 12, "x": 12, "y": 76},
            unit="percent",
            legend="{{node}}",
            legend_calcs=["last"],
            legend_display="table",
            legend_placement="right",
            time_from="1w",
            links=link_to("atlas-storage"),
        )
    )
    return {
        "uid": "atlas-overview",
        "title": "Atlas Overview",
        "folderUid": PUBLIC_FOLDER,
        "editable": False,
        "annotations": {"list": []},
        "panels": panels,
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "overview"],
        "templating": {
            "list": [
                namespace_scope_variable("namespace_scope_cpu", "CPU namespace filter"),
                namespace_scope_variable("namespace_scope_gpu", "GPU namespace filter"),
                namespace_scope_variable("namespace_scope_ram", "RAM namespace filter"),
            ]
        },
        "time": {"from": "now-1h", "to": "now"},
        "refresh": "1m",
        "links": [],
    }


def build_pods_dashboard():
    panels = []
    panels.append(
        stat_panel(
            1,
            "Problem Pods",
            PROBLEM_PODS_EXPR,
            {"h": 4, "w": 6, "x": 0, "y": 0},
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "red", "value": 1},
                ],
            },
        )
    )
    panels.append(
        stat_panel(
            2,
            "CrashLoop / ImagePull",
            CRASHLOOP_EXPR,
            {"h": 4, "w": 6, "x": 6, "y": 0},
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "red", "value": 1},
                ],
            },
        )
    )
    panels.append(
        stat_panel(
            3,
            "Stuck Terminating (>10m)",
            STUCK_TERMINATING_EXPR,
            {"h": 4, "w": 6, "x": 12, "y": 0},
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "red", "value": 1},
                ],
            },
        )
    )
    panels.append(
        stat_panel(
            4,
            "Control Plane Workloads",
            f'sum(kube_pod_info{{node=~"{CONTROL_REGEX}",namespace!~"{CP_ALLOWED_NS}"}})',
            {"h": 4, "w": 6, "x": 18, "y": 0},
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "red", "value": 1},
                ],
            },
        )
    )

    panels.append(
        table_panel(
            5,
            "Pods Not Running",
            PROBLEM_TABLE_EXPR,
            {"h": 10, "w": 24, "x": 0, "y": 4},
            unit="s",
            transformations=[{"id": "labelsToFields", "options": {}}],
        )
    )
    panels.append(
        table_panel(
            6,
            "CrashLoop / ImagePull",
            CRASHLOOP_TABLE_EXPR,
            {"h": 10, "w": 24, "x": 0, "y": 14},
            unit="s",
            transformations=[{"id": "labelsToFields", "options": {}}],
        )
    )
    panels.append(
        table_panel(
            7,
            "Terminating >10m",
            STUCK_TABLE_EXPR,
            {"h": 10, "w": 24, "x": 0, "y": 24},
            unit="s",
            transformations=[
                {"id": "labelsToFields", "options": {}},
                {"id": "filterByValue", "options": {"match": "Value", "operator": "gt", "value": 600}},
            ],
        )
    )
    panels.append(
        pie_panel(
            8,
            "Node Pod Share",
            '(sum(kube_pod_info{pod!="" , node!=""}) by (node) / clamp_min(sum(kube_pod_info{pod!="" , node!=""}), 1)) * 100',
            {"h": 8, "w": 12, "x": 12, "y": 34},
        )
    )
    panels.append(
        bargauge_panel(
            9,
            "Top Nodes by Pod Count",
            'topk(12, sum(kube_pod_info{pod!="" , node!=""}) by (node))',
            {"h": 8, "w": 12, "x": 0, "y": 34},
            unit="none",
            limit=12,
            decimals=0,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 50},
                    {"color": "orange", "value": 75},
                    {"color": "red", "value": 100},
                ],
            },
            instant=True,
        )
    )

    share_expr = (
        '(sum by (namespace,node) (kube_pod_info{pod!="" , node!=""}) '
        '/ on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=""}), 1) * 100)'
    )
    rank_terms = [
        f"(sum by (node) (kube_node_info{{node=\"{node}\"}}) * 0 + {idx * 1e-3})"
        for idx, node in enumerate(CONTROL_ALL + WORKER_NODES, start=1)
    ]
    rank_expr = " or ".join(rank_terms)
    score_expr = f"{share_expr} + on(node) group_left() ({rank_expr})"
    mask_expr = (
        f"{score_expr} == bool on(namespace) group_left() "
        f"(max by (namespace) ({score_expr}))"
    )
    panels.append(
        table_panel(
            10,
            "Namespace Plurality by Node v27",
            (
                f"{share_expr} * on(namespace,node) group_left() "
                f"({mask_expr})"
            ),
            {"h": 8, "w": 24, "x": 0, "y": 42},
            unit="percent",
            transformations=[
                {"id": "labelsToFields", "options": {}},
                {"id": "organize", "options": {"excludeByName": {"Time": True}}},
                {"id": "filterByValue", "options": {"match": "Value", "operator": "gt", "value": 0}},
                {
                    "id": "sortBy",
                    "options": {"fields": ["Value"], "order": "desc"},
                },
                {
                    "id": "groupBy",
                    "options": {
                        "fields": {
                            "namespace": {
                                "aggregations": [
                                    {"field": "Value", "operation": "max"},
                                    {"field": "node", "operation": "first"},
                                ]
                            }
                        },
                        "rowBy": ["namespace"],
                    },
                },
            ],
            instant=True,
            options={"showColumnFilters": False},
            filterable=False,
            footer={"show": False, "fields": "", "calcs": []},
            format="table",
        )
    )

    return {
        "uid": "atlas-pods",
        "title": "Atlas Pods",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-12h", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "pods"],
    }


def build_nodes_dashboard():
    panels = []
    panels.append(
        stat_panel(
            1,
            "Worker Nodes Ready",
            f'sum(kube_node_status_condition{{condition="Ready",status="true",node=~"{WORKER_REGEX}"}})',
            {"h": 4, "w": 8, "x": 0, "y": 0},
            value_suffix=WORKER_SUFFIX,
        )
    )
    panels.append(
        stat_panel(
            2,
            "Control Plane Ready",
            f'sum(kube_node_status_condition{{condition="Ready",status="true",node=~"{CONTROL_REGEX}"}})',
            {"h": 4, "w": 8, "x": 8, "y": 0},
            value_suffix=CONTROL_SUFFIX,
        )
    )
    panels.append(
        stat_panel(
            3,
            "Control Plane Workloads",
            f'sum(kube_pod_info{{node=~"{CONTROL_REGEX}",namespace!~"{CP_ALLOWED_NS}"}})',
            {"h": 4, "w": 8, "x": 16, "y": 0},
        )
    )
    panels.append(
        stat_panel(
            9,
            "API Server 5xx rate",
            APISERVER_5XX_RATE,
            {"h": 4, "w": 8, "x": 0, "y": 4},
            unit="req/s",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 0.05},
                    {"color": "orange", "value": 0.2},
                    {"color": "red", "value": 0.5},
                ],
            },
            decimals=3,
        )
    )
    panels.append(
        stat_panel(
            10,
            "API Server P99 latency",
            APISERVER_P99_LATENCY_MS,
            {"h": 4, "w": 8, "x": 8, "y": 4},
            unit="ms",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 250},
                    {"color": "orange", "value": 400},
                    {"color": "red", "value": 600},
                ],
            },
            decimals=1,
        )
    )
    panels.append(
        stat_panel(
            11,
            "etcd P99 latency",
            ETCD_P99_LATENCY_MS,
            {"h": 4, "w": 8, "x": 16, "y": 4},
            unit="ms",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 50},
                    {"color": "orange", "value": 100},
                    {"color": "red", "value": 200},
                ],
            },
            decimals=1,
        )
    )
    panels.append(
        timeseries_panel(
            4,
            "Node CPU",
            node_cpu_expr(),
            {"h": 9, "w": 24, "x": 0, "y": 8},
            unit="percent",
            legend="{{node}}",
            legend_calcs=["last"],
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            5,
            "Node RAM",
            node_mem_expr(),
            {"h": 9, "w": 24, "x": 0, "y": 17},
            unit="percent",
            legend="{{node}}",
            legend_calcs=["last"],
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            6,
            "Control Plane (incl. titan-db) CPU",
            node_cpu_expr(CONTROL_ALL_REGEX),
            {"h": 9, "w": 12, "x": 0, "y": 26},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            7,
            "Control Plane (incl. titan-db) RAM",
            node_mem_expr(CONTROL_ALL_REGEX),
            {"h": 9, "w": 12, "x": 12, "y": 26},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            8,
            "Root Filesystem Usage",
            root_usage_expr(),
            {"h": 9, "w": 24, "x": 0, "y": 35},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
            time_from="30d",
        )
    )
    panels.append(
        timeseries_panel(
            9,
            "Astraios Usage",
            astraios_usage_expr(),
            {"h": 9, "w": 24, "x": 0, "y": 44},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
            time_from="30d",
        )
    )
    return {
        "uid": "atlas-nodes",
        "title": "Atlas Nodes",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-12h", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "nodes"],
    }


def build_storage_dashboard():
    panels = []
    panels.append(
        stat_panel(
            1,
            "Astreae Usage",
            astreae_usage_expr("/mnt/astreae"),
            {"h": 5, "w": 6, "x": 0, "y": 0},
            unit="percent",
            thresholds=PERCENT_THRESHOLDS,
        )
    )
    panels.append(
        stat_panel(
            2,
            "Asteria Usage",
            astreae_usage_expr("/mnt/asteria"),
            {"h": 5, "w": 6, "x": 6, "y": 0},
            unit="percent",
            thresholds=PERCENT_THRESHOLDS,
        )
    )
    panels.append(
        stat_panel(
            3,
            "Astreae Free",
            astreae_free_expr("/mnt/astreae"),
            {"h": 5, "w": 6, "x": 12, "y": 0},
            unit="decbytes",
        )
    )
    panels.append(
        stat_panel(
            4,
            "Asteria Free",
            astreae_free_expr("/mnt/asteria"),
            {"h": 5, "w": 6, "x": 18, "y": 0},
            unit="decbytes",
        )
    )
    panels.append(
        timeseries_panel(
            5,
            "Astreae Per-Node Usage",
            filesystem_usage_expr("/mnt/astreae", LONGHORN_NODE_REGEX),
            {"h": 9, "w": 12, "x": 0, "y": 5},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
            time_from="30d",
        )
    )
    panels.append(
        timeseries_panel(
            6,
            "Asteria Per-Node Usage",
            filesystem_usage_expr("/mnt/asteria", LONGHORN_NODE_REGEX),
            {"h": 9, "w": 12, "x": 12, "y": 5},
            unit="percent",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
            time_from="30d",
        )
    )
    panels.append(
        timeseries_panel(
            7,
            "Astreae Usage History",
            astreae_usage_expr("/mnt/astreae"),
            {"h": 9, "w": 12, "x": 0, "y": 14},
            unit="percent",
            time_from="90d",
        )
    )
    panels.append(
        timeseries_panel(
            8,
            "Asteria Usage History",
            astreae_usage_expr("/mnt/asteria"),
            {"h": 9, "w": 12, "x": 12, "y": 14},
            unit="percent",
            time_from="90d",
        )
    )
    panels.append(
        stat_panel(
            30,
            "Maintenance Sweepers Ready",
            'kube_daemonset_status_number_ready{namespace="maintenance",daemonset="node-image-sweeper"} / on(namespace,daemonset) kube_daemonset_status_desired_number_scheduled{namespace="maintenance",daemonset="node-image-sweeper"} * 100',
            {"h": 4, "w": 12, "x": 0, "y": 44},
            unit="percent",
            thresholds=PERCENT_THRESHOLDS,
        )
    )
    panels.append(
        stat_panel(
            31,
            "Maintenance Cron Freshness (s)",
            'time() - max by (cronjob) (kube_cronjob_status_last_successful_time{namespace="maintenance",cronjob="image-sweeper"})',
            {"h": 4, "w": 12, "x": 12, "y": 44},
            unit="s",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 3600},
                    {"color": "red", "value": 10800},
                ],
            },
        )
    )
    return {
        "uid": "atlas-storage",
        "title": "Atlas Storage",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-12h", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "storage"],
    }


def build_network_dashboard():
    panels = []
    panels.append(
        stat_panel(
            1,
            "Ingress Success Rate (5m)",
            TRAEFIK_SLI_5M,
            {"h": 4, "w": 6, "x": 0, "y": 0},
            unit="percentunit",
            decimals=2,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "red", "value": None},
                    {"color": "orange", "value": 0.995},
                    {"color": "yellow", "value": 0.999},
                    {"color": "green", "value": 0.9995},
                ],
            },
        )
    )
    panels.append(
        stat_panel(
            2,
            "Error Budget Burn (1h)",
            traefik_burn("1h"),
            {"h": 4, "w": 6, "x": 6, "y": 0},
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 2},
                    {"color": "red", "value": 4},
                ],
            },
            decimals=2,
        )
    )
    panels.append(
        stat_panel(
            3,
            "Error Budget Burn (6h)",
            traefik_burn("6h"),
            {"h": 4, "w": 6, "x": 12, "y": 0},
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 2},
                    {"color": "red", "value": 4},
                ],
            },
            decimals=2,
        )
    )
    panels.append(
        stat_panel(
            4,
            "Edge P99 Latency (ms)",
            TRAEFIK_P99_LATENCY_MS,
            {"h": 4, "w": 6, "x": 18, "y": 0},
            unit="ms",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 200},
                    {"color": "orange", "value": 350},
                    {"color": "red", "value": 500},
                ],
            },
            decimals=1,
        )
    )
    panels.append(
        stat_panel(
            5,
            "Ingress Traffic",
            NET_INGRESS_EXPR,
            {"h": 4, "w": 8, "x": 0, "y": 4},
            unit="Bps",
        )
    )
    panels.append(
        stat_panel(
            6,
            "Egress Traffic",
            NET_EGRESS_EXPR,
            {"h": 4, "w": 8, "x": 8, "y": 4},
            unit="Bps",
        )
    )
    panels.append(
        stat_panel(
            7,
            "Intra-Cluster Traffic",
            NET_INTERNAL_EXPR,
            {"h": 4, "w": 8, "x": 16, "y": 4},
            unit="Bps",
        )
    )
    panels.append(
        timeseries_panel(
            8,
            "Per-Node Throughput",
            f'avg by (node) (({NET_NODE_TX_PHYS} + {NET_NODE_RX_PHYS}) * on(instance) group_left(node) {NODE_INFO})',
            {"h": 8, "w": 24, "x": 0, "y": 8},
            unit="Bps",
            legend="{{node}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        table_panel(
            9,
            "Top Namespaces",
            'topk(10, sum(rate(container_network_transmit_bytes_total{namespace!=""}[5m]) '
            '+ rate(container_network_receive_bytes_total{namespace!=""}[5m])) by (namespace))',
            {"h": 9, "w": 12, "x": 0, "y": 16},
            unit="Bps",
            transformations=[{"id": "labelsToFields", "options": {}}],
        )
    )
    panels.append(
        table_panel(
            10,
            "Top Pods",
            'topk(10, sum(rate(container_network_transmit_bytes_total{pod!=""}[5m]) '
            '+ rate(container_network_receive_bytes_total{pod!=""}[5m])) by (namespace,pod))',
            {"h": 9, "w": 12, "x": 12, "y": 16},
            unit="Bps",
            transformations=[{"id": "labelsToFields", "options": {}}],
        )
    )
    panels.append(
        timeseries_panel(
            11,
            "Traefik Routers (req/s)",
            f"topk(10, {TRAEFIK_ROUTER_EXPR})",
            {"h": 9, "w": 12, "x": 0, "y": 25},
            unit="req/s",
            legend="{{router}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            12,
            "Traefik Entrypoints (req/s)",
            'sum by (entrypoint) (rate(traefik_entrypoint_requests_total[5m]))',
            {"h": 9, "w": 12, "x": 12, "y": 25},
            unit="req/s",
            legend="{{entrypoint}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    return {
        "uid": "atlas-network",
        "title": "Atlas Network",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-12h", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "network"],
    }


def build_mail_dashboard():
    panels = []

    bounce_rate_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 5},
            {"color": "orange", "value": 8},
            {"color": "red", "value": 10},
        ],
    }
    limit_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 70},
            {"color": "orange", "value": 85},
            {"color": "red", "value": 95},
        ],
    }
    success_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "red", "value": None},
            {"color": "orange", "value": 90},
            {"color": "yellow", "value": 95},
            {"color": "green", "value": 98},
        ],
    }

    panels.append(
        stat_panel(
            1,
            "Sent (1d)",
            'max(postmark_outbound_sent{window="1d"})',
            {"h": 4, "w": 6, "x": 0, "y": 0},
            decimals=0,
        )
    )
    panels.append(
        stat_panel(
            2,
            "Sent (7d)",
            'max(postmark_outbound_sent{window="7d"})',
            {"h": 4, "w": 6, "x": 6, "y": 0},
            decimals=0,
        )
    )
    panels.append(
        {
            "id": 3,
            "type": "stat",
            "title": "Mail Bounces (1d)",
            "datasource": PROM_DS,
            "gridPos": {"h": 4, "w": 6, "x": 12, "y": 0},
            "targets": [
                {
                    "expr": 'max(postmark_outbound_bounce_rate{window="1d"})',
                    "refId": "A",
                    "legendFormat": "Rate",
                },
                {
                    "expr": 'max(postmark_outbound_bounced{window="1d"})',
                    "refId": "B",
                    "legendFormat": "Count",
                },
            ],
            "fieldConfig": {
                "defaults": {
                    "color": {"mode": "thresholds"},
                    "custom": {"displayMode": "auto"},
                    "thresholds": bounce_rate_thresholds,
                    "unit": "none",
                },
                "overrides": [
                    {
                        "matcher": {"id": "byName", "options": "Rate"},
                        "properties": [{"id": "unit", "value": "percent"}],
                    },
                    {
                        "matcher": {"id": "byName", "options": "Count"},
                        "properties": [{"id": "unit", "value": "none"}],
                    },
                ],
            },
            "options": {
                "colorMode": "value",
                "graphMode": "area",
                "justifyMode": "center",
                "reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
                "textMode": "name_and_value",
            },
        }
    )
    panels.append(
        stat_panel(
            4,
            "Success Rate (1d)",
            'clamp_min(100 - max(postmark_outbound_bounce_rate{window="1d"}), 0)',
            {"h": 4, "w": 6, "x": 18, "y": 0},
            unit="percent",
            thresholds=success_thresholds,
            decimals=1,
        )
    )

    panels.append(
        stat_panel(
            5,
            "Limit Used (30d)",
            "max(postmark_sending_limit_used_percent)",
            {"h": 4, "w": 6, "x": 0, "y": 4},
            thresholds=limit_thresholds,
            unit="percent",
            decimals=1,
        )
    )
    panels.append(
        stat_panel(
            6,
            "Send Limit (30d)",
            "max(postmark_sending_limit)",
            {"h": 4, "w": 6, "x": 6, "y": 4},
            decimals=0,
        )
    )
    panels.append(
        stat_panel(
            7,
            "Last Success",
            "max(postmark_last_success_timestamp_seconds)",
            {"h": 4, "w": 6, "x": 12, "y": 4},
            unit="dateTimeAsIso",
            decimals=0,
        )
    )
    panels.append(
        stat_panel(
            8,
            "Exporter Errors",
            "sum(postmark_request_errors_total)",
            {"h": 4, "w": 6, "x": 18, "y": 4},
            decimals=0,
        )
    )

    panels.append(
        timeseries_panel(
            13,
            "Bounce Rate (1d vs 7d)",
            "max by (window) (postmark_outbound_bounce_rate)",
            {"h": 8, "w": 12, "x": 0, "y": 12},
            unit="percent",
            legend="{{window}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            14,
            "Bounced (1d vs 7d)",
            "max by (window) (postmark_outbound_bounced)",
            {"h": 8, "w": 12, "x": 12, "y": 12},
            unit="none",
            legend="{{window}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            15,
            "Sent (1d vs 7d)",
            "max by (window) (postmark_outbound_sent)",
            {"h": 8, "w": 12, "x": 0, "y": 20},
            unit="none",
            legend="{{window}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            16,
            "Exporter Errors",
            "sum(postmark_request_errors_total)",
            {"h": 8, "w": 12, "x": 12, "y": 20},
            unit="none",
        )
    )

    return {
        "uid": "atlas-mail",
        "title": "Atlas Mail",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-30d", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "mail"],
    }


def build_jobs_dashboard():
    panels = []
    age_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 6},
            {"color": "orange", "value": 24},
            {"color": "red", "value": 48},
        ],
    }
    old_age_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 24},
            {"color": "orange", "value": 72},
            {"color": "red", "value": 168},
        ],
    }
    recent_error_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "red", "value": None},
            {"color": "orange", "value": 1},
            {"color": "yellow", "value": 6},
            {"color": "green", "value": 24},
        ],
    }
    task_error_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 1},
            {"color": "orange", "value": 3},
            {"color": "red", "value": 5},
        ],
    }
    count_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 1},
            {"color": "orange", "value": 2},
            {"color": "red", "value": 3},
        ],
    }
    schedule_status_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "red", "value": None},
            {"color": "yellow", "value": 0.5},
            {"color": "green", "value": 1},
        ],
    }

    panels.append(
        stat_panel(
            1,
            "Schedule Metrics Exported",
            ARIADNE_SCHEDULE_SIGNAL_COUNT,
            {"h": 4, "w": 4, "x": 0, "y": 0},
            unit="none",
            instant=True,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "red", "value": None},
                    {"color": "green", "value": 1},
                ],
            },
        )
    )
    panels.append(
        stat_panel(
            2,
            "Schedule Tasks Stale (>36h)",
            ARIADNE_SCHEDULE_STALE_COUNT,
            {"h": 4, "w": 4, "x": 4, "y": 0},
            unit="none",
            thresholds=count_thresholds,
        )
    )
    panels.append(
        stat_panel(
            3,
            "Schedule Tasks Missing Success",
            ARIADNE_SCHEDULE_MISSING_SUCCESS_COUNT,
            {"h": 4, "w": 4, "x": 8, "y": 0},
            unit="none",
            thresholds=count_thresholds,
        )
    )
    panels.append(
        stat_panel(
            4,
            "Schedule Tasks Failed Last Run",
            ARIADNE_SCHEDULE_FAILED_LAST_COUNT,
            {"h": 4, "w": 4, "x": 12, "y": 0},
            unit="none",
            thresholds=count_thresholds,
        )
    )
    panels.append(
        stat_panel(
            5,
            "Glue Jobs Stale (>36h)",
            GLUE_STALE_COUNT,
            {"h": 4, "w": 4, "x": 16, "y": 0},
            unit="none",
            thresholds=count_thresholds,
        )
    )
    panels.append(
        stat_panel(
            6,
            "Jenkins Workspace PV Backlog",
            JENKINS_WORKSPACE_PV_STALE_COUNT,
            {"h": 4, "w": 4, "x": 20, "y": 0},
            unit="none",
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 1},
                    {"color": "orange", "value": 10},
                    {"color": "red", "value": 25},
                ],
            },
        )
    )
    schedule_list_panel = table_panel(
        7,
        "Ariadne Schedules: Last Success (h, newest first)",
        ARIADNE_SCHEDULE_LAST_SUCCESS_RANGE_HOURS_FALLBACK,
        {"h": 8, "w": 12, "x": 0, "y": 4},
        unit="h",
        transformations=[
            {"id": "labelsToFields", "options": {}},
            {"id": "sortBy", "options": {"fields": ["Value"], "order": "asc"}},
        ],
        instant=True,
    )
    schedule_list_panel["description"] = "Primary schedule inventory ordered by recency so fresh jobs stay at the top."
    panels.append(schedule_list_panel)
    panels.append(
        bargauge_panel(
            8,
            "Ariadne Schedule Last Error (hours ago)",
            ARIADNE_SCHEDULE_LAST_ERROR_RANGE_HOURS_FALLBACK,
            {"h": 8, "w": 12, "x": 12, "y": 4},
            unit="h",
            instant=True,
            legend="{{task}}",
            sort_order="asc",
            thresholds=recent_error_thresholds,
            decimals=2,
        )
    )
    status_panel = bargauge_panel(
        9,
        "Ariadne Schedule Last Status",
        ARIADNE_SCHEDULE_LAST_STATUS_FALLBACK,
        {"h": 8, "w": 8, "x": 0, "y": 12},
        unit="none",
        instant=True,
        legend="{{task}}",
        sort_order="asc",
        thresholds=schedule_status_thresholds,
        decimals=0,
    )
    status_panel["description"] = "1 means the last run was ok. 0 means the last run ended in error."
    status_panel["fieldConfig"]["defaults"]["mappings"] = [
        {
            "type": "value",
            "options": {
                "-1": {"text": "pending"},
                "0": {"text": "error"},
                "1": {"text": "ok"},
            },
        }
    ]
    panels.append(status_panel)
    schedule_runs_panel = bargauge_panel(
        10,
        "Ariadne Schedule Runs (range)",
        ARIADNE_SCHEDULE_RUNS_RANGE_FALLBACK,
        {"h": 8, "w": 8, "x": 8, "y": 12},
        unit="none",
        instant=True,
        legend="{{task}}",
        thresholds={"mode": "absolute", "steps": [{"color": "green", "value": None}]},
    )
    schedule_runs_panel["description"] = "Number of runs by schedule task over the selected dashboard time range."
    panels.append(schedule_runs_panel)
    schedule_errors_panel = bargauge_panel(
        11,
        "Ariadne Schedule Errors (range)",
        ARIADNE_SCHEDULE_ERRORS_RANGE_FALLBACK,
        {"h": 8, "w": 8, "x": 16, "y": 12},
        unit="none",
        instant=True,
        legend="{{task}}",
        thresholds=task_error_thresholds,
    )
    schedule_errors_panel["description"] = "Error run count by schedule task over the selected dashboard time range."
    panels.append(schedule_errors_panel)
    panels.append(
        {
            "id": 12,
            "type": "timeseries",
            "title": "Ariadne Attempts / Failures",
            "datasource": PROM_DS,
            "gridPos": {"h": 7, "w": 12, "x": 0, "y": 20},
            "targets": [
                {"expr": ARIADNE_TASK_ATTEMPTS_SERIES, "refId": "A", "legendFormat": "Attempts"},
                {"expr": ARIADNE_TASK_FAILURES_SERIES, "refId": "B", "legendFormat": "Failures"},
            ],
            "fieldConfig": {
                "defaults": {"unit": "none"},
                "overrides": [
                    {
                        "matcher": {"id": "byName", "options": "Attempts"},
                        "properties": [
                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "green"}}
                        ],
                    },
                    {
                        "matcher": {"id": "byName", "options": "Failures"},
                        "properties": [
                            {"id": "color", "value": {"mode": "fixed", "fixedColor": "red"}}
                        ],
                    },
                ],
            },
            "options": {
                "legend": {"displayMode": "table", "placement": "right"},
                "tooltip": {"mode": "multi"},
            },
        }
    )
    panels.append(
        bargauge_panel(
            13,
            "One-off Job Pods (age hours)",
            ONEOFF_JOB_POD_AGE_HOURS,
            {"h": 7, "w": 12, "x": 12, "y": 20},
            unit="h",
            instant=True,
            legend="{{namespace}}/{{pod}}",
            thresholds=age_thresholds,
            limit=12,
            decimals=2,
        )
    )
    panels.append(
        bargauge_panel(
            14,
            "Glue Jobs Last Success (hours ago)",
            GLUE_LAST_SUCCESS_RANGE_HOURS,
            {"h": 6, "w": 12, "x": 0, "y": 27},
            unit="h",
            instant=True,
            legend="{{namespace}}/{{cronjob}}",
            thresholds=age_thresholds,
            decimals=2,
        )
    )
    panels.append(
        bargauge_panel(
            15,
            "Glue Jobs Last Schedule (hours ago)",
            GLUE_LAST_SCHEDULE_RANGE_HOURS,
            {"h": 6, "w": 12, "x": 12, "y": 27},
            unit="h",
            instant=True,
            legend="{{namespace}}/{{cronjob}}",
            thresholds=age_thresholds,
            decimals=2,
        )
    )
    panels.append(
        stat_panel(
            16,
            "Jenkins Cleanup Signal Present",
            JENKINS_CLEANUP_SIGNAL_COUNT,
            {"h": 4, "w": 4, "x": 0, "y": 33},
            unit="none",
            text_mode="name_and_value",
            legend="Signal",
            instant=True,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "red", "value": None},
                    {"color": "green", "value": 1},
                ],
            },
            orientation="vertical",
            wide_layout=True,
            graph_mode="none",
            justify_mode="auto",
        )
    )
    panels.append(
        stat_panel(
            17,
            "Jenkins Cleanup Last Run Age (h)",
            JENKINS_CLEANUP_LAST_RUN_AGE_HOURS,
            {"h": 4, "w": 4, "x": 4, "y": 33},
            unit="h",
            decimals=2,
            text_mode="name_and_value",
            legend="Last Run",
            instant=True,
            thresholds=age_thresholds,
            orientation="vertical",
            wide_layout=True,
            graph_mode="none",
            justify_mode="auto",
        )
    )
    panels.append(
        stat_panel(
            18,
            "Jenkins Cleanup Last Success Age (h)",
            JENKINS_CLEANUP_LAST_SUCCESS_AGE_HOURS,
            {"h": 4, "w": 4, "x": 8, "y": 33},
            unit="h",
            decimals=2,
            text_mode="name_and_value",
            legend="Last Success",
            instant=True,
            thresholds=age_thresholds,
            orientation="vertical",
            wide_layout=True,
            graph_mode="none",
            justify_mode="auto",
        )
    )
    panels.append(
        stat_panel(
            19,
            "Jenkins Cleanup Planned (last run)",
            JENKINS_CLEANUP_LAST_PLANNED,
            {"h": 4, "w": 4, "x": 12, "y": 33},
            unit="none",
            text_mode="name_and_value",
            legend="Planned",
            instant=True,
            orientation="vertical",
            wide_layout=True,
            graph_mode="none",
            justify_mode="auto",
        )
    )
    panels.append(
        stat_panel(
            20,
            "Jenkins Cleanup Deleted (last run)",
            JENKINS_CLEANUP_LAST_DELETED,
            {"h": 4, "w": 4, "x": 16, "y": 33},
            unit="none",
            text_mode="name_and_value",
            legend="Deleted",
            instant=True,
            orientation="vertical",
            wide_layout=True,
            graph_mode="none",
            justify_mode="auto",
        )
    )
    panels.append(
        stat_panel(
            21,
            "Ariadne Access Requests",
            ARIADNE_ACCESS_REQUESTS,
            {"h": 4, "w": 4, "x": 20, "y": 33},
            unit="none",
            instant=True,
        )
    )
    panels.append(
        timeseries_panel(
            22,
            "Jenkins Cleanup Runs (range)",
            None,
            {"h": 7, "w": 12, "x": 0, "y": 37},
            unit="none",
            targets=[
                {"refId": "A", "expr": JENKINS_CLEANUP_RUNS_RANGE, "legendFormat": "{{mode}}/{{status}}"},
            ],
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            23,
            "Jenkins Cleanup Objects (range)",
            None,
            {"h": 7, "w": 12, "x": 12, "y": 37},
            unit="none",
            targets=[
                {"refId": "A", "expr": JENKINS_CLEANUP_OBJECTS_RANGE, "legendFormat": "{{kind}}/{{action}}/{{mode}}"},
            ],
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        jenkins_weather_bargauge_panel(
            24,
            "Jenkins Build Weather (last run h, newest first)",
            JENKINS_BUILD_WEATHER_LAST_RUN_AGE_HOURS,
            {"h": 8, "w": 8, "x": 0, "y": 44},
            unit="h",
            decimals=2,
            sort_order="asc",
            limit=20,
            thresholds=age_thresholds,
            description=(
                "Jenkins homepage-style weather list: bars are color-coded by last build "
                "status, icons mirror job weather, and values show age since the last run."
            ),
        )
    )
    panels.append(
        jenkins_weather_bargauge_panel(
            25,
            "Jenkins Last Success (hours ago)",
            JENKINS_BUILD_WEATHER_LAST_SUCCESS_AGE_HOURS,
            {"h": 8, "w": 8, "x": 8, "y": 44},
            unit="h",
            decimals=2,
            sort_order="asc",
            limit=20,
            thresholds=old_age_thresholds,
            description="Per-job age since the most recent successful run.",
        )
    )
    panels.append(
        jenkins_weather_bargauge_panel(
            26,
            "Jenkins Last Duration (minutes)",
            JENKINS_BUILD_WEATHER_LAST_DURATION_MINUTES,
            {"h": 8, "w": 8, "x": 16, "y": 44},
            unit="m",
            decimals=2,
            sort_order="desc",
            limit=20,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 5},
                    {"color": "orange", "value": 15},
                    {"color": "red", "value": 30},
                ],
            },
            description="Most recent completed build duration per Jenkins job.",
        )
    )
    stale_volume_panel = bargauge_panel(
        27,
        "Jenkins Workspace PV Age (h, detached only)",
        JENKINS_WORKSPACE_PV_STALE_AGE_HOURS,
        {"h": 10, "w": 24, "x": 0, "y": 52},
        unit="h",
        instant=True,
        legend="{{name}} -> {{persistentvolume}}",
        thresholds=old_age_thresholds,
        decimals=1,
        limit=40,
    )
    stale_volume_panel["description"] = (
        "Oldest detached Jenkins workspace volumes first. This is the direct cleanup backlog view."
    )
    panels.append(stale_volume_panel)

    return {
        "uid": "atlas-jobs",
        "title": "Atlas Jobs",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-7d", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "jobs", "glue"],
    }


def build_testing_dashboard():
    panels = []
    pass_rate_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "red", "value": None},
            {"color": "orange", "value": 80},
            {"color": "yellow", "value": 95},
            {"color": "green", "value": 99},
        ],
    }
    failures_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "green", "value": None},
            {"color": "yellow", "value": 1},
            {"color": "orange", "value": 3},
            {"color": "red", "value": 5},
        ],
    }
    smell_thresholds = {
        "mode": "absolute",
        "steps": [
            {"color": "red", "value": None},
            {"color": "green", "value": 0},
            {"color": "yellow", "value": 1},
            {"color": "orange", "value": 3},
            {"color": "red", "value": 5},
        ],
    }

    pass_rate_panel = stat_panel(
        1,
        "Platform Test Success Rate (30d)",
        TEST_SUCCESS_RATE,
        {"h": 4, "w": 6, "x": 0, "y": 0},
        unit="percent",
        decimals=2,
        instant=True,
        thresholds=pass_rate_thresholds,
    )
    pass_rate_panel["description"] = "Overall success rate across tracked suites over the last 30 days."
    panels.append(pass_rate_panel)
    failures_panel = stat_panel(
        2,
        "Platform Test Failures (24h)",
        TEST_FAILURES_24H_TOTAL,
        {"h": 4, "w": 6, "x": 6, "y": 0},
        unit="none",
        instant=True,
        thresholds=failures_thresholds,
    )
    failures_panel["description"] = "Total failed runs in the last 24 hours."
    panels.append(failures_panel)
    activity_panel = table_panel(
        3,
        "Platform Test Activity (30d)",
        PLATFORM_TEST_ACTIVITY_30D,
        {"h": 8, "w": 12, "x": 12, "y": 0},
        unit="none",
        transformations=[
            {"id": "labelsToFields", "options": {}},
            {"id": "sortBy", "options": {"fields": ["Value"], "order": "desc"}},
        ],
        instant=True,
    )
    activity_panel["description"] = "Suite/status event counts over 30 days."
    panels.append(activity_panel)
    panels.append(
        bargauge_panel(
            4,
            "Platform Test Failures by Suite (24h)",
            PLATFORM_TEST_FAILURES_24H_BY_SUITE,
            {"h": 8, "w": 12, "x": 0, "y": 8},
            unit="none",
            instant=True,
            legend="{{suite}}",
            thresholds=failures_thresholds,
        )
    )
    panels.append(
        bargauge_panel(
            5,
            "Platform Test Success Rate by Suite (24h, lowest first)",
            PLATFORM_TEST_SUCCESS_RATE_24H_BY_SUITE,
            {"h": 8, "w": 12, "x": 12, "y": 8},
            unit="percent",
            instant=True,
            legend="{{suite}}",
            sort_order="asc",
            thresholds=pass_rate_thresholds,
            decimals=2,
        )
    )
    suite_panel = timeseries_panel(
        6,
        "Platform Test Success Rate by Suite",
        None,
        {"h": 8, "w": 24, "x": 0, "y": 16},
        unit="percent",
        targets=PLATFORM_TEST_SUCCESS_RATE_SUITE_TARGETS,
        legend_display="list",
        legend_placement="bottom",
    )
    suite_panel["fieldConfig"]["defaults"]["min"] = 0
    suite_panel["fieldConfig"]["defaults"]["max"] = 100
    suite_panel["fieldConfig"]["defaults"]["custom"] = {
        "drawStyle": "line",
        "lineInterpolation": "linear",
        "lineWidth": 2,
        "fillOpacity": 10,
        "showPoints": "always",
        "pointSize": 4,
        "spanNulls": True,
    }
    suite_panel["description"] = "Trend line per suite. Flat gaps mean no runs in that interval."
    panels.append(suite_panel)
    coverage_panel = bargauge_panel(
        7,
        "Quality Gate Coverage by Suite (%, gate 95)",
        QUALITY_GATE_COVERAGE_BY_SUITE_WITH_MISSING,
        {"h": 8, "w": 12, "x": 0, "y": 24},
        unit="percent",
        instant=True,
        legend="{{suite}}",
        sort_order="asc",
        thresholds=pass_rate_thresholds,
        decimals=2,
    )
    coverage_panel["description"] = (
        "Latest reported per-suite line coverage. The quality gate target is 95%. "
        "A value of -1 means that suite has runs but no coverage metric published yet."
    )
    coverage_panel["fieldConfig"]["defaults"]["mappings"] = [
        {
            "type": "value",
            "options": {
                "-1": {"text": "missing"},
            },
        }
    ]
    panels.append(coverage_panel)
    coverage_gap_panel = bargauge_panel(
        8,
        "Coverage Gap to 95% by Suite",
        QUALITY_GATE_COVERAGE_GAP_BY_SUITE,
        {"h": 8, "w": 12, "x": 12, "y": 24},
        unit="percent",
        instant=True,
        legend="{{suite}}",
        sort_order="desc",
        thresholds={
            "mode": "absolute",
            "steps": [
                {"color": "green", "value": None},
                {"color": "yellow", "value": 1},
                {"color": "orange", "value": 5},
                {"color": "red", "value": 10},
            ],
        },
        decimals=2,
    )
    coverage_gap_panel["description"] = "How far each suite is below the 95% target (0 means at or above target)."
    panels.append(coverage_gap_panel)
    smell_panel = bargauge_panel(
        9,
        "Code Smell Infractions by Suite (files >500 LOC)",
        QUALITY_GATE_SMELL_INFRACTIONS_BY_SUITE_WITH_MISSING,
        {"h": 8, "w": 24, "x": 0, "y": 32},
        unit="none",
        instant=True,
        legend="{{suite}}",
        sort_order="desc",
        thresholds=smell_thresholds,
    )
    smell_panel["description"] = (
        "Per-suite count of files violating the 500-line hygiene/code-smell threshold. "
        "A value of -1 means that suite has runs but no smell-infraction metric published yet."
    )
    smell_panel["fieldConfig"]["defaults"]["mappings"] = [
        {
            "type": "value",
            "options": {
                "-1": {"text": "missing"},
            },
        }
    ]
    panels.append(smell_panel)

    return {
        "uid": "atlas-testing",
        "title": "Atlas Testing",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-30d", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "testing", "quality"],
    }


def build_power_dashboard():
    panels = []

    panels.append(
        stat_panel(
            1,
            "UPS Current Load",
            UPS_CURRENT_ROW_EXPR,
            {"h": 8, "w": 12, "x": 0, "y": 0},
            unit="none",
            decimals=1,
            text_mode="name_and_value",
            legend="{{ups}} {{metric}}",
            instant=True,
            field_overrides=[
                {"matcher": {"id": "byRegexp", "options": ".*Draw$"}, "properties": [{"id": "unit", "value": "watt"}]},
                {"matcher": {"id": "byRegexp", "options": ".*Runtime$"}, "properties": [{"id": "unit", "value": "s"}]},
            ],
            orientation="vertical",
            wide_layout=False,
            title_size=14,
            value_size=24,
            description=(
                "Per-UPS live snapshot: draw, discharge runtime, and charging/discharging status."
            ),
        )
    )
    panels.append(
        timeseries_panel(
            2,
            "UPS History (Power Draw)",
            None,
            {"h": 8, "w": 12, "x": 12, "y": 0},
            unit="watt",
            targets=[
                {"refId": "A", "expr": ANANKE_UPS_DRAW_WATTS_DB_SERIES, "legendFormat": ANANKE_UPS_DB_NAME},
                {"refId": "B", "expr": ANANKE_UPS_DRAW_WATTS_TETHYS_SERIES, "legendFormat": ANANKE_UPS_TETHYS_NAME},
                {"refId": "C", "expr": ANANKE_UPS_DRAW_WATTS_TOTAL_SERIES, "legendFormat": "combined"},
            ],
            legend_display="table",
            legend_placement="right",
            description="Historical UPS power consumption in watts for titan-db, tethys, and combined load.",
        )
    )
    panels.append(
        stat_panel(
            3,
            "Current Climate",
            CLIMATE_CURRENT_ROW_EXPR,
            {"h": 8, "w": 12, "x": 0, "y": 8},
            unit="none",
            decimals=2,
            text_mode="name_and_value",
            legend="{{metric}}",
            instant=True,
            field_overrides=[
                {"matcher": {"id": "byName", "options": "Temp °C"}, "properties": [{"id": "unit", "value": "celsius"}]},
                {"matcher": {"id": "byName", "options": "Temp °F"}, "properties": [{"id": "unit", "value": "fahrenheit"}]},
                {"matcher": {"id": "byName", "options": "Humidity"}, "properties": [{"id": "unit", "value": "percent"}]},
                {"matcher": {"id": "byName", "options": "Pressure"}, "properties": [{"id": "unit", "value": "suffix:kPa"}]},
            ],
            orientation="vertical",
            wide_layout=False,
            title_size=16,
            value_size=28,
            description="Current tent values: Temp °C, Temp °F, Humidity, Pressure.",
        )
    )
    panels.append(
        timeseries_panel(
            4,
            "Climate History",
            None,
            {"h": 8, "w": 12, "x": 12, "y": 8},
            unit="none",
            targets=[
                {"refId": "A", "expr": CLIMATE_TEMP_SERIES, "legendFormat": "Temperature (°C)"},
                {"refId": "B", "expr": CLIMATE_TEMP_FAHRENHEIT_SERIES, "legendFormat": "Temperature (°F)"},
                {"refId": "C", "expr": CLIMATE_HUMIDITY_SERIES, "legendFormat": "Humidity (%)"},
                {"refId": "D", "expr": CLIMATE_PRESSURE_SERIES, "legendFormat": "Pressure (VPD kPa)"},
            ],
            field_overrides=[
                {
                    "matcher": {"id": "byName", "options": "Temperature (°C)"},
                    "properties": [
                        {"id": "unit", "value": "suffix:°C"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                },
                {
                    "matcher": {"id": "byName", "options": "Temperature (°F)"},
                    "properties": [
                        {"id": "unit", "value": "suffix:°F"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                },
                {
                    "matcher": {"id": "byName", "options": "Humidity (%)"},
                    "properties": [
                        {"id": "unit", "value": "suffix:%"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisPlacement", "value": "right"},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                },
                {
                    "matcher": {"id": "byName", "options": "Pressure (VPD kPa)"},
                    "properties": [
                        {"id": "unit", "value": "suffix:kPa"},
                        {"id": "custom.axisPlacement", "value": "right"},
                        {"id": "decimals", "value": 2},
                        {"id": "custom.axisCenteredZero", "value": False},
                    ],
                }
            ],
            legend_display="table",
            legend_placement="right",
            description="Historical tent temperature (C/F), humidity, and pressure proxy (VPD kPa).",
        )
    )
    panels.append(
        stat_panel(
            5,
            "Fan Activity",
            CLIMATE_FAN_CURRENT_ROW_EXPR,
            {"h": 8, "w": 12, "x": 0, "y": 16},
            unit="none",
            decimals=0,
            text_mode="name_and_value",
            legend="{{metric}}",
            instant=True,
            thresholds={
                "mode": "absolute",
                "steps": [
                    {"color": "green", "value": None},
                    {"color": "yellow", "value": 7},
                    {"color": "red", "value": 9},
                ],
            },
            field_overrides=[
                {"matcher": {"id": "byName", "options": "Outlet"}, "properties": [{"id": "decimals", "value": 0}]},
                {"matcher": {"id": "byName", "options": "Inlet In"}, "properties": [{"id": "decimals", "value": 0}]},
                {"matcher": {"id": "byName", "options": "Inlet Out"}, "properties": [{"id": "decimals", "value": 0}]},
                {"matcher": {"id": "byName", "options": "Interior"}, "properties": [{"id": "decimals", "value": 0}]},
            ],
            orientation="vertical",
            wide_layout=False,
            description="Current fan activity levels: outlet, inlet in, inlet out, interior (0-10).",
        )
    )
    panels.append(
        timeseries_panel(
            6,
            "Fan History (0-10)",
            None,
            {"h": 8, "w": 12, "x": 12, "y": 16},
            unit="none",
            max_value=10,
            targets=[
                {"refId": "A", "expr": CLIMATE_FAN_OUTLET_SERIES, "legendFormat": "Inside Outlet"},
                {"refId": "B", "expr": CLIMATE_FAN_INSIDE_INLET_SERIES, "legendFormat": "Inside Inlet"},
                {"refId": "C", "expr": CLIMATE_FAN_OUTSIDE_INLET_SERIES, "legendFormat": "Outside Inlet"},
                {"refId": "D", "expr": CLIMATE_FAN_INTERIOR_SERIES, "legendFormat": "Interior Fans"},
            ],
            legend_display="table",
            legend_placement="right",
            description="Historical fan activity for all four fan groups (0-10 scale).",
        )
    )

    return {
        "uid": "atlas-power",
        "title": "Atlas Power",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-24h", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "power", "climate"],
    }


def build_gpu_dashboard():
    panels = []
    gpu_scope = "$namespace_scope_gpu"
    panels.append(
        pie_panel(
            1,
            "Namespace GPU Share",
            namespace_gpu_share_expr(gpu_scope),
            {"h": 8, "w": 12, "x": 0, "y": 0},
            links=namespace_scope_links("namespace_scope_gpu"),
            description="Shares are normalized within the selected filter. Switching scope changes the denominator.",
        )
    )
    panels.append(
        timeseries_panel(
            2,
            "GPU Util by Namespace",
            namespace_gpu_usage_instant(gpu_scope),
            {"h": 8, "w": 12, "x": 12, "y": 0},
            unit="percent",
            legend="{{namespace}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        timeseries_panel(
            3,
            "GPU Util by Node",
            gpu_util_by_hostname(),
            {"h": 8, "w": 12, "x": 0, "y": 8},
            unit="percent",
            legend="{{Hostname}}",
            legend_display="table",
            legend_placement="right",
        )
    )
    panels.append(
        table_panel(
            4,
            "Top Pods by GPU Util",
            'topk(10, sum(DCGM_FI_DEV_GPU_UTIL{pod!=""}) by (namespace,pod,Hostname))',
            {"h": 8, "w": 12, "x": 12, "y": 8},
            unit="percent",
            transformations=[{"id": "labelsToFields", "options": {}}],
        )
    )
    return {
        "uid": "atlas-gpu",
        "title": "Atlas GPU",
        "folderUid": PRIVATE_FOLDER,
        "editable": True,
        "panels": panels,
        "time": {"from": "now-12h", "to": "now"},
        "annotations": {"list": []},
        "schemaVersion": 39,
        "style": "dark",
        "tags": ["atlas", "gpu"],
        "templating": {
            "list": [
                namespace_scope_variable("namespace_scope_cpu", "CPU namespace filter"),
                namespace_scope_variable("namespace_scope_gpu", "GPU namespace filter"),
                namespace_scope_variable("namespace_scope_ram", "RAM namespace filter"),
            ]
        },
    }


DASHBOARDS = {
    "atlas-overview": {
        "builder": build_overview,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-overview.yaml",
    },
    "atlas-pods": {
        "builder": build_pods_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-pods.yaml",
    },
    "atlas-nodes": {
        "builder": build_nodes_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-nodes.yaml",
    },
    "atlas-storage": {
        "builder": build_storage_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-storage.yaml",
    },
    "atlas-network": {
        "builder": build_network_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-network.yaml",
    },
    "atlas-mail": {
        "builder": build_mail_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-mail.yaml",
    },
    "atlas-jobs": {
        "builder": build_jobs_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-jobs.yaml",
    },
    "atlas-testing": {
        "builder": build_testing_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-testing.yaml",
    },
    "atlas-power": {
        "builder": build_power_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-power.yaml",
    },
    "atlas-gpu": {
        "builder": build_gpu_dashboard,
        "configmap": ROOT / "services" / "monitoring" / "grafana-dashboard-gpu.yaml",
    },
}


def write_json(uid, data):
    DASHBOARD_DIR.mkdir(parents=True, exist_ok=True)
    path = DASHBOARD_DIR / f"{uid}.json"
    path.write_text(json.dumps(data, indent=2) + "\n")


def render_configmap(uid, info):
    json_path = DASHBOARD_DIR / f"{uid}.json"
    payload = json.dumps(json.loads(json_path.read_text()), indent=2)
    indented = "\n".join("    " + line for line in payload.splitlines())
    output_path = info["configmap"]
    content = CONFIG_TEMPLATE.format(
        relative_path=output_path.relative_to(ROOT),
        name=output_path.stem,
        key=json_path.name,
        payload=indented,
    )
    output_path.write_text(content)
    print(f"Rendered {json_path.name} -> {output_path.relative_to(ROOT)}")


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--build", action="store_true", help="Regenerate dashboard JSON files from builders")
    args = parser.parse_args()

    if args.build:
        for uid, info in DASHBOARDS.items():
            write_json(uid, info["builder"]())

    for uid, info in DASHBOARDS.items():
        render_configmap(uid, info)


if __name__ == "__main__":
    main()