2862594c62
The 2026-08-18 metrics-storage outage exposed two defects in the availability pipeline that distorted the figure in opposite directions at once. The Overview panel fell back to a live one-hour Traefik ratio whenever the yearly rollup sample went stale for 48h, and rendered it under the same "365d" title. When the rollup stopped publishing on 2026-08-18 the panel quietly swapped a 365-day measurement for a 60-minute one and read 99.74% instead of the recorded 99.95%. The fallback is removed: a stale rollup now renders no value, and a new atlas-availability-rollup-stale alert pages at 26h, well before the panel goes blank at 48h. The yearly ratio also silently excluded the 34-hour telemetry gap, because missing days contribute zero requests and zero failures. Absent data was read as "nothing happened" — had Atlas genuinely been down in that window, the figure would still have said 99.95%. Availability keeps its measured-days-only definition, which is correct, but coverage is now published alongside it and shown in a new panel, so a telemetry gap lowers disclosed coverage instead of vanishing. The title reads "365d window" to stop implying 365 days of data exist; request-v4 begins 2026-05-01. The rollup job reported healthy runs across a day and a half of lost publishes: a read-only VictoriaMetrics accepts an import and discards it. It now reads each sample back and fails loudly when the write did not survive. Not addressed here: availability is still measured from inside the platform via Traefik counters, so it cannot distinguish "Atlas down" from "telemetry down", and misses failures that never reach Traefik (DNS, TLS, node dead). An external synthetic prober is the real fix and needs a hosting decision.
1092 lines
38 KiB
YAML
1092 lines
38 KiB
YAML
# services/monitoring/grafana-alerting-config.yaml
|
|
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: grafana-alerting
|
|
namespace: monitoring
|
|
labels:
|
|
grafana_alerting: "1"
|
|
data:
|
|
alerting.yaml: |
|
|
apiVersion: 1
|
|
contactPoints:
|
|
- orgId: 1
|
|
name: email-admins
|
|
receivers:
|
|
- uid: email-admins
|
|
type: email
|
|
settings:
|
|
addresses: ${GRAFANA_ALERT_EMAILS}
|
|
singleEmail: true
|
|
policies:
|
|
- orgId: 1
|
|
receiver: email-admins
|
|
group_by:
|
|
- grafana_folder
|
|
- alertname
|
|
group_wait: 1m
|
|
group_interval: 30m
|
|
repeat_interval: 12h
|
|
routes:
|
|
- receiver: email-admins
|
|
object_matchers:
|
|
- [severity, "=", "critical"]
|
|
group_wait: 30s
|
|
group_interval: 15m
|
|
repeat_interval: 4h
|
|
- receiver: email-admins
|
|
object_matchers:
|
|
- [severity, "=", "warning"]
|
|
group_wait: 10m
|
|
group_interval: 4h
|
|
repeat_interval: 48h
|
|
rules.yaml: |
|
|
apiVersion: 1
|
|
groups:
|
|
- orgId: 1
|
|
name: atlas-disk
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: disk-pressure-root
|
|
title: "Node rootfs high (>85%)"
|
|
condition: C
|
|
for: "10m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: avg by (node) ((avg by (instance) ((1 - (node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})) * 100)) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)"))
|
|
legendFormat: '{{node}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [85]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "{{ $labels.node }} rootfs >85% for 10m"
|
|
labels:
|
|
severity: warning
|
|
- uid: disk-growth-1h
|
|
title: "Node rootfs growing fast (>3Gi in 1h)"
|
|
condition: C
|
|
for: "30m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 3600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: max by (instance, node) ((clamp_min(delta((node_filesystem_size_bytes{mountpoint="/",fstype!~"tmpfs|overlay"} - node_filesystem_avail_bytes{mountpoint="/",fstype!~"tmpfs|overlay"})[1h:1m]), 0) / 1024 / 1024 / 1024) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)"))
|
|
legendFormat: '{{node}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [3]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "{{ $labels.node }} rootfs grew >3Gi in the last hour"
|
|
labels:
|
|
severity: warning
|
|
- orgId: 1
|
|
name: atlas-cpu
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: cpu-high-10m
|
|
title: "Node CPU high (>95% for 20m)"
|
|
condition: C
|
|
for: 20m
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: clamp_max(clamp_min(((1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100), 0), 100) * on(instance) group_left(node) label_replace(node_uname_info{nodename!=""}, "node", "$1", "nodename", "(.*)")
|
|
legendFormat: '{{node}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [95]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: NoData
|
|
execErrState: OK
|
|
annotations:
|
|
summary: "{{ $labels.node }} CPU >95% for 20m"
|
|
labels:
|
|
severity: warning
|
|
- orgId: 1
|
|
name: atlas-metrics
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: victoria-metrics-down
|
|
title: "VictoriaMetrics unavailable (>30m)"
|
|
condition: C
|
|
for: "30m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: sum(up{job="victoriametrics"})
|
|
legendFormat: victoriametrics
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [1]
|
|
type: lt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: Alerting
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "VictoriaMetrics is unavailable for >30m"
|
|
labels:
|
|
severity: critical
|
|
# The database can be up, scraped, and answering queries while
|
|
# refusing every new sample. These three rules watch the write path
|
|
# rather than liveness, and they alert on absent data because a
|
|
# stalled database stops producing the very series they read.
|
|
- uid: victoria-metrics-storage-low
|
|
title: "VictoriaMetrics storage headroom low (<15%)"
|
|
condition: C
|
|
for: "30m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: 100 * sum(vm_free_disk_space_bytes{job="victoriametrics"}) / clamp_min(sum(vm_free_disk_space_bytes{job="victoriametrics"}) + sum(vm_data_size_bytes{job="victoriametrics"}), 1)
|
|
legendFormat: free %
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [15]
|
|
type: lt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: Alerting
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "VictoriaMetrics has <15% free space on /storage; expand the volume before it stops accepting samples"
|
|
labels:
|
|
severity: warning
|
|
- uid: victoria-metrics-storage-readonly
|
|
title: "VictoriaMetrics storage is read-only"
|
|
condition: C
|
|
for: "5m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: max(vm_storage_is_read_only{job="victoriametrics"}) or on() vector(0)
|
|
legendFormat: read-only
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [0]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: Alerting
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "VictoriaMetrics flipped /storage to read-only and is discarding every new sample"
|
|
labels:
|
|
severity: critical
|
|
- uid: victoria-metrics-ingestion-stalled
|
|
title: "VictoriaMetrics is not storing samples"
|
|
condition: C
|
|
for: "15m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 900
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: sum(rate(vm_rows_added_to_storage_total{job="victoriametrics"}[10m])) or on() vector(0)
|
|
legendFormat: rows/s
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [1]
|
|
type: lt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: Alerting
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "VictoriaMetrics stored no samples for 15m; every dashboard is about to read empty"
|
|
labels:
|
|
severity: critical
|
|
# The rollup CronJob publishes one yearly availability sample per
|
|
# day, and the Overview panel refuses to substitute another
|
|
# measurement when that sample goes stale. Staleness therefore has
|
|
# to page before the panel goes blank at the 48h lookback.
|
|
- uid: atlas-availability-rollup-stale
|
|
title: "Atlas availability rollup is stale (>26h)"
|
|
condition: C
|
|
for: "30m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: (time() - tlast_over_time(atlas:availability:ratio_365d{scope="atlas",definition="request-v4"}[3d])) or on() vector(999999)
|
|
legendFormat: seconds since publish
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [93600]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: Alerting
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "Atlas availability rollup has not published in >26h; the Overview availability panel goes blank at 48h"
|
|
labels:
|
|
severity: warning
|
|
- orgId: 1
|
|
name: maintenance
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: maint-sweeper
|
|
title: "Maintenance sweeper not ready"
|
|
condition: C
|
|
for: "5m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: kube_daemonset_status_number_ready{namespace="maintenance",daemonset="node-image-sweeper"} / on(namespace,daemonset) kube_daemonset_status_desired_number_scheduled{namespace="maintenance",daemonset="node-image-sweeper"}
|
|
legendFormat: '{{daemonset}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [1]
|
|
type: lt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "node-image-sweeper not fully ready"
|
|
labels:
|
|
severity: warning
|
|
- uid: logging-node-log-rotation-not-ready
|
|
title: "Node log rotation guardrails not ready"
|
|
condition: C
|
|
for: "10m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: kube_daemonset_status_number_ready{namespace="logging",daemonset="node-log-rotation"} / on(namespace,daemonset) kube_daemonset_status_desired_number_scheduled{namespace="logging",daemonset="node-log-rotation"}
|
|
legendFormat: '{{daemonset}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [1]
|
|
type: lt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "node-log-rotation is not fully ready"
|
|
labels:
|
|
severity: warning
|
|
- uid: maint-ariadne-image-sweeper-stale
|
|
title: "Ariadne image sweeper stale (schedule >24h)"
|
|
condition: C
|
|
for: "5m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: time() - ariadne_schedule_last_success_timestamp_seconds{task="schedule.image_sweeper"}
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: '{{task}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [86400]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "Ariadne image sweeper stale >24h since last success"
|
|
labels:
|
|
severity: warning
|
|
- uid: maint-cron-stale
|
|
title: "Maintenance CronJobs stale (legacy disabled)"
|
|
condition: C
|
|
for: "5m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: vector(0)
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: legacy
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [1]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: OK
|
|
annotations:
|
|
summary: "Legacy cronjob alert disabled"
|
|
labels:
|
|
severity: info
|
|
- uid: maint-soteria-refresh-stale
|
|
title: "Soteria inventory refresh stale (>15m)"
|
|
condition: C
|
|
for: "15m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 900
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: time() - soteria_inventory_refresh_timestamp_seconds
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: soteria-refresh-age-seconds
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [900]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: Alerting
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "Soteria inventory telemetry has not refreshed in >15m"
|
|
labels:
|
|
severity: warning
|
|
- uid: maint-soteria-backup-unhealthy
|
|
title: "Soteria reports configured PVC backups unhealthy"
|
|
condition: C
|
|
for: "10m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: sum((pvc_backup_health == 0) and on(namespace,pvc,volume,driver) (pvc_backup_count > 0)) or on() vector(0)
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: unhealthy-pvcs
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [0]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "A previously configured PVC backup is stale or failed per Soteria"
|
|
labels:
|
|
severity: warning
|
|
- uid: maint-soteria-b2-scan-unhealthy
|
|
title: "Soteria B2 usage scan failing or stale"
|
|
condition: C
|
|
for: "15m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 1800
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: sum((((soteria_b2_scan_success < bool 1) and (time() - soteria_b2_scan_timestamp_seconds > 600)) or (time() - soteria_b2_scan_timestamp_seconds > 1800))) or on() vector(0)
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: soteria-b2-scan-unhealthy
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [0]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "Soteria B2 consumption scan is failing or stale for >15m"
|
|
labels:
|
|
severity: warning
|
|
- uid: maint-soteria-authz-denials
|
|
title: "Soteria authorization denials elevated"
|
|
condition: C
|
|
for: "10m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 900
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: sum(increase(soteria_authz_denials_total[15m])) or on() vector(0)
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: soteria-authz-denials-15m
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [10]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "Soteria saw >10 authorization denials in 15m"
|
|
labels:
|
|
severity: warning
|
|
- uid: maint-soteria-backup-job-storm
|
|
title: "Soteria backup job creation spike"
|
|
condition: C
|
|
for: "5m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 600
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: count(kube_job_created{namespace="maintenance",job_name=~"soteria-backup-.*"} > time() - 600) or on() vector(0)
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: soteria-backup-jobs-created-10m
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [8]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Alerting
|
|
annotations:
|
|
summary: "Soteria created >8 backup jobs in 10m (possible scheduler storm)"
|
|
labels:
|
|
severity: warning
|
|
- orgId: 1
|
|
name: ariadne
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: ariadne-schedule-error
|
|
title: "Ariadne schedule task failed"
|
|
condition: C
|
|
for: "15m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: max by (task) ((((time() - ariadne_schedule_last_error_timestamp_seconds{task=~"schedule\\..+"}) * on(task) group_left() (1 - ariadne_schedule_last_status{task=~"schedule\\..+"})) and on(task) (ariadne_schedule_next_run_timestamp_seconds{task=~"schedule\\..+"} < time() + 3600)))
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: '{{task}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [3600]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "Ariadne schedule has failed for >1h ({{ $labels.task }})"
|
|
labels:
|
|
severity: warning
|
|
- uid: ariadne-scheduler-stalled
|
|
title: "Ariadne scheduler behind (>15m)"
|
|
condition: C
|
|
for: "10m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
expr: time() - ariadne_schedule_next_run_timestamp_seconds{task=~"schedule\\..+"}
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
legendFormat: '{{task}}'
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [900]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "Ariadne scheduler behind for {{ $labels.task }}"
|
|
labels:
|
|
severity: warning
|
|
- orgId: 1
|
|
name: postmark
|
|
folder: Alerts
|
|
interval: 1m
|
|
rules:
|
|
- uid: postmark-bounce
|
|
title: "Postmark bounce rate high"
|
|
condition: C
|
|
for: "10m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: (max(postmark_outbound_bounce_rate{window="1d"}) and on() (max(postmark_outbound_sent{window="1d"}) >= 50) and on() (max(postmark_outbound_bounced{window="1d"}) >= 3)) or on() vector(0)
|
|
legendFormat: bounce 1d
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [5]
|
|
type: gt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: OK
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "Postmark 1d bounce rate >5% with at least 50 sent and 3 bounced"
|
|
labels:
|
|
severity: warning
|
|
- uid: postmark-api-down
|
|
title: "Postmark exporter down"
|
|
condition: C
|
|
for: "20m"
|
|
data:
|
|
- refId: A
|
|
relativeTimeRange:
|
|
from: 300
|
|
to: 0
|
|
datasourceUid: atlas-vm
|
|
model:
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
expr: avg_over_time(postmark_api_up[15m]) or on() vector(0)
|
|
legendFormat: api up
|
|
datasource:
|
|
type: prometheus
|
|
uid: atlas-vm
|
|
- refId: B
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: A
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
reducer: last
|
|
type: reduce
|
|
- refId: C
|
|
datasourceUid: __expr__
|
|
model:
|
|
expression: B
|
|
intervalMs: 60000
|
|
maxDataPoints: 43200
|
|
type: threshold
|
|
conditions:
|
|
- evaluator:
|
|
params: [1]
|
|
type: lt
|
|
operator:
|
|
type: and
|
|
reducer:
|
|
type: last
|
|
type: query
|
|
noDataState: NoData
|
|
execErrState: Error
|
|
annotations:
|
|
summary: "Postmark exporter reports sustained API outage"
|
|
labels:
|
|
severity: warning
|