bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

monitoring: refine mail overview panels

Browse Source
This commit is contained in:
Brad Stein 2026-01-06 02:34:52 -03:00
parent d5d2fc66b9
commit 5fcff4fc8a
3 changed files with 432 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

129
scripts/dashboards_render_atlas.py
View File

@ -926,15 +926,6 @@ def build_overview():
{"color": "red", "value": 10}, {"color": "red", "value": 10},
], ],
} }
mail_bounce_count_thresholds = {
"mode": "absolute",
"steps": [
{"color": "green", "value": None},
{"color": "yellow", "value": 1},
{"color": "orange", "value": 10},
{"color": "red", "value": 100},
],
}
mail_limit_thresholds = { mail_limit_thresholds = {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -944,56 +935,96 @@ def build_overview():
{"color": "red", "value": 95}, {"color": "red", "value": 95},
], ],
} }
mail_api_thresholds = { mail_success_thresholds = {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
{"color": "red", "value": None}, {"color": "red", "value": None},
{"color": "green", "value": 1}, {"color": "orange", "value": 90},
{"color": "yellow", "value": 95},
{"color": "green", "value": 98},
], ],
} }
mail_overview_panels = [ panels.append(
( stat_panel(
30, 30,
"Mail Bounce Rate (1d)", "Mail Sent (1d)",
'max(postmark_outbound_bounce_rate{window="1d"})', 'max(postmark_outbound_sent{window="1d"})',
"percent", {"h": 2, "w": 6, "x": 0, "y": 8},
mail_bounce_rate_thresholds, unit="none",
), links=link_to("atlas-mail"),
( )
31, )
"Mail Bounced (1d)", panels.append(
'max(postmark_outbound_bounced{window="1d"})', {
"none", "id": 31,
mail_bounce_count_thresholds, "type": "stat",
), "title": "Mail Bounces (1d)",
( "datasource": PROM_DS,
"gridPos": {"h": 2, "w": 6, "x": 6, "y": 8},
"targets": [
{
"expr": 'max(postmark_outbound_bounce_rate{window="1d"})',
"refId": "A",
"legendFormat": "Rate",
},
{
"expr": 'max(postmark_outbound_bounced{window="1d"})',
"refId": "B",
"legendFormat": "Count",
},
],
"fieldConfig": {
"defaults": {
"color": {"mode": "thresholds"},
"custom": {"displayMode": "auto"},
"thresholds": mail_bounce_rate_thresholds,
"unit": "none",
},
"overrides": [
{
"matcher": {"id": "byName", "options": "Rate"},
"properties": [{"id": "unit", "value": "percent"}],
},
{
"matcher": {"id": "byName", "options": "Count"},
"properties": [{"id": "unit", "value": "none"}],
},
],
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {"calcs": ["lastNotNull"], "fields": "", "values": False},
"textMode": "name_and_value",
},
"links": link_to("atlas-mail"),
}
)
panels.append(
stat_panel(
32, 32,
"Mail Success Rate (1d)",
'clamp_min(100 - max(postmark_outbound_bounce_rate{window="1d"}), 0)',
{"h": 2, "w": 6, "x": 12, "y": 8},
unit="percent",
thresholds=mail_success_thresholds,
decimals=1,
links=link_to("atlas-mail"),
)
)
panels.append(
stat_panel(
33,
"Mail Limit Used (30d)", "Mail Limit Used (30d)",
"max(postmark_sending_limit_used_percent)", "max(postmark_sending_limit_used_percent)",
"percent", {"h": 2, "w": 6, "x": 18, "y": 8},
mail_limit_thresholds, unit="percent",
), thresholds=mail_limit_thresholds,
( decimals=1,
33, links=link_to("atlas-mail"),
"Postmark API Up",
"max(postmark_api_up)",
"none",
mail_api_thresholds,
),
]
for idx, (panel_id, title, expr, unit, thresholds) in enumerate(mail_overview_panels):
panels.append(
stat_panel(
panel_id,
title,
expr,
{"h": 2, "w": 6, "x": 6 * idx, "y": 8},
unit=unit,
thresholds=thresholds,
decimals=1 if unit == "percent" else 0,
links=link_to("atlas-mail"),
)
) )
)
storage_panels = [ storage_panels = [
(23, "Astreae Usage", astreae_usage_expr("/mnt/astreae"), "percent"), (23, "Astreae Usage", astreae_usage_expr("/mnt/astreae"), "percent"),

324
services/monitoring/dashboards/atlas-overview.json
View File

@ -789,7 +789,7 @@
{ {
"id": 30, "id": 30,
"type": "stat", "type": "stat",
"title": "Mail Bounce Rate (1d)", "title": "Mail Sent (1d)",
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "atlas-vm" "uid": "atlas-vm"
@ -802,7 +802,7 @@
}, },
"targets": [ "targets": [
{ {
"expr": "max(postmark_outbound_bounce_rate{window=\"1d\"})", "expr": "max(postmark_outbound_sent{window=\"1d\"})",
"refId": "A" "refId": "A"
} }
], ],
@ -812,6 +812,81 @@
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "rgba(115, 115, 115, 1)",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none",
"custom": {
"displayMode": "auto"
}
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{
"id": 31,
"type": "stat",
"title": "Mail Bounces (1d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 6,
"y": 8
},
"targets": [
{
"expr": "max(postmark_outbound_bounce_rate{window=\"1d\"})",
"refId": "A",
"legendFormat": "Rate"
},
{
"expr": "max(postmark_outbound_bounced{window=\"1d\"})",
"refId": "B",
"legendFormat": "Count"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"displayMode": "auto"
},
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -833,6 +908,103 @@
} }
] ]
}, },
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Rate"
},
"properties": [
{
"id": "unit",
"value": "percent"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Count"
},
"properties": [
{
"id": "unit",
"value": "none"
}
]
}
]
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "name_and_value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{
"id": 32,
"type": "stat",
"title": "Mail Success Rate (1d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 12,
"y": 8
},
"targets": [
{
"expr": "clamp_min(100 - max(postmark_outbound_bounce_rate{window=\"1d\"}), 0)",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 90
},
{
"color": "yellow",
"value": 95
},
{
"color": "green",
"value": 98
}
]
},
"unit": "percent", "unit": "percent",
"custom": { "custom": {
"displayMode": "auto" "displayMode": "auto"
@ -863,83 +1035,7 @@
] ]
}, },
{ {
"id": 31, "id": 33,
"type": "stat",
"title": "Mail Bounced (1d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 6,
"y": 8
},
"targets": [
{
"expr": "max(postmark_outbound_bounced{window=\"1d\"})",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "orange",
"value": 10
},
{
"color": "red",
"value": 100
}
]
},
"unit": "none",
"custom": {
"displayMode": "auto"
},
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{
"id": 32,
"type": "stat", "type": "stat",
"title": "Mail Limit Used (30d)", "title": "Mail Limit Used (30d)",
"datasource": { "datasource": {
@ -949,7 +1045,7 @@
"gridPos": { "gridPos": {
"h": 2, "h": 2,
"w": 6, "w": 6,
"x": 12, "x": 18,
"y": 8 "y": 8
}, },
"targets": [ "targets": [
@ -1014,74 +1110,6 @@
} }
] ]
}, },
{
"id": 33,
"type": "stat",
"title": "Postmark API Up",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 18,
"y": 8
},
"targets": [
{
"expr": "max(postmark_api_up)",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none",
"custom": {
"displayMode": "auto"
},
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{ {
"id": 23, "id": 23,
"type": "stat", "type": "stat",

324
services/monitoring/grafana-dashboard-overview.yaml
View File

@ -798,7 +798,7 @@ data:
{ {
"id": 30, "id": 30,
"type": "stat", "type": "stat",
"title": "Mail Bounce Rate (1d)", "title": "Mail Sent (1d)",
"datasource": { "datasource": {
"type": "prometheus", "type": "prometheus",
"uid": "atlas-vm" "uid": "atlas-vm"
@ -811,7 +811,7 @@ data:
}, },
"targets": [ "targets": [
{ {
"expr": "max(postmark_outbound_bounce_rate{window=\"1d\"})", "expr": "max(postmark_outbound_sent{window=\"1d\"})",
"refId": "A" "refId": "A"
} }
], ],
@ -821,6 +821,81 @@ data:
"mode": "thresholds" "mode": "thresholds"
}, },
"mappings": [], "mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "rgba(115, 115, 115, 1)",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none",
"custom": {
"displayMode": "auto"
}
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{
"id": 31,
"type": "stat",
"title": "Mail Bounces (1d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 6,
"y": 8
},
"targets": [
{
"expr": "max(postmark_outbound_bounce_rate{window=\"1d\"})",
"refId": "A",
"legendFormat": "Rate"
},
{
"expr": "max(postmark_outbound_bounced{window=\"1d\"})",
"refId": "B",
"legendFormat": "Count"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"custom": {
"displayMode": "auto"
},
"thresholds": { "thresholds": {
"mode": "absolute", "mode": "absolute",
"steps": [ "steps": [
@ -842,6 +917,103 @@ data:
} }
] ]
}, },
"unit": "none"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "Rate"
},
"properties": [
{
"id": "unit",
"value": "percent"
}
]
},
{
"matcher": {
"id": "byName",
"options": "Count"
},
"properties": [
{
"id": "unit",
"value": "none"
}
]
}
]
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "name_and_value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{
"id": 32,
"type": "stat",
"title": "Mail Success Rate (1d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 12,
"y": 8
},
"targets": [
{
"expr": "clamp_min(100 - max(postmark_outbound_bounce_rate{window=\"1d\"}), 0)",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "orange",
"value": 90
},
{
"color": "yellow",
"value": 95
},
{
"color": "green",
"value": 98
}
]
},
"unit": "percent", "unit": "percent",
"custom": { "custom": {
"displayMode": "auto" "displayMode": "auto"
@ -872,83 +1044,7 @@ data:
] ]
}, },
{ {
"id": 31, "id": 33,
"type": "stat",
"title": "Mail Bounced (1d)",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 6,
"y": 8
},
"targets": [
{
"expr": "max(postmark_outbound_bounced{window=\"1d\"})",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "yellow",
"value": 1
},
{
"color": "orange",
"value": 10
},
{
"color": "red",
"value": 100
}
]
},
"unit": "none",
"custom": {
"displayMode": "auto"
},
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{
"id": 32,
"type": "stat", "type": "stat",
"title": "Mail Limit Used (30d)", "title": "Mail Limit Used (30d)",
"datasource": { "datasource": {
@ -958,7 +1054,7 @@ data:
"gridPos": { "gridPos": {
"h": 2, "h": 2,
"w": 6, "w": 6,
"x": 12, "x": 18,
"y": 8 "y": 8
}, },
"targets": [ "targets": [
@ -1023,74 +1119,6 @@ data:
} }
] ]
}, },
{
"id": 33,
"type": "stat",
"title": "Postmark API Up",
"datasource": {
"type": "prometheus",
"uid": "atlas-vm"
},
"gridPos": {
"h": 2,
"w": 6,
"x": 18,
"y": 8
},
"targets": [
{
"expr": "max(postmark_api_up)",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none",
"custom": {
"displayMode": "auto"
},
"decimals": 0
},
"overrides": []
},
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "center",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "value"
},
"links": [
{
"title": "Open atlas-mail dashboard",
"url": "/d/atlas-mail",
"targetBlank": true
}
]
},
{ {
"id": 23, "id": 23,
"type": "stat", "type": "stat",