From 72e7a39373bbe9c4ee5892868dd9e4c573188fdb Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Thu, 16 Apr 2026 19:30:31 -0300
Subject: [PATCH] monitoring: fix grafana no-data scrape gaps

---
 services/maintenance/metis-service.yaml |  2 +-
 services/monitoring/helmrelease.yaml    | 12 ++++++++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/services/maintenance/metis-service.yaml b/services/maintenance/metis-service.yaml
index 5e45c3c2..3e08eb69 100644
--- a/services/maintenance/metis-service.yaml
+++ b/services/maintenance/metis-service.yaml
@@ -6,7 +6,7 @@ metadata:
   namespace: maintenance
   annotations:
     prometheus.io/scrape: "true"
-    prometheus.io/port: "80"
+    prometheus.io/port: "8080"
     prometheus.io/path: "/metrics"
 spec:
   type: ClusterIP
diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml
index a0f6ca2d..fb975ea3 100644
--- a/services/monitoring/helmrelease.yaml
+++ b/services/monitoring/helmrelease.yaml
@@ -180,6 +180,9 @@ spec:
                 - action: keep
                   source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
                   regex: "true"
+                - action: keep
+                  source_labels: [__meta_kubernetes_pod_container_port_number]
+                  regex: ".+"
                 - action: drop
                   source_labels: [__meta_kubernetes_pod_container_port_name]
                   regex: ".*health.*"
@@ -194,6 +197,7 @@ spec:
 
             # --- kube-state-metrics (via its Service) ---
             - job_name: "kube-state-metrics"
+              max_scrape_size: 33554432
               kubernetes_sd_configs: [{ role: endpoints }]
               relabel_configs:
                 - action: keep
@@ -257,6 +261,14 @@ spec:
                 - action: keep
                   source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app_kubernetes_io_part_of]
                   regex: flux-system;flux
+                - action: keep
+                  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
+                  regex: "true"
+                - action: replace
+                  regex: (.+):(?:\d+);(\d+)
+                  replacement: $1:$2
+                  source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
+                  target_label: __address__
 
 ---