From b53c7d4a1c7cdad6edd4d9824be201c04cb2257c Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Sun, 11 Jan 2026 00:29:20 -0300
Subject: [PATCH] monitoring: wire grafana smtp sync and alerting provisioning

---
 .../monitoring/grafana-alerting-config.yaml   | 26 ++++++++++
 .../monitoring/grafana-smtp-sync-cronjob.yaml | 44 +++++++++++++++++
 .../monitoring/grafana-smtp-sync-rbac.yaml    | 49 +++++++++++++++++++
 .../monitoring/grafana-smtp-sync-script.yaml  | 39 +++++++++++++++
 .../grafana-smtp-sync-serviceaccount.yaml     |  6 +++
 services/monitoring/helmrelease.yaml          | 18 +++++++
 services/monitoring/kustomization.yaml        |  5 ++
 7 files changed, 187 insertions(+)
 create mode 100644 services/monitoring/grafana-alerting-config.yaml
 create mode 100644 services/monitoring/grafana-smtp-sync-cronjob.yaml
 create mode 100644 services/monitoring/grafana-smtp-sync-rbac.yaml
 create mode 100644 services/monitoring/grafana-smtp-sync-script.yaml
 create mode 100644 services/monitoring/grafana-smtp-sync-serviceaccount.yaml

diff --git a/services/monitoring/grafana-alerting-config.yaml b/services/monitoring/grafana-alerting-config.yaml
new file mode 100644
index 0000000..b805a25
--- /dev/null
+++ b/services/monitoring/grafana-alerting-config.yaml
@@ -0,0 +1,26 @@
+# services/monitoring/grafana-alerting-config.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-alerting
+  namespace: monitoring
+  labels:
+    grafana_alerting: "1"
+data:
+  alerting.yaml: |
+    apiVersion: 1
+    contactPoints:
+      - orgId: 1
+        name: email-admins
+        receivers:
+          - uid: email-admins
+            type: email
+            settings:
+              addresses: ${GRAFANA_ALERT_EMAILS}
+              singleEmail: true
+    policies:
+      - orgId: 1
+        receiver: email-admins
+        group_by:
+          - alertname
+        continue: true
diff --git a/services/monitoring/grafana-smtp-sync-cronjob.yaml b/services/monitoring/grafana-smtp-sync-cronjob.yaml
new file mode 100644
index 0000000..8922942
--- /dev/null
+++ b/services/monitoring/grafana-smtp-sync-cronjob.yaml
@@ -0,0 +1,44 @@
+# services/monitoring/grafana-smtp-sync-cronjob.yaml
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: grafana-smtp-sync
+  namespace: monitoring
+spec:
+  schedule: "15 3 * * *"
+  concurrencyPolicy: Forbid
+  jobTemplate:
+    spec:
+      template:
+        spec:
+          serviceAccountName: grafana-smtp-sync
+          restartPolicy: OnFailure
+          containers:
+            - name: sync
+              image: bitnami/kubectl:1.31
+              command: ["/bin/sh", "-c"]
+              args:
+                - |
+                  set -euo pipefail
+                  if ! command -v jq >/dev/null 2>&1; then
+                    apt-get update >/dev/null && apt-get install -y jq >/dev/null
+                  fi
+                  exec /scripts/sync.sh
+              env:
+                - name: SOURCE_NS
+                  value: mailu-mailserver
+                - name: SOURCE_SECRET
+                  value: mailu-postmark-relay
+                - name: TARGET_NS
+                  value: monitoring
+                - name: TARGET_SECRET
+                  value: grafana-smtp
+              volumeMounts:
+                - name: script
+                  mountPath: /scripts
+                  readOnly: true
+          volumes:
+            - name: script
+              configMap:
+                name: grafana-smtp-sync-script
+                defaultMode: 0555
diff --git a/services/monitoring/grafana-smtp-sync-rbac.yaml b/services/monitoring/grafana-smtp-sync-rbac.yaml
new file mode 100644
index 0000000..aa5388e
--- /dev/null
+++ b/services/monitoring/grafana-smtp-sync-rbac.yaml
@@ -0,0 +1,49 @@
+# services/monitoring/grafana-smtp-sync-rbac.yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: grafana-smtp-sync
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+    resourceNames:
+      - mailu-postmark-relay
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: grafana-smtp-sync
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: grafana-smtp-sync
+subjects:
+  - kind: ServiceAccount
+    name: grafana-smtp-sync
+    namespace: monitoring
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: grafana-smtp-sync
+  namespace: monitoring
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: grafana-smtp-sync
+  namespace: monitoring
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: grafana-smtp-sync
+subjects:
+  - kind: ServiceAccount
+    name: grafana-smtp-sync
+    namespace: monitoring
diff --git a/services/monitoring/grafana-smtp-sync-script.yaml b/services/monitoring/grafana-smtp-sync-script.yaml
new file mode 100644
index 0000000..cccfd48
--- /dev/null
+++ b/services/monitoring/grafana-smtp-sync-script.yaml
@@ -0,0 +1,39 @@
+# services/monitoring/grafana-smtp-sync-script.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana-smtp-sync-script
+  namespace: monitoring
+data:
+  sync.sh: |
+    #!/bin/sh
+    set -euo pipefail
+
+    SOURCE_NS=${SOURCE_NS:-mailu-mailserver}
+    SOURCE_SECRET=${SOURCE_SECRET:-mailu-postmark-relay}
+    TARGET_NS=${TARGET_NS:-monitoring}
+    TARGET_SECRET=${TARGET_SECRET:-grafana-smtp}
+
+    tmp=$(mktemp)
+    cleanup() { rm -f "$tmp"; }
+    trap cleanup EXIT
+
+    kubectl -n "$SOURCE_NS" get secret "$SOURCE_SECRET" -o json > "$tmp"
+
+    user=$(jq -r '.data["relay-username"]' "$tmp")
+    pass=$(jq -r '.data["relay-password"]' "$tmp")
+
+    if [ -z "$user" ] || [ -z "$pass" ] || [ "$user" = "null" ] || [ "$pass" = "null" ]; then
+      echo "missing credentials from $SOURCE_NS/$SOURCE_SECRET" >&2
+      exit 1
+    fi
+
+    cat <<SECRET | kubectl -n "$TARGET_NS" apply -f -
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: $TARGET_SECRET
+    stringData:
+      username: $(echo "$user" | base64 -d)
+      password: $(echo "$pass" | base64 -d)
+    SECRET
diff --git a/services/monitoring/grafana-smtp-sync-serviceaccount.yaml b/services/monitoring/grafana-smtp-sync-serviceaccount.yaml
new file mode 100644
index 0000000..6ad0e18
--- /dev/null
+++ b/services/monitoring/grafana-smtp-sync-serviceaccount.yaml
@@ -0,0 +1,6 @@
+# services/monitoring/grafana-smtp-sync-serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: grafana-smtp-sync
+  namespace: monitoring
diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml
index ddd24e5..704b91d 100644
--- a/services/monitoring/helmrelease.yaml
+++ b/services/monitoring/helmrelease.yaml
@@ -264,6 +264,11 @@ spec:
       GF_AUTH_ANONYMOUS_ENABLED: "true"
       GF_AUTH_ANONYMOUS_ORG_NAME: "Overview"
       GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
+      GF_SMTP_ENABLED: "true"
+      GF_SMTP_HOST: "smtp.postmarkapp.com:587"
+      GF_SMTP_FROM: "alerts@bstein.dev"
+      GF_SMTP_FROM_NAME: "Atlas Alerts"
+      GRAFANA_ALERT_EMAILS: "alerts@bstein.dev"
       GF_SECURITY_ALLOW_EMBEDDING: "true"
       GF_AUTH_GENERIC_OAUTH_ENABLED: "true"
       GF_AUTH_GENERIC_OAUTH_NAME: "Keycloak"
@@ -286,6 +291,15 @@ spec:
         hide_version: true
       users:
         default_theme: dark
+    envValueFrom:
+      GF_SMTP_USER:
+        secretKeyRef:
+          name: grafana-smtp
+          key: username
+      GF_SMTP_PASSWORD:
+        secretKeyRef:
+          name: grafana-smtp
+          key: password
     ingress:
       enabled: true
       ingressClassName: traefik
@@ -402,6 +416,10 @@ spec:
         mountPath: /etc/grafana/provisioning/folders
         configMap: grafana-folders
         readOnly: true
+      - name: grafana-alerting
+        mountPath: /etc/grafana/provisioning/alerting
+        configMap: grafana-alerting
+        readOnly: true
 
 ---
 
diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml
index 1b3887d..7198ffc 100644
--- a/services/monitoring/kustomization.yaml
+++ b/services/monitoring/kustomization.yaml
@@ -16,6 +16,11 @@ resources:
   - dcgm-exporter.yaml
   - postmark-exporter-service.yaml
   - postmark-exporter-deployment.yaml
+  - grafana-alerting-config.yaml
+  - grafana-smtp-sync-serviceaccount.yaml
+  - grafana-smtp-sync-rbac.yaml
+  - grafana-smtp-sync-script.yaml
+  - grafana-smtp-sync-cronjob.yaml
   - grafana-folders.yaml
   - helmrelease.yaml
   - grafana-org-bootstrap.yaml