# services/monitoring/helmrelease.yaml
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: kube-state-metrics
  namespace: monitoring
spec:
  interval: 15m
  chart:
    spec:
      chart: kube-state-metrics
      version: "~6.0.0"
      sourceRef:
        kind: HelmRepository
        name: prometheus
        namespace: flux-system
  values:
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: kubernetes.io/hostname
                  operator: NotIn
                  values:
                    - titan-22
    prometheusScrape: false
    metricLabelsAllowlist:
      - cronjobs=[atlas.bstein.dev/glue]

---

apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: node-exporter
  namespace: monitoring
spec:
  interval: 15m
  install:
    disableWait: true
  chart:
    spec:
      chart: prometheus-node-exporter
      version: "~4.0.0"
      sourceRef:
        kind: HelmRepository
        name: prometheus
        namespace: flux-system
  upgrade:
    disableWait: true
  values:
    hostNetwork: false
    rbac:
      pspEnabled: false
    service:
      port: 19100
      targetPort: 19100
      annotations:
        prometheus.io/scrape: "true"
        prometheus.io/port: "19100"

---

apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: victoria-metrics-single
  namespace: monitoring
spec:
  interval: 15m
  upgrade:
    disableWait: true
  chart:
    spec:
      chart: victoria-metrics-single
      version: "~0.15.0"      # or omit to track appVersion
      sourceRef:
        kind: HelmRepository
        name: victoria-metrics
        namespace: flux-system
  values:
    server:
      # keep 1 year; supports "d", "y"
      extraArgs:
        retentionPeriod: "1y"   # VM flag -retentionPeriod=1y. :contentReference[oaicite:11]{index=11}
        promscrape.configCheckInterval: "30s"
        search.maxConcurrentRequests: "4"
        search.maxQueryDuration: "1m"
        search.maxQueueDuration: "30s"

      persistentVolume:
        enabled: true
        size: 100Gi
      resources:
        requests:
          cpu: 500m
          memory: 2Gi
        limits:
          cpu: "2"
          memory: 4Gi
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
              - matchExpressions:
                  - key: longhorn-host
                    operator: In
                    values:
                      - "true"
                  - key: kubernetes.io/hostname
                    operator: NotIn
                    values:
                      # Longhorn engine-image probes repeatedly fail on these
                      # attachment hosts and can surface volume I/O errors.
                      - titan-14
                      - titan-18
                      - titan-20
                      - titan-21
                      - titan-22
                      - titan-24
          preferredDuringSchedulingIgnoredDuringExecution:
            - weight: 100
              preference:
                matchExpressions:
                  - key: hardware
                    operator: In
                    values:
                      - rpi5
            - weight: 50
              preference:
                matchExpressions:
                  - key: hardware
                    operator: In
                    values:
                      - rpi4
            - weight: 25
              preference:
                matchExpressions:
                  - key: node-role.kubernetes.io/storage-backbone
                    operator: Exists

      # Enable built-in Kubernetes scraping
      scrape:
        enabled: true            # chart enables promscrape. :contentReference[oaicite:12]{index=12}
        config:
          global:
            scrape_interval: 15s

          scrape_configs:
            # VM self-metrics
            - job_name: victoriametrics
              static_configs:
                - targets: ["localhost:8428"]

            # --- K8s control-plane & nodes (from VM docs guide) ---
            - job_name: "kubernetes-apiservers"
              kubernetes_sd_configs: [{ role: endpoints }]
              scheme: https
              tls_config:
                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                insecure_skip_verify: true
              bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
              relabel_configs:
                - action: keep
                  source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_service_name,__meta_kubernetes_endpoint_port_name]
                  regex: default;kubernetes;https

            - job_name: "kubernetes-nodes"
              scheme: https
              tls_config:
                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                insecure_skip_verify: true
              bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
              kubernetes_sd_configs: [{ role: node }]
              relabel_configs:
                - action: labelmap
                  regex: __meta_kubernetes_node_label_(.+)
                - target_label: __address__
                  replacement: kubernetes.default.svc:443
                - source_labels: [__meta_kubernetes_node_name]
                  regex: (.+)
                  target_label: __metrics_path__
                  replacement: /api/v1/nodes/$1/proxy/metrics

            - job_name: "kubernetes-nodes-cadvisor"
              scheme: https
              tls_config:
                ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
                insecure_skip_verify: true
              bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
              kubernetes_sd_configs: [{ role: node }]
              relabel_configs:
                - action: labelmap
                  regex: __meta_kubernetes_node_label_(.+)
                - target_label: __address__
                  replacement: kubernetes.default.svc:443
                - source_labels: [__meta_kubernetes_node_name]
                  regex: (.+)
                  target_label: __metrics_path__
                  replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor

            # --- Annotated Services (generic autodiscovery) ---
            - job_name: "kubernetes-service-endpoints"
              kubernetes_sd_configs: [{ role: endpoints }]
              relabel_configs:
                - action: keep
                  source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
                  regex: "true"
                - action: replace
                  source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
                  regex: (https?)
                  target_label: __scheme__
                - action: replace
                  source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
                  target_label: __metrics_path__
                - action: replace
                  regex: (.+)(?::\d+);(\d+)
                  replacement: $1:$2
                  source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
                  target_label: __address__
                - action: replace
                  source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_node_name]
                  regex: "node-exporter-prometheus-node-exporter;(.+)"
                  target_label: node
                  replacement: "$1"
              metric_relabel_configs:
                - action: replace
                  source_labels: [__name__, node]
                  regex: "node_uname_info;(.+)"
                  target_label: nodename
                  replacement: "$1"

            # --- Annotated Pods (generic autodiscovery) ---
            - job_name: "kubernetes-pods"
              kubernetes_sd_configs: [{ role: pod }]
              relabel_configs:
                - action: keep
                  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
                  regex: "true"
                - action: keep
                  source_labels: [__meta_kubernetes_pod_container_port_number]
                  regex: ".+"
                - action: drop
                  source_labels: [__meta_kubernetes_pod_container_port_name]
                  regex: ".*health.*"
                - action: replace
                  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
                  target_label: __metrics_path__
                - action: replace
                  regex: (.+):(?:\d+);(\d+)
                  replacement: $1:$2
                  source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
                  target_label: __address__

            # --- kube-state-metrics (via its Service) ---
            - job_name: "kube-state-metrics"
              max_scrape_size: 67108864
              kubernetes_sd_configs: [{ role: endpoints }]
              relabel_configs:
                - action: keep
                  source_labels: [__meta_kubernetes_service_label_app_kubernetes_io_name]
                  regex: kube-state-metrics

            # --- Longhorn ---
            - job_name: "longhorn-backend"
              static_configs:
                - targets: ["longhorn-backend.longhorn-system.svc:9500"]
              metrics_path: /metrics

            # --- titan-db node_exporter (external control-plane DB host) ---
            - job_name: "titan-db"
              static_configs:
                - targets: ["192.168.22.10:9100"]
              relabel_configs:
                - source_labels: [__address__]
                  target_label: instance
                  replacement: titan-db

            # --- titan-jh node_exporter (external control-plane host) ---
            - job_name: "titan-jh"
              static_configs:
                - targets: ["192.168.22.8:9100"]
              relabel_configs:
                - source_labels: [__address__]
                  target_label: instance
                  replacement: titan-jh

            # --- Ananke power telemetry (host-level daemon on UPS hosts) ---
            - job_name: "ananke-power"
              static_configs:
                - targets: ["192.168.22.10:9560"]
                  labels:
                    instance: titan-db
                - targets: ["192.168.22.26:9560"]
                  labels:
                    instance: titan-24

            # --- cert-manager (pods expose on 9402) ---
            - job_name: "cert-manager"
              kubernetes_sd_configs: [{ role: pod }]
              relabel_configs:
                - action: keep
                  source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app_kubernetes_io_name]
                  regex: cert-manager;cert-manager
                - action: drop
                  source_labels: [__meta_kubernetes_pod_container_port_name]
                  regex: ".*health.*"
                - action: replace
                  source_labels: [__address__]
                  regex: "(.+):\\d+"
                  replacement: "$1:9402"
                  target_label: __address__

            # --- Flux controllers (default :8080/metrics) ---
            - job_name: "flux"
              kubernetes_sd_configs: [{ role: pod }]
              relabel_configs:
                - action: keep
                  source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_pod_label_app_kubernetes_io_part_of]
                  regex: flux-system;flux
                - action: keep
                  source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
                  regex: "true"
                - action: replace
                  regex: (.+):(?:\d+);(\d+)
                  replacement: $1:$2
                  source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
                  target_label: __address__

---

apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: grafana
  namespace: monitoring
spec:
  interval: 15m
  chart:
    spec:
      chart: grafana
      version: "~8.5.0"
      sourceRef:
        kind: HelmRepository
        name: grafana
        namespace: flux-system
  install:
    remediation: { retries: 3 }
    timeout: 15m
  upgrade:
    remediation:
      retries: 3
      remediateLastFailure: true
    cleanupOnFail: true
    timeout: 15m
  values:
    admin:
      existingSecret: grafana-admin
      userKey: admin-user
      passwordKey: admin-password
    serviceAccount:
      create: false
      name: monitoring-vault-sync
      automountServiceAccountToken: true
    podAnnotations:
      vault.hashicorp.com/agent-inject: "true"
      vault.hashicorp.com/role: "monitoring"
      vault.hashicorp.com/agent-requests-cpu: "25m"
      vault.hashicorp.com/agent-requests-mem: "64Mi"
      vault.hashicorp.com/agent-limits-cpu: "250m"
      vault.hashicorp.com/agent-limits-mem: "128Mi"
      monitoring.bstein.dev/restart-rev: "13"
      vault.hashicorp.com/agent-inject-secret-grafana-env.sh: "kv/data/atlas/monitoring/grafana-admin"
      vault.hashicorp.com/agent-inject-template-grafana-env.sh: |
        {{ with secret "kv/data/atlas/monitoring/grafana-admin" }}
        export GF_SECURITY_ADMIN_USER="{{ index .Data.data "admin-user" }}"
        export GF_SECURITY_ADMIN_PASSWORD="{{ index .Data.data "admin-password" }}"
        {{ end }}
        {{ with secret "kv/data/atlas/shared/postmark-relay" }}
        export GF_SMTP_USER="{{ index .Data.data "apikey" }}"
        export GF_SMTP_PASSWORD="{{ index .Data.data "apikey" }}"
        {{ end }}
    persistence:
      enabled: true
      size: 20Gi
      storageClassName: astreae
    initChownData:
      enabled: false
    resources:
      requests:
        cpu: 100m
        memory: 512Mi
      limits:
        cpu: "1"
        memory: 1Gi
    readinessProbe:
      httpGet:
        path: /api/health
        port: 3000
      periodSeconds: 10
      timeoutSeconds: 5
      failureThreshold: 6
    livenessProbe:
      httpGet:
        path: /api/health
        port: 3000
      initialDelaySeconds: 60
      periodSeconds: 10
      timeoutSeconds: 30
      failureThreshold: 10
    nodeSelector:
      kubernetes.io/arch: arm64
      node-role.kubernetes.io/worker: "true"
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: hardware
                  operator: In
                  values:
                    - rpi5
                - key: node-role.kubernetes.io/control-plane
                  operator: DoesNotExist
                - key: node-role.kubernetes.io/master
                  operator: DoesNotExist
                - key: atlas.bstein.dev/spillover
                  operator: DoesNotExist
                - key: kubernetes.io/hostname
                  operator: NotIn
                  values:
                    - titan-0a
                    - titan-0b
                    - titan-0c
                    - titan-08
        preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            preference:
              matchExpressions:
                - key: hardware
                  operator: In
                  values:
                    - rpi5
          - weight: 70
            preference:
              matchExpressions:
                - key: hardware
                  operator: In
                  values:
                    - rpi4
    deploymentStrategy:
      type: Recreate
    service:
      type: ClusterIP
    env:
      GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "grafana"
      GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: ""
      GF_AUTH_ANONYMOUS_ENABLED: "true"
      GF_AUTH_ANONYMOUS_ORG_NAME: "Overview"
      GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
      GF_SMTP_ENABLED: "true"
      GF_SMTP_HOST: "smtp.postmarkapp.com:587"
      GF_SMTP_FROM_ADDRESS: "no-reply-grafana@bstein.dev"
      GF_SMTP_FROM_NAME: "Atlas Grafana"
      GRAFANA_ALERT_EMAILS: "brad@bstein.dev"
      GF_SECURITY_ALLOW_EMBEDDING: "true"
      GF_AUTH_GENERIC_OAUTH_ENABLED: "true"
      GF_AUTH_GENERIC_OAUTH_NAME: "Keycloak"
      GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: "true"
      GF_AUTH_GENERIC_OAUTH_SCOPES: "openid profile email groups"
      GF_AUTH_GENERIC_OAUTH_AUTH_URL: "https://sso.bstein.dev/realms/atlas/protocol/openid-connect/auth"
      GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "https://sso.bstein.dev/realms/atlas/protocol/openid-connect/token"
      GF_AUTH_GENERIC_OAUTH_API_URL: "https://sso.bstein.dev/realms/atlas/protocol/openid-connect/userinfo"
      GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: "contains(groups, 'admin') && 'Admin' || 'Viewer'"
      GF_AUTH_GENERIC_OAUTH_USE_PKCE: "true"
      GF_AUTH_GENERIC_OAUTH_TLS_SKIP_VERIFY_INSECURE: "false"
      GF_AUTH_GENERIC_OAUTH_ALLOW_INSECURE_EMAIL_LOOKUP: "true"
      GF_AUTH_GENERIC_OAUTH_EMAIL_ATTRIBUTE_PATH: "email"
      GF_AUTH_SIGNOUT_REDIRECT_URL: "https://sso.bstein.dev/realms/atlas/protocol/openid-connect/logout?redirect_uri=https://metrics.bstein.dev/"
    grafana.ini:
      server:
        domain: metrics.bstein.dev
        root_url: https://metrics.bstein.dev/
      dashboards:
        default_home_dashboard_path: /var/lib/grafana/dashboards/overview/atlas-overview.json
      auth.anonymous:
        hide_version: true
      users:
        default_theme: dark
      date_formats:
        default_timezone: America/Chicago
    ingress:
      enabled: true
      ingressClassName: traefik
      annotations:
        cert-manager.io/cluster-issuer: letsencrypt
        traefik.ingress.kubernetes.io/router.entrypoints: websecure
        traefik.ingress.kubernetes.io/router.tls: "true"
      hosts:
        - metrics.bstein.dev
      path: /
      tls:
        - secretName: grafana-metrics-tls
          hosts:
            - metrics.bstein.dev
    datasources:
      datasources.yaml:
        apiVersion: 1
        datasources:
          - name: VictoriaMetrics
            type: prometheus
            access: proxy
            url: http://victoria-metrics-single-server:8428
            isDefault: true
            jsonData:
              timeInterval: "15s"
            uid: atlas-vm
            orgId: 1
          - name: VictoriaMetrics
            type: prometheus
            access: proxy
            url: http://victoria-metrics-single-server:8428
            isDefault: true
            jsonData:
              timeInterval: "15s"
            uid: atlas-vm
            orgId: 2
    dashboardProviders:
      dashboardproviders.yaml:
        apiVersion: 1
        providers:
          - name: overview
            orgId: 1
            folder: Overview
            type: file
            disableDeletion: false
            editable: false
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/overview
          - name: overview-public
            orgId: 2
            folder: Overview
            type: file
            disableDeletion: false
            editable: false
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/overview-public
          - name: pods
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/pods
          - name: nodes
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/nodes
          - name: storage
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/storage
          - name: gpu
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/gpu
          - name: network
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/network
          - name: mail
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/mail
          - name: testing
            orgId: 1
            folder: Atlas Public
            type: file
            disableDeletion: false
            editable: false
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/testing
          - name: testing-public
            orgId: 2
            folder: Atlas Public
            type: file
            disableDeletion: false
            editable: false
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/testing-public
          - name: power
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/power
          - name: cassandra
            orgId: 1
            folder: Atlas Internal
            type: file
            disableDeletion: false
            editable: true
            updateIntervalSeconds: 10
            options:
              path: /var/lib/grafana/dashboards/cassandra
    dashboardsConfigMaps:
      overview: grafana-dashboard-overview
      overview-public: grafana-dashboard-overview
      pods: grafana-dashboard-pods
      nodes: grafana-dashboard-nodes
      storage: grafana-dashboard-storage
      gpu: grafana-dashboard-gpu
      network: grafana-dashboard-network
      mail: grafana-dashboard-mail
      testing: grafana-dashboard-testing
      testing-public: grafana-dashboard-testing
      power: grafana-dashboard-power
      cassandra: grafana-dashboard-cassandra
    extraConfigmapMounts:
      - name: grafana-folders
        mountPath: /etc/grafana/provisioning/folders
        configMap: grafana-folders
        readOnly: true
      - name: grafana-alerting
        mountPath: /etc/grafana/provisioning/alerting
        configMap: grafana-alerting
        readOnly: true
  postRenderers:
    - kustomize:
        patches:
          - target:
              kind: Deployment
              name: grafana
            patch: |-
              apiVersion: apps/v1
              kind: Deployment
              metadata:
                name: grafana
              spec:
                template:
                  spec:
                    serviceAccountName: monitoring-vault-sync
                    automountServiceAccountToken: true
                    containers:
                      - name: grafana
                        command:
                          - /entrypoint.sh
                        args:
                          - /run.sh
                        env:
                          - name: GF_SECURITY_ADMIN_USER
                            $patch: delete
                          - name: GF_SECURITY_ADMIN_PASSWORD
                            $patch: delete
                          - name: GF_SMTP_USER
                            $patch: delete
                          - name: GF_SMTP_PASSWORD
                            $patch: delete
                          - name: VAULT_ENV_FILE
                            value: /vault/secrets/grafana-env.sh
                        volumeMounts:
                          - name: monitoring-vault-entrypoint
                            mountPath: /entrypoint.sh
                            subPath: vault-entrypoint.sh
                    volumes:
                      - name: monitoring-vault-entrypoint
                        configMap:
                          name: monitoring-vault-entrypoint
                          defaultMode: 493

---

apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
  name: alertmanager
  namespace: monitoring
spec:
  interval: 15m
  chart:
    spec:
      chart: alertmanager
      version: "~1.9.0"
      sourceRef:
        kind: HelmRepository
        name: prometheus
        namespace: flux-system
  values:
    affinity:
      nodeAffinity:
        requiredDuringSchedulingIgnoredDuringExecution:
          nodeSelectorTerms:
            - matchExpressions:
                - key: kubernetes.io/hostname
                  operator: NotIn
                  values:
                    - titan-22
    ingress:
      enabled: true
      ingressClassName: traefik
      annotations:
        cert-manager.io/cluster-issuer: letsencrypt
        traefik.ingress.kubernetes.io/router.entrypoints: websecure
        traefik.ingress.kubernetes.io/router.tls: "true"
      hosts:
        - host: alerts.bstein.dev
          paths:
            - path: /
              pathType: Prefix
      tls:
        - secretName: alerts-bstein-dev-tls
          hosts:
            - alerts.bstein.dev
    config:
      global:
        resolve_timeout: 5m
        # Mailu accepts relay for local domains from inside the cluster, so no
        # submission credentials are needed for mail addressed to bstein.dev.
        smtp_smarthost: mailu-front.mailu-mailserver.svc.cluster.local:25
        smtp_from: alertmanager@bstein.dev
        smtp_require_tls: false
        # Mailu rejects the default "HELO localhost" with 504 5.5.2; the
        # greeting has to be a fully-qualified name.
        smtp_hello: alerts.bstein.dev
      route:
        receiver: default
        group_wait: 30s
        group_interval: 5m
        repeat_interval: 2h
        routes:
          # Triage escalations are the only alerts mailed today; everything
          # else stays on the silent default receiver until it is tuned.
          - receiver: email
            matchers:
              - service = hermes-triage
            group_wait: 60s
            repeat_interval: 12h
      receivers:
        - name: default
        - name: email
          email_configs:
            - to: brad@bstein.dev
              send_resolved: true
              headers:
                Subject: '[atlas] {{ .Status | toUpper }} {{ .CommonLabels.alertname }} {{ .CommonLabels.jenkins_job }}/{{ .CommonLabels.build }}'
