monitoring(soteria): tune PVC backup age thresholds for nightly cadence

clusters/atlas/flux-system/applications/kustomization.yaml

@@ -25,6 +25,7 @@ resources:
   - mailu/kustomization.yaml
   - jenkins/kustomization.yaml
   - ai-llm/kustomization.yaml
+  - typhon/kustomization.yaml
   - nextcloud/kustomization.yaml
   - nextcloud-mail-sync/kustomization.yaml
   - outline/kustomization.yaml

clusters/atlas/flux-system/applications/typhon/kustomization.yaml

@@ -0,0 +1,29 @@
+# clusters/atlas/flux-system/applications/typhon/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: typhon
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/typhon
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  targetNamespace: climate
+  dependsOn:
+    - name: vault
+    - name: vault-csi
+    - name: monitoring
+  healthChecks:
+    - apiVersion: apps/v1
+      kind: Deployment
+      name: typhon
+      namespace: climate
+    - apiVersion: v1
+      kind: Service
+      name: typhon
+      namespace: climate
+  wait: false
+  timeout: 20m

scripts/dashboards_render_atlas.py

@@ -1631,16 +1631,16 @@ def build_overview():
                 "mode": "absolute",
                 "steps": [
                     {"color": "green", "value": None},
-                    {"color": "yellow", "value": 6},
-                    {"color": "orange", "value": 12},
-                    {"color": "red", "value": 24},
+                    {"color": "yellow", "value": 20},
+                    {"color": "orange", "value": 40},
+                    {"color": "red", "value": 50},
                 ],
             },
         )
     )
     panels[-1]["links"] = link_to("atlas-storage")
     panels[-1]["description"] = (
-        "Oldest successful backup age in hours by PVC. PVCs with missing or unhealthy backup state are forced to 999h so the red bars stay visible in the overview."
+        "Oldest successful backup age in hours by PVC with nightly cadence thresholds (green <=20h, yellow <40h, orange <50h, red >=50h). PVCs with missing or unhealthy backup state are forced to 999h so critical bars stay visible."
     )
 
     panels.append(

services/monitoring/dashboards/atlas-overview.json

@@ -2053,15 +2053,15 @@
               },
               {
                 "color": "yellow",
-                "value": 6
+                "value": 20
               },
               {
                 "color": "orange",
-                "value": 12
+                "value": 40
               },
               {
                 "color": "red",
-                "value": 24
+                "value": 50
               }
             ]
           }
@@ -2097,7 +2097,7 @@
           "targetBlank": true
         }
       ],
-      "description": "Oldest successful backup age in hours by PVC. PVCs with missing or unhealthy backup state are forced to 999h so the red bars stay visible in the overview."
+      "description": "Oldest successful backup age in hours by PVC with nightly cadence thresholds (green <=20h, yellow <40h, orange <50h, red >=50h). PVCs with missing or unhealthy backup state are forced to 999h so critical bars stay visible."
     },
     {
       "id": 30,

services/monitoring/grafana-dashboard-overview.yaml

@@ -2062,15 +2062,15 @@ data:
                   },
                   {
                     "color": "yellow",
-                    "value": 6
+                    "value": 20
                   },
                   {
                     "color": "orange",
-                    "value": 12
+                    "value": 40
                   },
                   {
                     "color": "red",
-                    "value": 24
+                    "value": 50
                   }
                 ]
               }
@@ -2106,7 +2106,7 @@ data:
               "targetBlank": true
             }
           ],
-          "description": "Oldest successful backup age in hours by PVC. PVCs with missing or unhealthy backup state are forced to 999h so the red bars stay visible in the overview."
+          "description": "Oldest successful backup age in hours by PVC with nightly cadence thresholds (green <=20h, yellow <40h, orange <50h, red >=50h). PVCs with missing or unhealthy backup state are forced to 999h so critical bars stay visible."
         },
         {
           "id": 30,

services/typhon/deployment.yaml

@@ -0,0 +1,155 @@
+# services/typhon/deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: typhon
+  namespace: climate
+  labels:
+    app: typhon
+spec:
+  replicas: 1
+  revisionHistoryLimit: 3
+  selector:
+    matchLabels:
+      app: typhon
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  template:
+    metadata:
+      labels:
+        app: typhon
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9108"
+        prometheus.io/path: "/metrics"
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "typhon"
+        vault.hashicorp.com/agent-inject-secret-aci-env.sh: "kv/data/atlas/climate/typhon/ac-infinity"
+        vault.hashicorp.com/agent-inject-template-aci-env.sh: |
+          {{- with secret "kv/data/atlas/climate/typhon/ac-infinity" -}}
+          {{- if index .Data.data "ACI_EMAIL" }}
+          export ACI_EMAIL="{{ index .Data.data "ACI_EMAIL" }}"
+          {{- end }}
+          {{- if index .Data.data "ACI_PASSWORD" }}
+          export ACI_PASSWORD="{{ index .Data.data "ACI_PASSWORD" }}"
+          {{- end }}
+          {{- if index .Data.data "ACI_HOST" }}
+          export ACI_HOST="{{ index .Data.data "ACI_HOST" }}"
+          {{- end }}
+          {{- if index .Data.data "TYPHON_MODE" }}
+          export TYPHON_MODE="{{ index .Data.data "TYPHON_MODE" }}"
+          {{- end }}
+          {{- if index .Data.data "ENABLE_CONTROL_API" }}
+          export ENABLE_CONTROL_API="{{ index .Data.data "ENABLE_CONTROL_API" }}"
+          {{- end }}
+          {{- if index .Data.data "TY_BLE_DEFAULT_MAC" }}
+          export TY_BLE_DEFAULT_MAC="{{ index .Data.data "TY_BLE_DEFAULT_MAC" }}"
+          {{- end }}
+          {{- if index .Data.data "TY_BLE_ALLOWED_MACS" }}
+          export TY_BLE_ALLOWED_MACS="{{ index .Data.data "TY_BLE_ALLOWED_MACS" }}"
+          {{- end }}
+          {{- if index .Data.data "TY_BLE_DEVICE_TYPE" }}
+          export TY_BLE_DEVICE_TYPE="{{ index .Data.data "TY_BLE_DEVICE_TYPE" }}"
+          {{- end }}
+          {{- if index .Data.data "TY_BLE_SCAN_TIMEOUT_MS" }}
+          export TY_BLE_SCAN_TIMEOUT_MS="{{ index .Data.data "TY_BLE_SCAN_TIMEOUT_MS" }}"
+          {{- end }}
+          {{- if index .Data.data "TY_BLE_PORT_BASE" }}
+          export TY_BLE_PORT_BASE="{{ index .Data.data "TY_BLE_PORT_BASE" }}"
+          {{- end }}
+          {{- end -}}
+        typhon.bstein.dev/restart-rev: "2"
+    spec:
+      serviceAccountName: typhon
+      imagePullSecrets:
+        - name: harbor-regcred
+      nodeSelector:
+        kubernetes.io/arch: amd64
+        kubernetes.io/hostname: titan-22
+      containers:
+        - name: typhon
+          image: registry.bstein.dev/bstein/typhon:main
+          imagePullPolicy: Always
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -eu
+              if [ -f /vault/secrets/aci-env.sh ]; then
+                . /vault/secrets/aci-env.sh
+              fi
+              : "${TYPHON_MODE:=cloud}"
+              : "${ENABLE_CONTROL_API:=false}"
+              exec node dist/index.js
+          env:
+            - name: TYPHON_MODE
+              value: "cloud"
+            - name: ENABLE_CONTROL_API
+              value: "false"
+            - name: POLL_INTERVAL_SECONDS
+              value: "30"
+            - name: REQUEST_TIMEOUT_MS
+              value: "10000"
+            - name: LISTEN_PORT
+              value: "9108"
+            - name: CONTROL_LISTEN_PORT
+              value: "9110"
+            - name: TY_BLE_DEVICE_TYPE
+              value: "11"
+            - name: TY_BLE_SCAN_TIMEOUT_MS
+              value: "20000"
+            - name: TY_BLE_PORT_BASE
+              value: "1"
+            - name: LOG_LEVEL
+              value: "info"
+          ports:
+            - name: metrics
+              containerPort: 9108
+            - name: control
+              containerPort: 9110
+          volumeMounts:
+            - name: tmp
+              mountPath: /tmp
+            - name: dbus-socket
+              mountPath: /run/dbus/system_bus_socket
+              readOnly: true
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 20
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: metrics
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 3
+          securityContext:
+            runAsNonRoot: true
+            runAsUser: 65532
+            runAsGroup: 65532
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              drop: ["ALL"]
+          resources:
+            requests:
+              cpu: 100m
+              memory: 128Mi
+            limits:
+              cpu: 500m
+              memory: 512Mi
+      volumes:
+        - name: tmp
+          emptyDir: {}
+        - name: dbus-socket
+          hostPath:
+            path: /run/dbus/system_bus_socket
+            type: Socket

services/typhon/kustomization.yaml

@@ -0,0 +1,12 @@
+# services/typhon/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: climate
+resources:
+  - namespace.yaml
+  - serviceaccount.yaml
+  - secretproviderclass.yaml
+  - vault-sync-deployment.yaml
+  - deployment.yaml
+  - service.yaml
+  - networkpolicy.yaml

services/typhon/namespace.yaml

@@ -0,0 +1,5 @@
+# services/typhon/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: climate

services/typhon/networkpolicy.yaml

@@ -0,0 +1,53 @@
+# services/typhon/networkpolicy.yaml
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: typhon
+  namespace: climate
+spec:
+  podSelector:
+    matchLabels:
+      app: typhon
+  policyTypes:
+    - Ingress
+    - Egress
+  ingress:
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: monitoring
+      ports:
+        - protocol: TCP
+          port: 9108
+    - from:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: climate
+      ports:
+        - protocol: TCP
+          port: 9110
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: vault
+      ports:
+        - protocol: TCP
+          port: 8200
+    - to:
+        - ipBlock:
+            cidr: 0.0.0.0/0
+      ports:
+        - protocol: TCP
+          port: 80
+        - protocol: TCP
+          port: 443

services/typhon/secretproviderclass.yaml

@@ -0,0 +1,21 @@
+# services/typhon/secretproviderclass.yaml
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: typhon-vault
+  namespace: climate
+spec:
+  provider: vault
+  parameters:
+    vaultAddress: "http://vault.vault.svc.cluster.local:8200"
+    roleName: "typhon"
+    objects: |
+      - objectName: "harbor-pull__dockerconfigjson"
+        secretPath: "kv/data/atlas/shared/harbor-pull"
+        secretKey: "dockerconfigjson"
+  secretObjects:
+    - secretName: harbor-regcred
+      type: kubernetes.io/dockerconfigjson
+      data:
+        - objectName: harbor-pull__dockerconfigjson
+          key: .dockerconfigjson

services/typhon/service.yaml

@@ -0,0 +1,21 @@
+# services/typhon/service.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: typhon
+  namespace: climate
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9108"
+    prometheus.io/path: "/metrics"
+spec:
+  type: ClusterIP
+  selector:
+    app: typhon
+  ports:
+    - name: metrics
+      port: 9108
+      targetPort: metrics
+    - name: control
+      port: 9110
+      targetPort: control

services/typhon/serviceaccount.yaml

@@ -0,0 +1,6 @@
+# services/typhon/serviceaccount.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: typhon
+  namespace: climate

services/typhon/vault-sync-deployment.yaml

@@ -0,0 +1,34 @@
+# services/typhon/vault-sync-deployment.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: typhon-vault-sync
+  namespace: climate
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: typhon-vault-sync
+  template:
+    metadata:
+      labels:
+        app: typhon-vault-sync
+    spec:
+      serviceAccountName: typhon
+      containers:
+        - name: sync
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - "sleep infinity"
+          volumeMounts:
+            - name: vault-secrets
+              mountPath: /vault/secrets
+              readOnly: true
+      volumes:
+        - name: vault-secrets
+          csi:
+            driver: secrets-store.csi.k8s.io
+            readOnly: true
+            volumeAttributes:
+              secretProviderClass: typhon-vault