infrastructure/postgres/service.yaml

@@ -4,10 +4,6 @@ kind: Service
 metadata:
   name: postgres-service
   namespace: postgres
-  annotations:
-    prometheus.io/scrape: "true"
-    prometheus.io/port: "9187"
-    prometheus.io/path: "/metrics"
 spec:
   clusterIP: None
   ports:
@@ -15,9 +11,5 @@ spec:
       port: 5432
       protocol: TCP
       targetPort: 5432
-    - name: metrics
-      port: 9187
-      protocol: TCP
-      targetPort: 9187
   selector:
     app: postgres

infrastructure/postgres/statefulset.yaml

@@ -58,23 +58,6 @@ spec:
             - name: vault-secrets
               mountPath: /mnt/vault
               readOnly: true
-        - name: postgres-exporter
-          image: quay.io/prometheuscommunity/postgres-exporter:v0.15.0
-          ports:
-            - name: metrics
-              containerPort: 9187
-              protocol: TCP
-          env:
-            - name: DATA_SOURCE_URI
-              value: "localhost:5432/postgres?sslmode=disable"
-            - name: DATA_SOURCE_USER
-              value: postgres
-            - name: DATA_SOURCE_PASS_FILE
-              value: /mnt/vault/postgres_password
-          volumeMounts:
-            - name: vault-secrets
-              mountPath: /mnt/vault
-              readOnly: true
       volumes:
         - name: vault-secrets
           csi:

scripts/dashboards_render_atlas.py

@@ -371,9 +371,8 @@ ARIADNE_TEST_SUCCESS_RATE = (
 ARIADNE_TEST_FAILURES_24H = (
     'sum by (result) (max_over_time(ariadne_ci_tests_total{repo="ariadne",result=~"failed|error"}[24h]))'
 )
-POSTGRES_CONN_USED = (
-    'label_replace(sum(pg_stat_activity_count), "conn", "used", "__name__", ".*") '
-    'or label_replace(max(pg_settings_max_connections), "conn", "max", "__name__", ".*")'
+POSTGRES_CONN_USED_PCT = (
+    "100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)"
 )
 POSTGRES_CONN_HOTTEST = 'topk(1, sum by (datname) (pg_stat_activity_count))'
 ONEOFF_JOB_OWNER = (
@@ -1139,15 +1138,14 @@ def build_overview():
         )
     )
     panels.append(
-        stat_panel(
+        gauge_panel(
             34,
             "Postgres Connections Used",
-            POSTGRES_CONN_USED,
+            POSTGRES_CONN_USED_PCT,
             {"h": 3, "w": 4, "x": 16, "y": 8},
-            decimals=0,
-            text_mode="name_and_value",
-            legend="{{conn}}",
-            instant=True,
+            min_value=0,
+            max_value=100,
+            thresholds=PERCENT_THRESHOLDS,
         )
     )
     panels.append(

services/jenkins/deployment.yaml

@@ -108,9 +108,7 @@ spec:
               containerPort: 50000
           env:
             - name: JAVA_OPTS
-              value: "-Xms512m -Xmx2048m -Duser.timezone=America/Chicago"
-            - name: TZ
-              value: "America/Chicago"
+              value: "-Xms512m -Xmx2048m"
             - name: JENKINS_OPTS
               value: "--webroot=/var/jenkins_cache/war"
             - name: JENKINS_SLAVE_AGENT_PORT

services/monitoring/dashboards/atlas-overview.json

@@ -1112,7 +1112,7 @@
     },
     {
       "id": 34,
-      "type": "stat",
+      "type": "gauge",
       "title": "Postgres Connections Used",
       "datasource": {
         "type": "prometheus",
@@ -1126,43 +1126,39 @@
       },
       "targets": [
         {
-          "expr": "label_replace(sum(pg_stat_activity_count), \"conn\", \"used\", \"__name__\", \".*\") or label_replace(max(pg_settings_max_connections), \"conn\", \"max\", \"__name__\", \".*\")",
-          "refId": "A",
-          "legendFormat": "{{conn}}",
-          "instant": true
+          "expr": "100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)",
+          "refId": "A"
         }
       ],
       "fieldConfig": {
         "defaults": {
-          "color": {
-            "mode": "thresholds"
-          },
-          "mappings": [],
+          "min": 0,
+          "max": 100,
           "thresholds": {
             "mode": "absolute",
             "steps": [
               {
-                "color": "rgba(115, 115, 115, 1)",
+                "color": "green",
                 "value": null
               },
               {
-                "color": "green",
-                "value": 1
+                "color": "yellow",
+                "value": 50
+              },
+              {
+                "color": "orange",
+                "value": 75
+              },
+              {
+                "color": "red",
+                "value": 91.5
               }
             ]
-          },
-          "unit": "none",
-          "custom": {
-            "displayMode": "auto"
-          },
-          "decimals": 0
+          }
         },
         "overrides": []
       },
       "options": {
-        "colorMode": "value",
-        "graphMode": "area",
-        "justifyMode": "center",
         "reduceOptions": {
           "calcs": [
             "lastNotNull"
@@ -1170,7 +1166,9 @@
           "fields": "",
           "values": false
         },
-        "textMode": "name_and_value"
+        "orientation": "auto",
+        "showThresholdMarkers": false,
+        "showThresholdLabels": false
       }
     },
     {

services/monitoring/grafana-dashboard-overview.yaml

@@ -1121,7 +1121,7 @@ data:
         },
         {
           "id": 34,
-          "type": "stat",
+          "type": "gauge",
           "title": "Postgres Connections Used",
           "datasource": {
             "type": "prometheus",
@@ -1135,43 +1135,39 @@ data:
           },
           "targets": [
             {
-              "expr": "label_replace(sum(pg_stat_activity_count), \"conn\", \"used\", \"__name__\", \".*\") or label_replace(max(pg_settings_max_connections), \"conn\", \"max\", \"__name__\", \".*\")",
-              "refId": "A",
-              "legendFormat": "{{conn}}",
-              "instant": true
+              "expr": "100 * sum(pg_stat_activity_count) / clamp_min(max(pg_settings_max_connections), 1)",
+              "refId": "A"
             }
           ],
           "fieldConfig": {
             "defaults": {
-              "color": {
-                "mode": "thresholds"
-              },
-              "mappings": [],
+              "min": 0,
+              "max": 100,
               "thresholds": {
                 "mode": "absolute",
                 "steps": [
                   {
-                    "color": "rgba(115, 115, 115, 1)",
+                    "color": "green",
                     "value": null
                   },
                   {
-                    "color": "green",
-                    "value": 1
+                    "color": "yellow",
+                    "value": 50
+                  },
+                  {
+                    "color": "orange",
+                    "value": 75
+                  },
+                  {
+                    "color": "red",
+                    "value": 91.5
                   }
                 ]
-              },
-              "unit": "none",
-              "custom": {
-                "displayMode": "auto"
-              },
-              "decimals": 0
+              }
             },
             "overrides": []
           },
           "options": {
-            "colorMode": "value",
-            "graphMode": "area",
-            "justifyMode": "center",
             "reduceOptions": {
               "calcs": [
                 "lastNotNull"
@@ -1179,7 +1175,9 @@ data:
               "fields": "",
               "values": false
             },
-            "textMode": "name_and_value"
+            "orientation": "auto",
+            "showThresholdMarkers": false,
+            "showThresholdLabels": false
           }
         },
         {