diff --git a/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml b/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml index 0df5a6adb..4d64d9a05 100644 --- a/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml +++ b/infrastructure/longhorn/core/longhorn-disk-tags-ensure-job.yaml @@ -2,7 +2,7 @@ apiVersion: batch/v1 kind: Job metadata: - name: longhorn-disk-tags-ensure-3 + name: longhorn-disk-tags-ensure-4 namespace: longhorn-system spec: backoffLimit: 0 diff --git a/services/monitoring/grafana-dashboard-cassandra.yaml b/services/monitoring/grafana-dashboard-cassandra.yaml new file mode 100644 index 000000000..e8f59a434 --- /dev/null +++ b/services/monitoring/grafana-dashboard-cassandra.yaml @@ -0,0 +1,611 @@ +# services/monitoring/grafana-dashboard-cassandra.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-cassandra + labels: + grafana_dashboard: "1" +data: + cassandra-rollup.json: | + { + "uid": "cassandra-rollup", + "title": "Cassandra", + "folderUid": "atlas-internal", + "editable": true, + "tags": [ + "cassandra", + "veles", + "migration" + ], + "time": { + "from": "now-6h", + "to": "now" + }, + "refresh": "30s", + "schemaVersion": 39, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Cassandra Ready Containers", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 0 + }, + "targets": [ + { + "expr": "sum(kube_pod_container_status_ready{namespace=\"cassandra\"}) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 4 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 2, + "type": "stat", + "title": "Legacy Veles Ready Containers", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 0 + }, + "targets": [ + { + "expr": "sum(kube_pod_container_status_ready{namespace=\"veles\"}) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-green", + "value": 4 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 3, + "type": "stat", + "title": "Cassandra + Veles Problem Pods", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 0 + }, + "targets": [ + { + "expr": "sum(max by (namespace,pod) (kube_pod_status_phase{namespace=~\"cassandra|veles\",phase!~\"Running|Succeeded\"})) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 4, + "type": "stat", + "title": "Cassandra + Veles Restarts 1h", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 0 + }, + "targets": [ + { + "expr": "sum(increase(kube_pod_container_status_restarts_total{namespace=~\"cassandra|veles\"}[1h])) or on() vector(0)", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 1 + }, + { + "color": "dark-red", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + } + }, + { + "id": 5, + "type": "timeseries", + "title": "Ready Containers By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 4 + }, + "targets": [ + { + "expr": "sum by (namespace) (kube_pod_container_status_ready{namespace=~\"cassandra|veles\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "Problem Pods By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 4 + }, + "targets": [ + { + "expr": "sum by (namespace) (max by (namespace,pod) (kube_pod_status_phase{namespace=~\"cassandra|veles\",phase!~\"Running|Succeeded\"}))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 7, + "type": "timeseries", + "title": "CPU By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 12 + }, + "targets": [ + { + "expr": "sum by (namespace) (rate(container_cpu_usage_seconds_total{namespace=~\"cassandra|veles\",container!=\"\",pod!=\"\"}[5m]))", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Memory By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 12 + }, + "targets": [ + { + "expr": "sum by (namespace) (container_memory_working_set_bytes{namespace=~\"cassandra|veles\",container!=\"\",pod!=\"\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 9, + "type": "bargauge", + "title": "PVC Used By Namespace", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 20 + }, + "targets": [ + { + "expr": "sum by (namespace) (kubelet_volume_stats_used_bytes{namespace=~\"cassandra|veles\"})", + "legendFormat": "{{namespace}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + } + }, + { + "id": 10, + "type": "timeseries", + "title": "PVC Usage Percent", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 20 + }, + "targets": [ + { + "expr": "100 * sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_used_bytes{namespace=~\"cassandra|veles\"}) / sum by (namespace,persistentvolumeclaim) (kubelet_volume_stats_capacity_bytes{namespace=~\"cassandra|veles\"})", + "legendFormat": "{{namespace}} / {{persistentvolumeclaim}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 11, + "type": "timeseries", + "title": "Migration Jobs", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 28 + }, + "targets": [ + { + "expr": "sum by (namespace,job_name) (kube_job_status_failed{namespace=~\"cassandra|veles\",job_name=~\"cassandra-.*|veles-.*\"})", + "legendFormat": "failed {{namespace}} / {{job_name}}", + "refId": "A" + }, + { + "expr": "sum by (namespace,job_name) (kube_job_status_succeeded{namespace=~\"cassandra|veles\",job_name=~\"cassandra-.*|veles-.*\"})", + "legendFormat": "succeeded {{namespace}} / {{job_name}}", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "id": 12, + "type": "timeseries", + "title": "Titan-23 Root Filesystem", + "datasource": { + "type": "prometheus", + "uid": "atlas-vm" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 28 + }, + "targets": [ + { + "expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/\",fstype!~\"tmpfs|overlay\"} * 100) * on(instance) group_left(nodename) node_uname_info{nodename=\"titan-23\"}", + "legendFormat": "titan-23 /", + "refId": "A" + }, + { + "expr": "100 - (node_filesystem_avail_bytes{mountpoint=\"/mnt/veles\",fstype!~\"tmpfs|overlay\"} / node_filesystem_size_bytes{mountpoint=\"/mnt/veles\",fstype!~\"tmpfs|overlay\"} * 100) * on(instance) group_left(nodename) node_uname_info{nodename=\"titan-23\"}", + "legendFormat": "titan-23 /mnt/veles", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-green", + "value": null + }, + { + "color": "dark-yellow", + "value": 70 + }, + { + "color": "dark-red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + } + ] + } diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index 15c86edaa..f976dbb56 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -600,6 +600,15 @@ spec: updateIntervalSeconds: 10 options: path: /var/lib/grafana/dashboards/power + - name: cassandra + orgId: 1 + folder: Atlas Internal + type: file + disableDeletion: false + editable: true + updateIntervalSeconds: 10 + options: + path: /var/lib/grafana/dashboards/cassandra dashboardsConfigMaps: overview: grafana-dashboard-overview overview-public: grafana-dashboard-overview @@ -612,6 +621,7 @@ spec: testing: grafana-dashboard-testing testing-public: grafana-dashboard-testing power: grafana-dashboard-power + cassandra: grafana-dashboard-cassandra extraConfigmapMounts: - name: grafana-folders mountPath: /etc/grafana/provisioning/folders diff --git a/services/monitoring/kustomization.yaml b/services/monitoring/kustomization.yaml index 16df6f3f4..6bc0549a8 100644 --- a/services/monitoring/kustomization.yaml +++ b/services/monitoring/kustomization.yaml @@ -14,6 +14,7 @@ resources: - grafana-dashboard-network.yaml - grafana-dashboard-gpu.yaml - grafana-dashboard-gitops.yaml + - grafana-dashboard-cassandra.yaml - grafana-dashboard-power.yaml - grafana-dashboard-mail.yaml - grafana-dashboard-testing.yaml