atlas-iac/docs/cluster-audit-20261002/evidence-summary.json

669 lines
17 KiB
JSON

{
"audit_date_utc": "2026-10-02",
"revision": "d114ce71e8495d5025a90ee6b5e19c5d911f58b9",
"read_only": true,
"first_snapshot": "2026-10-02T06:54:43.315277+00:00",
"later_snapshot": "2026-10-02T07:18:07.565624+00:00",
"inventory": {
"workloads": 177,
"deployments_statefulsets_single_replica": 117,
"without_readiness_any_container": 77,
"namespace_workloads": {
"ai": 3,
"bstein-dev-home": 4,
"cassandra": 4,
"cert-manager": 3,
"climate": 2,
"comms": 12,
"crypto": 6,
"finance": 2,
"flux-system": 6,
"game-stream": 3,
"gitea": 1,
"harbor": 6,
"health": 1,
"hermes-scm": 1,
"hermes": 26,
"jellyfin": 3,
"jenkins": 2,
"kube-system": 11,
"logging": 10,
"longhorn-system": 12,
"mailu-mailserver": 12,
"maintenance": 15,
"metallb-system": 2,
"monitoring": 12,
"nextcloud": 2,
"outline": 2,
"planka": 1,
"quality": 3,
"sso": 4,
"sui-metrics": 1,
"traefik": 1,
"vault": 2,
"vaultwarden": 1,
"postgres": 1
}
},
"pod_summary": {
"total": 635,
"phases": {
"Running": 592,
"Succeeded": 36,
"Failed": 3,
"Pending": 4
},
"unready_nonterminal": 41,
"unready_on_offline_nodes": 35
},
"node_capacity": [
{
"node": "titan-04",
"hardware": "rpi5",
"unschedulable": true,
"taints": [
{
"effect": "NoSchedule",
"key": "node.kubernetes.io/unschedulable",
"timeAdded": "2026-09-14T01:23:43Z"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:52:01Z",
"lastTransitionTime": "2026-09-14T01:45:01Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 20,
"cpu_alloc": 3.6,
"cpu_requests": 0.69,
"cpu_limits": 6.0,
"memory_alloc_GiB": 6.63,
"memory_requests_GiB": 1.31,
"memory_limits_GiB": 6.17
},
{
"node": "titan-05",
"hardware": "rpi5",
"unschedulable": true,
"taints": [
{
"effect": "NoSchedule",
"key": "node.kubernetes.io/unschedulable",
"timeAdded": "2026-08-21T19:11:20Z"
},
{
"effect": "NoSchedule",
"key": "node.kubernetes.io/unreachable",
"timeAdded": "2026-08-21T19:11:50Z"
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/unreachable",
"timeAdded": "2026-08-21T19:12:00Z"
}
],
"ready": {
"lastHeartbeatTime": "2026-08-21T19:08:08Z",
"lastTransitionTime": "2026-08-21T19:11:50Z",
"message": "Kubelet stopped posting node status.",
"reason": "NodeStatusUnknown",
"status": "Unknown",
"type": "Ready"
},
"pods": 17,
"cpu_alloc": 3.6,
"cpu_requests": 0.53,
"cpu_limits": 4.4,
"memory_alloc_GiB": 6.63,
"memory_requests_GiB": 1.0,
"memory_limits_GiB": 4.55
},
{
"node": "titan-06",
"hardware": "rpi5",
"unschedulable": false,
"taints": [
{
"effect": "NoSchedule",
"key": "node.kubernetes.io/unreachable",
"timeAdded": "2026-09-21T13:17:45Z"
},
{
"effect": "NoExecute",
"key": "node.kubernetes.io/unreachable",
"timeAdded": "2026-09-21T13:17:52Z"
}
],
"ready": {
"lastHeartbeatTime": "2026-09-21T13:16:04Z",
"lastTransitionTime": "2026-09-21T13:17:45Z",
"message": "Kubelet stopped posting node status.",
"reason": "NodeStatusUnknown",
"status": "Unknown",
"type": "Ready"
},
"pods": 18,
"cpu_alloc": 3.6,
"cpu_requests": 0.53,
"cpu_limits": 4.4,
"memory_alloc_GiB": 6.63,
"memory_requests_GiB": 1.0,
"memory_limits_GiB": 4.55
},
{
"node": "titan-07",
"hardware": "rpi5",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:50:01Z",
"lastTransitionTime": "2026-06-23T05:48:56Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 38,
"cpu_alloc": 3.6,
"cpu_requests": 3.58,
"cpu_limits": 21.3,
"memory_alloc_GiB": 6.63,
"memory_requests_GiB": 6.16,
"memory_limits_GiB": 27.71
},
{
"node": "titan-08",
"hardware": "rpi5",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:51:11Z",
"lastTransitionTime": "2026-10-01T04:21:37Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 40,
"cpu_alloc": 3.6,
"cpu_requests": 3.17,
"cpu_limits": 19.95,
"memory_alloc_GiB": 6.63,
"memory_requests_GiB": 5.1,
"memory_limits_GiB": 20.59
},
{
"node": "titan-0a",
"hardware": "rpi5",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:52:31Z",
"lastTransitionTime": "2025-09-28T12:10:51Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 21,
"cpu_alloc": 4.0,
"cpu_requests": 0.27,
"cpu_limits": 1.85,
"memory_alloc_GiB": 7.75,
"memory_requests_GiB": 0.52,
"memory_limits_GiB": 2.02
},
{
"node": "titan-0b",
"hardware": null,
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:50:28Z",
"lastTransitionTime": "2026-04-17T00:04:44Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 20,
"cpu_alloc": 4.0,
"cpu_requests": 0.35,
"cpu_limits": 2.85,
"memory_alloc_GiB": 7.75,
"memory_requests_GiB": 0.67,
"memory_limits_GiB": 2.77
},
{
"node": "titan-0c",
"hardware": "rpi5",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:51:42Z",
"lastTransitionTime": "2026-04-11T19:48:14Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 22,
"cpu_alloc": 4.0,
"cpu_requests": 0.37,
"cpu_limits": 1.85,
"memory_alloc_GiB": 7.75,
"memory_requests_GiB": 0.58,
"memory_limits_GiB": 2.02
},
{
"node": "titan-11",
"hardware": "rpi5",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:51:31Z",
"lastTransitionTime": "2026-05-19T00:46:47Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 56,
"cpu_alloc": 3.6,
"cpu_requests": 3.55,
"cpu_limits": 26.9,
"memory_alloc_GiB": 6.63,
"memory_requests_GiB": 6.54,
"memory_limits_GiB": 27.84
},
{
"node": "titan-12",
"hardware": "rpi4",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:53:15Z",
"lastTransitionTime": "2026-04-20T02:04:34Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 45,
"cpu_alloc": 3.6,
"cpu_requests": 3.49,
"cpu_limits": 23.65,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 5.09,
"memory_limits_GiB": 30.05
},
{
"node": "titan-13",
"hardware": "rpi4",
"unschedulable": false,
"taints": [
{
"effect": "PreferNoSchedule",
"key": "atlas.bstein.dev/spillover",
"value": "true"
},
{
"effect": "PreferNoSchedule",
"key": "longhorn",
"value": "true"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:53:24Z",
"lastTransitionTime": "2026-04-11T21:39:25Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 28,
"cpu_alloc": 3.6,
"cpu_requests": 1.85,
"cpu_limits": 12.5,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 2.41,
"memory_limits_GiB": 10.42
},
{
"node": "titan-14",
"hardware": "rpi4",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:51:10Z",
"lastTransitionTime": "2026-06-18T21:08:20Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 26,
"cpu_alloc": 3.6,
"cpu_requests": 2.07,
"cpu_limits": 13.25,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 2.5,
"memory_limits_GiB": 14.3
},
{
"node": "titan-15",
"hardware": "rpi4",
"unschedulable": false,
"taints": [
{
"effect": "PreferNoSchedule",
"key": "atlas.bstein.dev/spillover",
"value": "true"
},
{
"effect": "PreferNoSchedule",
"key": "longhorn",
"value": "true"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:53:09Z",
"lastTransitionTime": "2026-09-14T06:23:47Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 25,
"cpu_alloc": 3.6,
"cpu_requests": 1.46,
"cpu_limits": 19.35,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 2.28,
"memory_limits_GiB": 28.73
},
{
"node": "titan-17",
"hardware": "rpi4",
"unschedulable": false,
"taints": [
{
"effect": "PreferNoSchedule",
"key": "atlas.bstein.dev/spillover",
"value": "true"
},
{
"effect": "PreferNoSchedule",
"key": "longhorn",
"value": "true"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:50:09Z",
"lastTransitionTime": "2026-04-12T01:01:13Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 24,
"cpu_alloc": 3.6,
"cpu_requests": 1.18,
"cpu_limits": 7.65,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 1.48,
"memory_limits_GiB": 6.7
},
{
"node": "titan-18",
"hardware": "rpi4",
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:51:57Z",
"lastTransitionTime": "2026-08-08T22:11:51Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 27,
"cpu_alloc": 3.6,
"cpu_requests": 2.0,
"cpu_limits": 15.8,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 3.88,
"memory_limits_GiB": 16.8
},
{
"node": "titan-19",
"hardware": "rpi4",
"unschedulable": false,
"taints": [
{
"effect": "PreferNoSchedule",
"key": "atlas.bstein.dev/spillover",
"value": "true"
},
{
"effect": "PreferNoSchedule",
"key": "longhorn",
"value": "true"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:54:15Z",
"lastTransitionTime": "2026-08-16T01:50:17Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 22,
"cpu_alloc": 3.6,
"cpu_requests": 1.28,
"cpu_limits": 6.6,
"memory_alloc_GiB": 6.5,
"memory_requests_GiB": 2.16,
"memory_limits_GiB": 8.3
},
{
"node": "titan-20",
"hardware": null,
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:50:02Z",
"lastTransitionTime": "2026-08-23T22:32:56Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 18,
"cpu_alloc": 6.0,
"cpu_requests": 4.53,
"cpu_limits": 12.1,
"memory_alloc_GiB": 14.56,
"memory_requests_GiB": 10.97,
"memory_limits_GiB": 18.33
},
{
"node": "titan-21",
"hardware": null,
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:49:54Z",
"lastTransitionTime": "2026-09-26T15:57:39Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 28,
"cpu_alloc": 6.0,
"cpu_requests": 4.78,
"cpu_limits": 19.05,
"memory_alloc_GiB": 14.56,
"memory_requests_GiB": 7.12,
"memory_limits_GiB": 21.83
},
{
"node": "titan-22",
"hardware": "amd64",
"unschedulable": false,
"taints": [
{
"effect": "PreferNoSchedule",
"key": "atlas.bstein.dev/media-primary",
"value": "true"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:51:09Z",
"lastTransitionTime": "2026-09-30T16:15:01Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 26,
"cpu_alloc": 20.0,
"cpu_requests": 6.13,
"cpu_limits": 21.3,
"memory_alloc_GiB": 31.1,
"memory_requests_GiB": 9.47,
"memory_limits_GiB": 28.08
},
{
"node": "titan-23",
"hardware": "oceanus",
"unschedulable": false,
"taints": [
{
"effect": "NoSchedule",
"key": "veles.bstein.dev/simulation",
"value": "true"
}
],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:52:52Z",
"lastTransitionTime": "2026-09-30T15:46:40Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 34,
"cpu_alloc": 48.0,
"cpu_requests": 8.48,
"cpu_limits": 8.8,
"memory_alloc_GiB": 251.55,
"memory_requests_GiB": 10.88,
"memory_limits_GiB": 27.95
},
{
"node": "titan-24",
"hardware": null,
"unschedulable": false,
"taints": [],
"ready": {
"lastHeartbeatTime": "2026-10-02T06:50:15Z",
"lastTransitionTime": "2026-09-30T15:46:41Z",
"message": "kubelet is posting ready status",
"reason": "KubeletReady",
"status": "True",
"type": "Ready"
},
"pods": 39,
"cpu_alloc": 24.0,
"cpu_requests": 5.89,
"cpu_limits": 27.7,
"memory_alloc_GiB": 62.72,
"memory_requests_GiB": 19.69,
"memory_limits_GiB": 40.52
}
],
"k3s_database": {
"host": "titan-db",
"address": "192.168.22.10:5432",
"os": "Ubuntu 24.10",
"postgres_metadata": {
"settings": {
"rc": 0,
"output": "archive_mode|off\ndata_directory|/var/lib/postgresql/16/main\nmax_connections|100\nmax_replication_slots|10\nmax_wal_senders|10\nserver_version|16.9 (Ubuntu 16.9-0ubuntu0.24.10.1)\nshared_buffers|16384\nwal_level|replica"
},
"recovery": {
"rc": 0,
"output": "f"
},
"replication": {
"rc": 0,
"output": ""
},
"slots": {
"rc": 0,
"output": ""
},
"archiver": {
"rc": 0,
"output": "0||0|"
},
"db_sizes": {
"rc": 0,
"output": "postgres|7500 kB\nk3s|606 MB"
},
"connections": {
"rc": 0,
"output": "||5\nk3s|idle|9\npostgres|active|1"
}
},
"inspected_backup_schedule_files": [],
"root_postgres_crontabs": {
"root": {
"rc": 1,
"noncomment_entries": 0,
"has_database_backup": false
},
"postgres": {
"rc": 1,
"noncomment_entries": 0,
"has_database_backup": false
}
},
"latest_discovered_database_dump_utc": "2025-08-29T22:43:56Z",
"backup_validity": "untested",
"independent_backups": "unknown"
},
"longhorn_summary": {
"robustness": {
"unknown": 63,
"healthy": 71
},
"state": {
"detached": 62,
"attached": 72
},
"replica_targets": {
"2": 31,
"3": 99,
"1": 4
}
},
"flux_definition_creation_only_count": 46,
"flux_definition_fields_different": 22,
"validation": {
"yaml_files_parsed": 794,
"flux_paths_rendered": 58,
"lan_tls_hosts": 39,
"application_functional_tests": false,
"restore_tests": false,
"deliberate_failure_tests": false,
"changes_applied": false
}
}