From 9c0af401e94854c96a342aec11a8afa4e5d22a73 Mon Sep 17 00:00:00 2001 From: jenkins Date: Thu, 6 Aug 2026 04:42:12 -0300 Subject: [PATCH] docs: add cluster architecture diagrams --- mermaid/HardwareArchitecture.mmd | 503 ++++++++++++++++++ mermaid/HardwareArchitecture.svg | 67 +++ mermaid/ServiceAccessArchitecture.mmd | 705 ++++++++++++++++++++++++++ mermaid/ServiceAccessArchitecture.svg | 67 +++ mermaid/TestAutomation.mmd | 213 ++++++++ mermaid/TestAutomation.svg | 67 +++ 6 files changed, 1622 insertions(+) create mode 100644 mermaid/HardwareArchitecture.mmd create mode 100644 mermaid/HardwareArchitecture.svg create mode 100644 mermaid/ServiceAccessArchitecture.mmd create mode 100644 mermaid/ServiceAccessArchitecture.svg create mode 100644 mermaid/TestAutomation.mmd create mode 100644 mermaid/TestAutomation.svg diff --git a/mermaid/HardwareArchitecture.mmd b/mermaid/HardwareArchitecture.mmd new file mode 100644 index 000000000..921171920 --- /dev/null +++ b/mermaid/HardwareArchitecture.mmd @@ -0,0 +1,503 @@ +%% Titan Lab physical hardware architecture. +%% Solid links are physical network, storage, power, or sensor paths. +%% Dashed links are administration, telemetry, or recovery control. +%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "NETWORK_SIMPLEX", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%% +flowchart TB + internet["Internet"]:::external + ac_cloud["AC Infinity account"]:::external + + subgraph lab["Titan Lab"] + direction TB + + subgraph link_legend["Link colors"] + direction LR + + subgraph legend_network["Network"] + direction TB + legend_1g["1 Gbps"]:::legend1g + legend_25g["2.5 Gbps"]:::legend25g + legend_external["External network"]:::legendExternal + end + + subgraph legend_resources["Resources"] + direction TB + legend_local["Local attachment"]:::legendLocal + legend_shared["Shared storage"]:::legendShared + legend_gpu["GPU"]:::legendGpu + end + + subgraph legend_operations["Operations"] + direction TB + legend_power["Power and NUT"]:::legendPower + legend_telemetry["Telemetry and environment"]:::legendTelemetry + legend_control["Administration and recovery"]:::legendControl + legend_safety["Protective shutdown"]:::legendSafety + end + end + + subgraph network["Network fabric"] + direction LR + + router["ASUS RT-AX88U
Router"]:::network + cisco["Cisco CBS110-24T
1 Gbps switch"]:::network + trendnet["TRENDnet 9-port
2.5 Gbps switch"]:::network + + router ==> cisco + router ==> trendnet + end + + subgraph atlas["Atlas Kubernetes cluster"] + direction TB + + subgraph control_plane["High-availability control plane, ARM64"] + direction LR + + subgraph titan_0a["titan-0a"] + direction TB + t0a_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::control + t0a_storage["Local
500 GiB boot SSD"]:::localStorage + t0a_host --- t0a_storage + end + + subgraph titan_0b["titan-0b"] + direction TB + t0b_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::control + t0b_storage["Local
500 GiB boot SSD"]:::localStorage + t0b_host --- t0b_storage + end + + subgraph titan_0c["titan-0c"] + direction TB + t0c_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::control + t0c_storage["Local
500 GiB boot SSD"]:::localStorage + t0c_host --- t0c_storage + end + + t0a_host --- t0b_host + t0b_host --- t0c_host + end + + subgraph workers["Workers"] + direction TB + + subgraph pi5_workers["Newer workers, ARM64 Raspberry Pi 5"] + direction TB + + subgraph pi5_row_a[" "] + direction LR + + subgraph titan_04["titan-04"] + direction TB + t04_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::pi5 + t04_storage["Local
SD root
64 GiB astraios"]:::localStorage + t04_host --- t04_storage + end + + subgraph titan_05["titan-05"] + direction TB + t05_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::pi5 + t05_storage["Local
SD root
64 GiB astraios"]:::localStorage + t05_host --- t05_storage + end + + subgraph titan_06["titan-06"] + direction TB + t06_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::pi5 + t06_storage["Local
SD root
64 GiB astraios"]:::localStorage + t06_host --- t06_storage + end + end + + subgraph pi5_row_b[" "] + direction LR + + subgraph titan_07["titan-07"] + direction TB + t07_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::pi5 + t07_storage["Local
SD root
64 GiB astraios"]:::localStorage + t07_host --- t07_storage + end + + subgraph titan_08["titan-08"] + direction TB + t08_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::pi5 + t08_storage["Local
SD root
64 GiB astraios"]:::localStorage + t08_host --- t08_storage + end + + subgraph titan_11["titan-11"] + direction TB + t11_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::pi5 + t11_storage["Local
SD root
64 GiB astraios"]:::localStorage + t11_host --- t11_storage + end + end + end + + subgraph pi4_workers["Older workers, ARM64 Raspberry Pi 4"] + direction TB + + subgraph pi4_row_a[" "] + direction LR + + subgraph titan_12["titan-12"] + direction TB + t12_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t12_root["Local
SD root"]:::localStorage + t12_host --- t12_root + end + + subgraph titan_13["titan-13"] + direction TB + t13_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t13_root["Local
SD root"]:::localStorage + t13_astreae["Longhorn
8 TiB Astreae"]:::sharedStorage + t13_asteria["Longhorn
12 TiB Asteria"]:::sharedStorage + t13_host --- t13_root + t13_host --- t13_astreae + t13_host --- t13_asteria + end + + subgraph titan_14["titan-14"] + direction TB + t14_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t14_root["Local
SD root"]:::localStorage + t14_host --- t14_root + end + + subgraph titan_15["titan-15"] + direction TB + t15_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t15_root["Local
SD root"]:::localStorage + t15_astreae["Longhorn
8 TiB Astreae"]:::sharedStorage + t15_asteria["Longhorn
12 TiB Asteria"]:::sharedStorage + t15_host --- t15_root + t15_host --- t15_astreae + t15_host --- t15_asteria + end + end + + subgraph pi4_row_b[" "] + direction LR + + subgraph titan_17["titan-17"] + direction TB + t17_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t17_root["Local
SD root"]:::localStorage + t17_astreae["Longhorn
8 TiB Astreae"]:::sharedStorage + t17_asteria["Longhorn
12 TiB Asteria"]:::sharedStorage + t17_host --- t17_root + t17_host --- t17_astreae + t17_host --- t17_asteria + end + + subgraph titan_18["titan-18"] + direction TB + t18_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t18_root["Local
SD root"]:::localStorage + t18_host --- t18_root + end + + subgraph titan_19["titan-19"] + direction TB + t19_host["Raspberry Pi 4
8 GiB RAM
Armbian"]:::pi4 + t19_root["Local
SD root"]:::localStorage + t19_astreae["Longhorn
8 TiB Astreae"]:::sharedStorage + t19_asteria["Longhorn
12 TiB Asteria"]:::sharedStorage + t19_host --- t19_root + t19_host --- t19_astreae + t19_host --- t19_asteria + end + end + end + + subgraph accelerator_workers["Accelerators and high-capacity workers"] + direction TB + + subgraph jetson_workers["ARM64 NVIDIA Jetson"] + direction LR + + subgraph titan_20["titan-20"] + direction TB + t20_host["Jetson Xavier NX
16 GiB RAM
Ubuntu 20.04
Accelerator"]:::accelerator + t20_gpu["GPU
NVIDIA Xavier"]:::gpu + t20_storage["Local
256 GiB disk"]:::localStorage + t20_host --- t20_gpu + t20_host --- t20_storage + end + + subgraph titan_21["titan-21"] + direction TB + t21_host["Jetson Xavier NX
16 GiB RAM
Ubuntu 20.04
Accelerator"]:::accelerator + t21_gpu["GPU
NVIDIA Xavier"]:::gpu + t21_storage["Local
256 GiB disk"]:::localStorage + t21_host --- t21_gpu + t21_host --- t21_storage + end + end + + subgraph x86_workers["x86_64 systems"] + direction LR + + subgraph titan_22["titan-22"] + direction TB + t22_host["X830 mini PC
Core i9-12900H
32 GiB RAM
Debian 13"]:::accelerator + t22_gpu["GPU
NVIDIA RTX 3050 Ti"]:::gpu + t22_igpu["Intel Iris Xe
Not pooled"]:::localGpu + t22_storage["Local
1 TiB disk"]:::localStorage + t22_host --- t22_gpu + t22_host --- t22_igpu + t22_host --- t22_storage + end + + subgraph titan_23["titan-23, Oceanus"] + direction TB + t23_host["EPYC 74F3
24 cores, 48 threads
256 GiB RAM
Debian 13"]:::server + t23_gpu["ASPEED graphics
Not pooled"]:::localGpu + t23_root["Local
1 TiB disk"]:::localStorage + t23_data["Local
4 TiB Veles
4 TiB SUI"]:::localStorage + t23_host --- t23_gpu + t23_host --- t23_root + t23_host --- t23_data + end + + subgraph titan_24["titan-24, Tethys"] + direction TB + t24_host["Ryzen 9 3900X
24 threads, 64 GiB RAM
Debian 13"]:::accelerator + t24_gpu["GPU
NVIDIA RTX 3080"]:::gpu + t24_root["Local
500 GiB disk"]:::localStorage + t24_data["Local
500 GiB home
500 GiB temporary"]:::localStorage + t24_host --- t24_gpu + t24_host --- t24_root + t24_host --- t24_data + end + end + end + + pi5_workers ~~~ pi4_workers + pi4_workers ~~~ accelerator_workers + end + + subgraph virtual_resources["Virtual resource pools"] + direction LR + + subgraph cluster_storage["Longhorn storage classes"] + direction LR + + astreae["Astreae
4 x 8 TiB
32 TiB"]:::sharedPool + asteria["Asteria
4 x 12 TiB
48 TiB"]:::sharedPool + end + + subgraph cluster_gpus["Kubernetes GPU pool"] + direction LR + + arm_gpu_pool["NVIDIA ARM64
titan-20 and titan-21"]:::gpuPool + x86_gpu_pool["NVIDIA x86_64
titan-22 and titan-24"]:::gpuPool + end + end + + subgraph cluster_telemetry["Cluster telemetry"] + direction LR + + typhon["Typhon
Kubernetes workload"]:::clusterService + victoria["VictoriaMetrics"]:::telemetry + grafana["Grafana"]:::telemetry + + typhon ==>|environment metrics| victoria + victoria --> grafana + end + end + + subgraph outside_cluster["Lab systems outside the Atlas cluster"] + direction TB + + subgraph support_hosts["Support and host-level services"] + direction LR + + subgraph titan_db["titan-db"] + direction TB + + tdb_host["Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04"]:::support + tdb_storage["Local
500 GiB boot SSD"]:::localStorage + tdb_postgres["PostgreSQL
HA control-plane database"]:::hostSoftware + ananke_db["NUT and Ananke
Host services
Automatic recovery"]:::hostService + + tdb_host --- tdb_storage + tdb_host --- tdb_postgres + tdb_host --- ananke_db + end + + subgraph titan_jh["titan-jh"] + direction TB + + tjh_host["Raspberry Pi 5
8 GiB RAM
Arch Linux ARM
Jump host"]:::support + tjh_storage["Local
256 GiB boot SSD"]:::localStorage + lesavka["Lesavka
Remote control for Tethys"]:::hostService + + tjh_host --- tjh_storage + tjh_host --- lesavka + end + + ananke_t24["NUT and Ananke on titan-24
Host services outside Kubernetes
UPS telemetry and safe shutdown"]:::hostService + end + + subgraph failed_hardware["Boards awaiting replacement"] + direction LR + + titan_09["titan-09
Raspberry Pi 5"]:::failed + titan_10["titan-10
Raspberry Pi 5"]:::failed + titan_16["titan-16
Raspberry Pi 4"]:::failed + end + end + + subgraph facilities["Power and enclosure"] + direction LR + + subgraph protected_power["UPS protection"] + direction LR + + subgraph pyrphoros_power["Pyrphoros"] + direction TB + + pyrphoros["CyberPower
850 VA UPS"]:::power + pyrphoros_loads["Protected loads
Router and Cisco switch
Control plane and Pi workers
Jetsons, titan-22, titan-db
Cooling equipment"]:::powerLoad + pyrphoros --> pyrphoros_loads + end + + subgraph statera_power["Statera"] + direction TB + + statera["CyberPower
1500PFCLCD"]:::power + statera_loads["Protected loads
TRENDnet switch
titan-23, titan-24, titan-jh"]:::powerLoad + statera --> statera_loads + end + end + + subgraph cooling["Enclosure cooling"] + direction TB + + enclosure["Filtered equipment tent
AC-cooled intake
Controlled exhaust"]:::environment + ac_controller["AC Infinity controller
Temperature
Humidity
Pressure"]:::environment + fans["Controlled intake
and exhaust fans"]:::environment + + enclosure --- ac_controller + ac_controller --> fans + end + end + + cisco ==> control_plane + cisco ==> pi5_workers + cisco ==> pi4_workers + cisco ==> jetson_workers + cisco ==> titan_db + trendnet ==> titan_jh + + trendnet ==> x86_workers + + t0b_host -.->|HA PostgreSQL| tdb_postgres + tjh_host -.->|administration| control_plane + tjh_host -.->|administration| workers + lesavka -.->|remote control| t24_host + + t13_astreae ==> astreae + t15_astreae ==> astreae + t17_astreae ==> astreae + t19_astreae ==> astreae + t13_asteria ==> asteria + t15_asteria ==> asteria + t17_asteria ==> asteria + t19_asteria ==> asteria + + t20_gpu ==> arm_gpu_pool + t21_gpu ==> arm_gpu_pool + t22_gpu ==> x86_gpu_pool + t24_gpu ==> x86_gpu_pool + + t24_host -.->|same physical host| ananke_t24 + pyrphoros -->|NUT| ananke_db + statera -->|NUT| ananke_t24 + ananke_db -.->|power metrics| victoria + ananke_t24 -.->|power metrics| victoria + ananke_db -.->|database and cluster recovery| t0a_host + ananke_t24 -.->|low-battery shutdown| control_plane + + ac_controller --> ac_cloud + ac_cloud -.->|environment data| typhon + end + + internet --> router + + classDef external fill:#292d33,stroke:#a0a8b3,color:#ffffff,stroke-width:2px + classDef network fill:#17324a,stroke:#65b5e8,color:#ffffff,stroke-width:2px + classDef control fill:#163e48,stroke:#65d2df,color:#ffffff,stroke-width:2px + classDef pi5 fill:#173d2a,stroke:#68d391,color:#ffffff,stroke-width:2px + classDef pi4 fill:#293b23,stroke:#9acb70,color:#ffffff,stroke-width:2px + classDef accelerator fill:#382b50,stroke:#b49aef,color:#ffffff,stroke-width:2px + classDef server fill:#3c3046,stroke:#c4a4dc,color:#ffffff,stroke-width:2px + classDef support fill:#293845,stroke:#9fc5dc,color:#ffffff,stroke-width:2px + classDef localStorage fill:#1d3040,stroke:#6f9fbd,color:#ffffff + classDef sharedStorage fill:#493b18,stroke:#efc65b,color:#ffffff,stroke-width:2px + classDef sharedPool fill:#594817,stroke:#ffd166,color:#ffffff,stroke-width:3px + classDef gpu fill:#482c5f,stroke:#d0a6ff,color:#ffffff,stroke-width:2px + classDef localGpu fill:#292d35,stroke:#9ba4b2,color:#ffffff + classDef gpuPool fill:#573476,stroke:#d8b4fe,color:#ffffff,stroke-width:3px + classDef clusterService fill:#153c38,stroke:#5fc9bc,color:#ffffff,stroke-width:2px + classDef telemetry fill:#203651,stroke:#67a9e9,color:#ffffff,stroke-width:2px + classDef hostSoftware fill:#243a45,stroke:#73b8d0,color:#ffffff,stroke-width:2px + classDef hostService fill:#48351c,stroke:#e3ae59,color:#ffffff,stroke-width:2px + classDef failed fill:#3d1f25,stroke:#f06b78,color:#ffffff,stroke-width:2px,stroke-dasharray:8 6 + classDef power fill:#49301c,stroke:#ec9f55,color:#ffffff,stroke-width:2px + classDef powerLoad fill:#3b2d20,stroke:#c98e55,color:#ffffff + classDef environment fill:#153d3a,stroke:#58c9bd,color:#ffffff,stroke-width:2px + classDef legend1g fill:#0b1726,stroke:#4ea1ff,color:#ffffff,stroke-width:3px + classDef legend25g fill:#082027,stroke:#22d3ee,color:#ffffff,stroke-width:3px + classDef legendExternal fill:#17162c,stroke:#818cf8,color:#ffffff,stroke-width:3px + classDef legendLocal fill:#171b20,stroke:#a8b3c2,color:#ffffff,stroke-width:3px + classDef legendShared fill:#2f270d,stroke:#f6c453,color:#ffffff,stroke-width:3px + classDef legendGpu fill:#271536,stroke:#c084fc,color:#ffffff,stroke-width:3px + classDef legendPower fill:#301b0c,stroke:#fb923c,color:#ffffff,stroke-width:3px + classDef legendTelemetry fill:#0b2917,stroke:#4ade80,color:#ffffff,stroke-width:3px + classDef legendControl fill:#32132a,stroke:#f472b6,color:#ffffff,stroke-width:3px + classDef legendSafety fill:#321217,stroke:#fb7185,color:#ffffff,stroke-width:3px + + style lab fill:#020202,stroke:#89919c,stroke-width:4px + style link_legend fill:#050505,stroke:#89919c,stroke-width:2px + style legend_network fill:#03070b,stroke:#4ea1ff,stroke-width:1px + style legend_resources fill:#080706,stroke:#f6c453,stroke-width:1px + style legend_operations fill:#080506,stroke:#f472b6,stroke-width:1px + style network fill:#03070b,stroke:#65b5e8,stroke-width:3px + style atlas fill:#020608,stroke:#65d2df,stroke-width:4px + style control_plane fill:#061014,stroke:#65d2df,stroke-width:3px + style workers fill:#030603,stroke:#72b879,stroke-width:3px + style pi5_workers fill:#050d08,stroke:#68d391,stroke-width:2px + style pi5_row_a fill:transparent,stroke:transparent + style pi5_row_b fill:transparent,stroke:transparent + style pi4_workers fill:#080d05,stroke:#9acb70,stroke-width:2px + style pi4_row_a fill:transparent,stroke:transparent + style pi4_row_b fill:transparent,stroke:transparent + style accelerator_workers fill:#09060d,stroke:#b49aef,stroke-width:2px + style jetson_workers fill:#0b0710,stroke:#b49aef,stroke-width:2px + style x86_workers fill:#0b0710,stroke:#c4a4dc,stroke-width:2px + style virtual_resources fill:#08080b,stroke:#a8a2b3,stroke-width:3px + style cluster_storage fill:#100d05,stroke:#ffd166,stroke-width:2px + style cluster_gpus fill:#0d0813,stroke:#d8b4fe,stroke-width:2px + style cluster_telemetry fill:#04100f,stroke:#5fc9bc,stroke-width:2px + style outside_cluster fill:#07090b,stroke:#9fc5dc,stroke-width:3px + style support_hosts fill:#07090b,stroke:#9fc5dc,stroke-width:2px + style failed_hardware fill:#0d0708,stroke:#f06b78,stroke-width:2px,stroke-dasharray:8 6 + style facilities fill:#070604,stroke:#ec9f55,stroke-width:3px + style protected_power fill:#0e0905,stroke:#ec9f55,stroke-width:2px + style pyrphoros_power fill:#100b06,stroke:#d99a58,stroke-width:2px + style statera_power fill:#100b06,stroke:#d99a58,stroke-width:2px + style cooling fill:#04100f,stroke:#58c9bd,stroke-width:2px + + linkStyle 0,54,55,56,57,58,59 stroke:#4ea1ff,color:#4ea1ff,stroke-width:3px + linkStyle 1,60 stroke:#22d3ee,color:#22d3ee,stroke-width:4px + linkStyle 86 stroke:#818cf8,color:#818cf8,stroke-width:3px + linkStyle 2,3,4,7,8,9,10,11,12,13,14,17,18,21,24,25,29,31,34,36,37,39,40,45,48 stroke:#a8b3c2,color:#a8b3c2,stroke-width:2px + linkStyle 15,16,19,20,22,23,26,27,65,66,67,68,69,70,71,72 stroke:#f6c453,color:#f6c453,stroke-width:3px + linkStyle 28,30,32,33,35,38,73,74,75,76 stroke:#c084fc,color:#c084fc,stroke-width:3px + linkStyle 50,51,78,79 stroke:#fb923c,color:#fb923c,stroke-width:3px + linkStyle 43,44,52,53,80,81,84,85 stroke:#4ade80,color:#4ade80,stroke-width:3px + linkStyle 5,6,46,47,49,61,62,63,64,77,82 stroke:#f472b6,color:#f472b6,stroke-width:2px + linkStyle 83 stroke:#fb7185,color:#fb7185,stroke-width:4px diff --git a/mermaid/HardwareArchitecture.svg b/mermaid/HardwareArchitecture.svg new file mode 100644 index 000000000..ddd96e8d6 --- /dev/null +++ b/mermaid/HardwareArchitecture.svg @@ -0,0 +1,67 @@ +

Titan Lab

Power and enclosure

Lab systems outside the Atlas cluster

Atlas Kubernetes cluster

Network fabric

Link colors

Enclosure cooling

UPS protection

Boards awaiting replacement

Support and host-level services

Cluster telemetry

Virtual resource pools

Workers

High-availability control plane, ARM64

Operations

Resources

Network

Statera

Pyrphoros

titan-jh

titan-db

Kubernetes GPU pool

Longhorn storage classes

Accelerators and high-capacity workers

Older workers, ARM64 Raspberry Pi 4

Newer workers, ARM64 Raspberry Pi 5

titan-0c

titan-0b

titan-0a

x86_64 systems

ARM64 NVIDIA Jetson

titan-24, Tethys

titan-23, Oceanus

titan-22

titan-21

titan-20

titan-19

titan-18

titan-17

titan-15

titan-14

titan-13

titan-12

titan-11

titan-08

titan-07

titan-06

titan-05

titan-04

Filtered equipment tent
AC-cooled intake
Controlled exhaust

AC Infinity controller
Temperature
Humidity
Pressure

Controlled intake
and exhaust fans

CyberPower
1500PFCLCD

Protected loads
TRENDnet switch
titan-23, titan-24, titan-jh

CyberPower
850 VA UPS

Protected loads
Router and Cisco switch
Control plane and Pi workers
Jetsons, titan-22, titan-db
Cooling equipment

titan-09
Raspberry Pi 5

titan-10
Raspberry Pi 5

titan-16
Raspberry Pi 4

Raspberry Pi 5
8 GiB RAM
Arch Linux ARM
Jump host

Local
256 GiB boot SSD

Lesavka
Remote control for Tethys

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
500 GiB boot SSD

PostgreSQL
HA control-plane database

NUT and Ananke
Host services
Automatic recovery

NUT and Ananke on titan-24
Host services outside Kubernetes
UPS telemetry and safe shutdown

Typhon
Kubernetes workload

VictoriaMetrics

Grafana

NVIDIA ARM64
titan-20 and titan-21

NVIDIA x86_64
titan-22 and titan-24

Astreae
4 x 8 TiB
32 TiB

Asteria
4 x 12 TiB
48 TiB

Ryzen 9 3900X
24 threads, 64 GiB RAM
Debian 13

GPU
NVIDIA RTX 3080

Local
500 GiB disk

Local
500 GiB home
500 GiB temporary

EPYC 74F3
24 cores, 48 threads
256 GiB RAM
Debian 13

ASPEED graphics
Not pooled

Local
1 TiB disk

Local
4 TiB Veles
4 TiB SUI

X830 mini PC
Core i9-12900H
32 GiB RAM
Debian 13

GPU
NVIDIA RTX 3050 Ti

Intel Iris Xe
Not pooled

Local
1 TiB disk

Jetson Xavier NX
16 GiB RAM
Ubuntu 20.04
Accelerator

GPU
NVIDIA Xavier

Local
256 GiB disk

Jetson Xavier NX
16 GiB RAM
Ubuntu 20.04
Accelerator

GPU
NVIDIA Xavier

Local
256 GiB disk

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Longhorn
8 TiB Astreae

Longhorn
12 TiB Asteria

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Longhorn
8 TiB Astreae

Longhorn
12 TiB Asteria

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Longhorn
8 TiB Astreae

Longhorn
12 TiB Asteria

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Longhorn
8 TiB Astreae

Longhorn
12 TiB Asteria

Raspberry Pi 4
8 GiB RAM
Armbian

Local
SD root

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
SD root
64 GiB astraios

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
SD root
64 GiB astraios

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
SD root
64 GiB astraios

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
SD root
64 GiB astraios

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
SD root
64 GiB astraios

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
SD root
64 GiB astraios

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
500 GiB boot SSD

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
500 GiB boot SSD

Raspberry Pi 5
8 GiB RAM
Ubuntu 24.04

Local
500 GiB boot SSD

ASUS RT-AX88U
Router

Cisco CBS110-24T
1 Gbps switch

TRENDnet 9-port
2.5 Gbps switch

Power and NUT

Telemetry and environment

Administration and recovery

Protective shutdown

Local attachment

Shared storage

GPU

1 Gbps

2.5 Gbps

External network

AC Infinity account

Internet

environment metrics

HA PostgreSQL

administration

administration

remote control

same physical host

NUT

NUT

power metrics

power metrics

database and cluster recovery

low-battery shutdown

environment data

 \ No newline at end of file diff --git a/mermaid/ServiceAccessArchitecture.mmd b/mermaid/ServiceAccessArchitecture.mmd new file mode 100644 index 000000000..af951ee93 --- /dev/null +++ b/mermaid/ServiceAccessArchitecture.mmd @@ -0,0 +1,705 @@ +%% Atlas service access architecture. +%% Domains, identity profiles, entry services, and selected internal dependencies. +%% Node placement and short labels carry the detail. +%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "LINEAR_SEGMENTS", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000", "lineColor": "#8b95a5", "primaryTextColor": "#ffffff", "clusterBkg": "#03040d", "clusterBorder": "#6d7485"}, "themeCSS": "& { background-color: #000000 !important; }"}}%% +flowchart TB + subgraph system["Atlas Service Access Architecture"] + direction TB + + subgraph legend["Legend"] + direction LR + + subgraph node_key["Nodes"] + direction LR + key_domain["DNS name"]:::domain + key_access["OIDC access"]:::access + key_oauth["Access proxy"]:::oauth + key_directory["LDAP or synced account"]:::directory + key_entry["Entry service"]:::user + key_internal["Internal service"]:::support + key_data["Data service"]:::data + key_job["Job or worker"]:::ephemeral + key_external["Outside Kubernetes"]:::external + end + + subgraph link_key["Links"] + direction LR + key_ingress["Blue
Domain to identity"]:::linkIngress + key_service["Green
Identity to service"]:::linkService + key_direct["Gold
Direct access"]:::linkDirect + key_auth["Purple
Identity control"]:::linkAuth + key_data_link["Violet
Data flow"]:::linkData + key_telemetry["Teal
Telemetry"]:::linkTelemetry + key_control["Orange
Delivery and control"]:::linkControl + key_internal_link["Gray
Internal flow"]:::linkInternal + end + end + + subgraph application_band["Applications and Communications"] + direction LR + + subgraph application_domains["Domains"] + direction TB + portal_domain["bstein.dev"]:::domain + portal_chat_domain["chat.ai.bstein.dev"]:::domain + cassandra_domain["cassandra.bstein.dev"]:::domain + element_domain["live.bstein.dev"]:::domain + matrix_domain["matrix.live.bstein.dev"]:::domain + call_domain["call.live.bstein.dev"]:::domain + livekit_domain["kit.live.bstein.dev"]:::domain + turn_domain["turn.live.bstein.dev"]:::domain + end + + subgraph application_access["Keycloak Access"] + direction LR + + subgraph application_atlas_access["atlas realm"] + direction TB + portal_oidc_access["Portal API
OIDC token"]:::access + portal_chat_oidc_access["Portal Chat
OIDC"]:::access + element_oidc_access["Element Web
OIDC through MAS"]:::access + matrix_oidc_access["Matrix
OIDC through MAS"]:::access + end + + subgraph application_cassandra_access["cassandra realm"] + direction TB + cassandra_oidc_access["Cassandra
OIDC"]:::access + end + end + + subgraph application_services["Cluster Services"] + direction TB + + application_delivery["Flux delivery
Application workloads"]:::control + application_state_clients["Shared PostgreSQL
Portal and Matrix"]:::control + + subgraph website_ns["bstein-dev-home"] + direction LR + website_frontend["Portal"]:::user + website_chat_gateway["Chat gateway"]:::user + website_backend["Backend"]:::support + + website_frontend --> website_backend + website_chat_gateway --> website_backend + end + + subgraph cassandra_ns["cassandra"] + direction LR + cassandra_frontend["Cassandra UI"]:::user + cassandra_backend["API"]:::support + cassandra_postgres["PostgreSQL"]:::data + cassandra_generator["Generator"]:::ephemeral + cassandra_simulation["Simulation jobs"]:::ephemeral + cassandra_retention["Retention job"]:::ephemeral + cassandra_artifacts["Artifacts"]:::data + + cassandra_frontend --> cassandra_backend + cassandra_backend --> cassandra_postgres + cassandra_generator --> cassandra_backend + cassandra_simulation --> cassandra_backend + cassandra_simulation --> cassandra_artifacts + cassandra_retention --> cassandra_artifacts + end + + subgraph comms_ns["comms"] + direction LR + comms_element_web["Element Web"]:::user + comms_element_call["Element Call"]:::user + comms_wellknown["Matrix discovery"]:::support + comms_guest["Guest registration"]:::support + comms_mas["Matrix auth service"]:::support + comms_synapse["Synapse"]:::support + comms_replication["Synapse replication"]:::support + comms_redis["Redis"]:::data + comms_livekit_token["LiveKit token service"]:::support + comms_livekit["LiveKit"]:::support + comms_coturn["Coturn"]:::support + comms_atlasbot["Atlasbot"]:::support + + comms_element_web --> comms_mas + comms_element_web --> comms_synapse + comms_element_call --> comms_livekit_token + comms_livekit_token --> comms_livekit + comms_mas --> comms_synapse + comms_synapse --> comms_replication + comms_synapse --> comms_redis + comms_guest --> comms_synapse + comms_atlasbot --> comms_synapse + end + end + + portal_domain portal_identity@-->|HTTPS| portal_oidc_access + portal_oidc_access portal_entry@--> website_frontend + portal_chat_domain portal_chat_identity@-->|HTTPS| portal_chat_oidc_access + portal_chat_oidc_access portal_chat_entry@--> website_chat_gateway + cassandra_domain cassandra_identity@-->|HTTPS| cassandra_oidc_access + cassandra_oidc_access cassandra_entry@--> cassandra_frontend + element_domain element_identity@-->|HTTPS| element_oidc_access + element_oidc_access element_entry@--> comms_element_web + matrix_domain matrix_identity@-->|HTTPS| matrix_oidc_access + matrix_oidc_access matrix_entry@--> comms_synapse + matrix_domain matrix_api@-->|Matrix API| comms_synapse + call_domain call_entry@-->|HTTPS| comms_element_call + livekit_domain livekit_entry@-->|HTTPS and WebRTC| comms_livekit + turn_domain turn_entry@-->|TURN| comms_coturn + end + + subgraph personal_band["Personal Services"] + direction LR + + subgraph personal_domains["Domains"] + direction LR + + subgraph personal_domains_a["Accounts and Health"] + direction TB + budget_domain["budget.bstein.dev"]:::domain + firefly_domain["money.bstein.dev"]:::domain + wolf_domain["wolf.bstein.dev"]:::domain + moonlight_domain["moonlight.bstein.dev"]:::domain + wger_domain["health.bstein.dev"]:::domain + chat_domain["chat.bstein.dev"]:::domain + end + + subgraph personal_domains_b["Media and Productivity"] + direction TB + jellyfin_domain["stream.bstein.dev"]:::domain + pegasus_domain["pegasus.bstein.dev"]:::domain + mailu_domain["mail.bstein.dev"]:::domain + cloud_domain["cloud.bstein.dev"]:::domain + office_domain["office.bstein.dev"]:::domain + outline_domain["notes.bstein.dev"]:::domain + planka_domain["tasks.bstein.dev"]:::domain + vaultwarden_domain["vault.bstein.dev"]:::domain + monero_domain["monero.bstein.dev"]:::domain + end + end + + subgraph personal_access["atlas realm and directory access"] + direction LR + + subgraph personal_access_a["Accounts"] + direction TB + budget_oidc_access["Actual Budget
OpenID"]:::access + firefly_synced_access["Firefly
Synced account"]:::directory + wolf_oauth_access["Wolf
Access proxy"]:::oauth + wger_synced_access["Wger
Synced account"]:::directory + chat_oidc_access["Hermes Chat
OIDC"]:::access + end + + subgraph personal_access_b["Media and Productivity"] + direction TB + jellyfin_ldap_access["Jellyfin
LDAP"]:::directory + cloud_oidc_access["Nextcloud
OIDC"]:::access + outline_oidc_access["Outline
OIDC"]:::access + planka_oidc_access["Planka
OIDC"]:::access + end + end + + subgraph personal_services["Cluster Services"] + direction LR + + personal_delivery["Flux delivery
Personal workloads"]:::control + personal_state_clients["Shared PostgreSQL
Personal services"]:::control + + subgraph personal_services_a["Accounts, Health, and Media"] + direction TB + + subgraph finance_ns["finance"] + direction LR + finance_budget["Actual Budget"]:::user + finance_firefly["Firefly"]:::user + finance_sync["Account sync"]:::ephemeral + finance_import["Import job"]:::ephemeral + finance_sync --> finance_firefly + finance_import --> finance_firefly + end + + subgraph game_ns["game-stream"] + direction LR + game_proxy["Wolf access proxy"]:::oauth + game_manager["Wolf manager"]:::support + game_api["Wolf API"]:::support + game_wolf["Wolf"]:::support + game_gatekeeper["Gatekeeper"]:::support + game_moonlight["Moonlight"]:::user + game_proxy --> game_manager + game_manager --> game_api + game_api --> game_wolf + game_gatekeeper --> game_wolf + game_moonlight --> game_wolf + end + + subgraph health_ns["health"] + direction LR + health_wger["Wger"]:::user + health_sync["Account sync"]:::ephemeral + health_admin["Admin sync"]:::ephemeral + health_sync --> health_wger + health_admin --> health_wger + end + + subgraph hermes_chat_ns["hermes-chat"] + direction LR + hermes_chat["Hermes Chat"]:::user + end + + subgraph jellyfin_ns["jellyfin"] + direction LR + streaming_pegasus["Pegasus"]:::user + streaming_jellyfin["Jellyfin"]:::user + streaming_pegasus --> streaming_jellyfin + end + end + + subgraph personal_services_b["Mail and Productivity"] + direction TB + + subgraph mail_ns["mailu-mailserver"] + direction LR + mail_front["Mail web"]:::user + mail_protocols["Mail protocols"]:::user + mail_admin["Admin"]:::support + mail_dovecot["Dovecot"]:::support + mail_postfix["Postfix"]:::support + mail_rspamd["Rspamd"]:::support + mail_clamav["ClamAV"]:::support + mail_extractors["Tika and oletools"]:::support + mail_redis["Redis"]:::data + mail_front --> mail_admin + mail_protocols --> mail_dovecot + mail_protocols --> mail_postfix + mail_postfix --> mail_rspamd + mail_rspamd --> mail_clamav + mail_rspamd --> mail_extractors + mail_admin --> mail_redis + end + + subgraph cloud_ns["nextcloud"] + direction LR + cloud_nextcloud["Nextcloud"]:::user + cloud_collabora["Collabora"]:::support + cloud_mail_sync["Mail sync"]:::ephemeral + cloud_maintenance["Maintenance job"]:::ephemeral + cloud_nextcloud --> cloud_collabora + cloud_mail_sync --> cloud_nextcloud + cloud_maintenance --> cloud_nextcloud + end + + subgraph notes_tasks_ns["Notes and Tasks"] + direction LR + outline_service["Outline"]:::user + outline_redis["Redis"]:::data + planka_service["Planka"]:::user + outline_service --> outline_redis + end + + subgraph vaultwarden_ns["vaultwarden"] + direction LR + vaultwarden_service["Vaultwarden"]:::user + end + + subgraph crypto_ns["crypto"] + direction LR + crypto_wallet["Wallet RPC"]:::support + crypto_p2pool["P2Pool"]:::support + crypto_monerod["Monerod"]:::user + crypto_wallet --> crypto_monerod + crypto_p2pool --> crypto_monerod + end + end + end + + budget_domain budget_identity@-->|HTTPS| budget_oidc_access + budget_oidc_access budget_entry@--> finance_budget + firefly_domain firefly_identity@-->|HTTPS| firefly_synced_access + firefly_synced_access firefly_entry@--> finance_firefly + wolf_domain wolf_identity@-->|HTTPS| wolf_oauth_access + wolf_oauth_access wolf_entry@--> game_manager + moonlight_domain moonlight_entry@-->|Streaming| game_moonlight + wger_domain wger_identity@-->|HTTPS| wger_synced_access + wger_synced_access wger_entry@--> health_wger + chat_domain chat_identity@-->|HTTPS| chat_oidc_access + chat_oidc_access chat_entry@--> hermes_chat + jellyfin_domain jellyfin_identity@-->|HTTPS| jellyfin_ldap_access + jellyfin_ldap_access jellyfin_entry@--> streaming_jellyfin + pegasus_domain pegasus_entry@-->|HTTPS| streaming_pegasus + mailu_domain mail_web_entry@-->|HTTPS| mail_front + mailu_domain mail_protocol_entry@-->|Mail protocols| mail_protocols + cloud_domain cloud_identity@-->|HTTPS| cloud_oidc_access + cloud_oidc_access cloud_entry@--> cloud_nextcloud + office_domain office_entry@-->|WOPI| cloud_collabora + outline_domain outline_identity@-->|HTTPS| outline_oidc_access + outline_oidc_access outline_entry@--> outline_service + planka_domain planka_identity@-->|HTTPS| planka_oidc_access + planka_oidc_access planka_entry@--> planka_service + vaultwarden_domain vaultwarden_entry@-->|HTTPS and local auth| vaultwarden_service + monero_domain monero_entry@-->|RPC| crypto_monerod + end + + subgraph platform_band["Platform Services"] + direction LR + + subgraph platform_domains["Domains"] + direction LR + + subgraph platform_domains_a["Delivery"] + direction TB + gitops_domain["cd.bstein.dev"]:::domain + gitea_domain["scm.bstein.dev"]:::domain + harbor_domain["registry.bstein.dev"]:::domain + hermes_domain["agent.bstein.dev"]:::domain + jenkins_domain["ci.bstein.dev"]:::domain + end + + subgraph platform_domains_b["Operations"] + direction TB + logs_domain["logs.bstein.dev"]:::domain + longhorn_domain["longhorn.bstein.dev"]:::domain + grafana_domain["metrics.bstein.dev"]:::domain + alerts_domain["alerts.bstein.dev"]:::domain + quality_domain["quality.bstein.dev"]:::domain + keycloak_domain["sso.bstein.dev"]:::domain + vault_domain["secret.bstein.dev"]:::domain + end + end + + subgraph platform_access["Keycloak Access"] + direction LR + + subgraph platform_access_a["atlas realm delivery"] + direction TB + gitops_oidc_access["Weave GitOps
OIDC"]:::access + gitea_atlas_oidc_access["Gitea
atlas realm"]:::access + harbor_oidc_access["Harbor UI
OIDC"]:::access + hermes_oidc_access["Hermes Agent
OIDC"]:::access + jenkins_oidc_access["Jenkins
OIDC"]:::access + end + + subgraph platform_access_b["atlas realm operations"] + direction TB + logs_oauth_access["OpenSearch Dashboards
Access proxy"]:::oauth + longhorn_oauth_access["Longhorn
Admin proxy"]:::oauth + grafana_oidc_access["Grafana
OIDC"]:::access + quality_oauth_access["SonarQube
Admin and developer proxy"]:::oauth + keycloak_admin_access["Keycloak console
Realm admin"]:::access + vault_oidc_access["Vault
Admin OIDC"]:::access + end + + subgraph platform_access_c["cassandra realm client"] + direction TB + gitea_cassandra_oidc_access["Gitea
OIDC"]:::access + end + end + + subgraph platform_services["Cluster Services"] + direction LR + + platform_delivery["Flux delivery
Platform workloads"]:::control + platform_state_clients["Shared PostgreSQL
Platform services"]:::control + + subgraph delivery_services["Delivery and Automation"] + direction TB + + subgraph flux_ns["flux-system"] + direction LR + flux_source["Source controller"]:::support + flux_kustomize["Kustomize controller"]:::support + flux_helm["Helm controller"]:::support + flux_image_reflector["Image reflector"]:::support + flux_image_automation["Image automation"]:::support + flux_notification["Notification controller"]:::support + flux_webhook["Webhook receiver"]:::support + flux_weave["Weave GitOps"]:::user + flux_source --> flux_kustomize + flux_source --> flux_helm + flux_image_reflector --> flux_image_automation + flux_webhook --> flux_notification + flux_weave --> flux_source + end + + subgraph gitea_ns["gitea"] + direction LR + gitea_http["Gitea web"]:::user + gitea_ssh["Gitea SSH"]:::support + end + + subgraph harbor_ns["harbor"] + direction LR + harbor_portal["Harbor portal"]:::user + harbor_core["Harbor core"]:::support + harbor_jobservice["Job service"]:::support + harbor_registry["OCI registry"]:::data + harbor_redis["Redis"]:::data + harbor_portal --> harbor_core + harbor_jobservice --> harbor_core + harbor_core --> harbor_registry + harbor_core --> harbor_redis + end + + subgraph hermes_ns["hermes"] + direction LR + hermes_agent["Hermes Agent"]:::user + hermes_model_gate["Model gate"]:::support + hermes_gpt_oss["gpt-oss:20b
Fallback model"]:::support + hermes_agent --> hermes_model_gate + hermes_model_gate --> hermes_gpt_oss + end + + subgraph jenkins_ns["jenkins"] + direction LR + jenkins_service["Jenkins"]:::user + jenkins_agents["Build agents"]:::ephemeral + jenkins_service --> jenkins_agents + end + end + + subgraph operations_services["Operations"] + direction TB + + subgraph logging_ns["logging"] + direction LR + logging_oauth_proxy["Dashboard proxy"]:::oauth + logging_dashboards["OpenSearch Dashboards"]:::user + logging_opensearch["OpenSearch"]:::data + logging_fluent_bit["Fluent Bit"]:::support + logging_otel["OpenTelemetry"]:::support + logging_data_prepper["Data Prepper"]:::support + logging_oauth_proxy --> logging_dashboards + logging_dashboards --> logging_opensearch + logging_fluent_bit --> logging_opensearch + logging_otel --> logging_data_prepper + logging_data_prepper --> logging_opensearch + end + + subgraph longhorn_ns["longhorn-system"] + direction LR + longhorn_oauth_proxy["Longhorn proxy"]:::oauth + longhorn_frontend["Longhorn UI"]:::user + longhorn_backend["Longhorn manager"]:::data + longhorn_webhooks["Admission and recovery"]:::support + longhorn_oauth_proxy --> longhorn_frontend + longhorn_frontend --> longhorn_backend + longhorn_webhooks --> longhorn_backend + end + + subgraph monitoring_ns["monitoring"] + direction LR + monitoring_grafana["Grafana"]:::user + monitoring_victoria["VictoriaMetrics"]:::data + monitoring_exporters["Cluster and node exporters"]:::support + monitoring_gpu_exporters["GPU exporters"]:::support + monitoring_quality_gateway["Quality gateway"]:::support + monitoring_postmark["Postmark exporter"]:::support + monitoring_vmalert["vmalert"]:::support + monitoring_alertmanager["Alertmanager"]:::user + monitoring_exporters --> monitoring_victoria + monitoring_gpu_exporters --> monitoring_victoria + monitoring_quality_gateway --> monitoring_victoria + monitoring_postmark --> monitoring_victoria + monitoring_victoria --> monitoring_grafana + monitoring_vmalert --> monitoring_alertmanager + end + end + + subgraph security_services["Identity and Security"] + direction TB + + subgraph quality_ns["quality"] + direction LR + quality_oauth_proxy["SonarQube proxy"]:::oauth + quality_sonarqube["SonarQube"]:::user + quality_exporter["Quality exporter"]:::support + quality_oauth_proxy --> quality_sonarqube + quality_exporter --> quality_sonarqube + end + + subgraph sso_ns["sso"] + direction LR + sso_oauth_proxy["Shared access proxy"]:::oauth + sso_keycloak["Keycloak"]:::user + atlas_realm["atlas realm"]:::control + cassandra_realm["cassandra realm"]:::control + sso_openldap["OpenLDAP"]:::directory + sso_jobs["Realm bootstrap and checks"]:::ephemeral + sso_keycloak --> atlas_realm + sso_keycloak --> cassandra_realm + sso_jobs --> sso_keycloak + end + + subgraph vault_ns["vault"] + direction LR + vault_service["Vault"]:::user + vault_internal["Vault cluster service"]:::support + vault_injector["Secret injector"]:::support + vault_jobs["Configuration jobs"]:::ephemeral + vault_service --> vault_internal + vault_injector --> vault_service + vault_jobs --> vault_service + end + end + end + + gitops_domain gitops_identity@-->|HTTPS| gitops_oidc_access + gitops_oidc_access gitops_entry@--> flux_weave + gitea_domain gitea_atlas_identity@-->|HTTPS| gitea_atlas_oidc_access + gitea_atlas_oidc_access gitea_atlas_entry@--> gitea_http + gitea_domain gitea_cassandra_identity@-->|HTTPS| gitea_cassandra_oidc_access + gitea_cassandra_oidc_access gitea_cassandra_entry@--> gitea_http + gitea_domain gitea_ssh_entry@-->|SSH| gitea_ssh + harbor_domain harbor_identity@-->|HTTPS| harbor_oidc_access + harbor_oidc_access harbor_entry@--> harbor_portal + harbor_domain harbor_registry_entry@-->|OCI| harbor_registry + hermes_domain hermes_identity@-->|HTTPS| hermes_oidc_access + hermes_oidc_access hermes_entry@--> hermes_agent + jenkins_domain jenkins_identity@-->|HTTPS| jenkins_oidc_access + jenkins_oidc_access jenkins_entry@--> jenkins_service + logs_domain logs_identity@-->|HTTPS| logs_oauth_access + logs_oauth_access logs_entry@--> logging_oauth_proxy + longhorn_domain longhorn_identity@-->|HTTPS| longhorn_oauth_access + longhorn_oauth_access longhorn_entry@--> longhorn_oauth_proxy + grafana_domain grafana_identity@-->|HTTPS| grafana_oidc_access + grafana_oidc_access grafana_entry@--> monitoring_grafana + alerts_domain alerts_entry@-->|HTTPS| monitoring_alertmanager + quality_domain quality_identity@-->|HTTPS| quality_oauth_access + quality_oauth_access quality_entry@--> quality_oauth_proxy + keycloak_domain keycloak_identity@-->|HTTPS| keycloak_admin_access + keycloak_admin_access keycloak_entry@--> sso_keycloak + vault_domain vault_identity@-->|HTTPS| vault_oidc_access + vault_oidc_access vault_entry@--> vault_service + end + + subgraph shared_plane["Shared Cluster Services"] + direction LR + + subgraph ingress_plane["Ingress and Networking"] + direction TB + metallb_control["MetalLB
Service addresses"]:::control + traefik_service["Traefik
Load balancer"]:::support + traefik_controller["Traefik
TLS and routing"]:::support + cert_manager["cert-manager"]:::support + cert_webhooks["Certificate webhooks"]:::support + metallb_control --> traefik_service + traefik_service --> traefik_controller + cert_webhooks --> cert_manager + cert_manager --> traefik_controller + end + + subgraph shared_state["Shared State and Models"] + direction TB + postgres_service["Shared PostgreSQL
Application and platform databases"]:::data + ai_ollama["Ollama
Local model service"]:::support + default_zot_proxy["Zot access proxy"]:::oauth + end + + subgraph maintenance_ns["maintenance"] + direction TB + maintenance_ariadne["Ariadne"]:::support + maintenance_metis_proxy["Metis proxy"]:::oauth + maintenance_metis["Metis"]:::support + maintenance_soteria_proxy["Soteria proxy"]:::oauth + maintenance_soteria["Soteria"]:::support + maintenance_node_ops["Node operations"]:::ephemeral + maintenance_jobs["Repair and migration jobs"]:::ephemeral + maintenance_metis_proxy --> maintenance_metis + maintenance_soteria_proxy --> maintenance_soteria + maintenance_ariadne --> maintenance_node_ops + maintenance_ariadne --> maintenance_jobs + end + + subgraph telemetry_services["Additional Telemetry"] + direction TB + climate_typhon["Typhon
Tent climate"]:::support + sui_metrics["SUI metrics"]:::support + end + end + + subgraph external_systems["Host and Physical Systems"] + direction LR + ananke_hosts["Ananke
titan-db and titan-24"]:::external + pyrphoros["Pyrphoros UPS"]:::external + statera["Statera UPS"]:::external + tent_system["AC Infinity
Controller and cloud"]:::external + ananke_hosts --> pyrphoros + ananke_hosts --> statera + end + + firefly_synced_access firefly_directory@-.-> sso_openldap + wger_synced_access wger_directory@-.-> sso_openldap + jellyfin_ldap_access jellyfin_directory@-.-> sso_openldap + + website_backend website_model@-.-> ai_ollama + website_backend website_automation@-.-> maintenance_ariadne + hermes_chat hermes_chat_model@-.-> hermes_model_gate + hermes_agent hermes_jenkins@-.-> jenkins_service + hermes_agent hermes_metrics@-.-> monitoring_victoria + hermes_agent hermes_ariadne@-.-> maintenance_ariadne + + gitea_http gitea_jenkins@-.->|SCM webhook| jenkins_service + gitea_http gitea_flux@-.->|Flux webhook| flux_webhook + jenkins_agents jenkins_gitea@-.->|Clone and fetch| gitea_http + jenkins_agents jenkins_quality@-.->|Quality scan| quality_sonarqube + jenkins_agents jenkins_harbor@-.->|Images and artifacts| harbor_registry + flux_source flux_gitea@-.->|Desired state| gitea_ssh + flux_image_reflector flux_harbor@-.->|Image metadata| harbor_registry + flux_image_automation flux_commit@-.->|Manifest update| gitea_ssh + flux_kustomize flux_apps@-.-> application_delivery + flux_kustomize flux_personal@-.-> personal_delivery + flux_kustomize flux_platform@-.-> platform_delivery + + application_state_clients app_postgres@-.-> postgres_service + personal_state_clients personal_postgres@-.-> postgres_service + platform_state_clients platform_postgres@-.-> postgres_service + cloud_nextcloud nextcloud_mail@-.->|Mail delivery| mail_protocols + + quality_exporter quality_metrics@-.-> monitoring_victoria + sui_metrics sui_telemetry@-.-> monitoring_victoria + monitoring_victoria ananke_telemetry@-.-> ananke_hosts + climate_typhon climate_source@--> tent_system + climate_typhon climate_telemetry@-.-> monitoring_victoria + + class portal_identity,portal_chat_identity,cassandra_identity,element_identity,matrix_identity,budget_identity,firefly_identity,wolf_identity,wger_identity,chat_identity,jellyfin_identity,cloud_identity,outline_identity,planka_identity,gitops_identity,gitea_atlas_identity,gitea_cassandra_identity,harbor_identity,hermes_identity,jenkins_identity,logs_identity,longhorn_identity,grafana_identity,quality_identity,keycloak_identity,vault_identity routeIngress + class portal_entry,portal_chat_entry,cassandra_entry,element_entry,matrix_entry,budget_entry,firefly_entry,wolf_entry,wger_entry,chat_entry,jellyfin_entry,cloud_entry,outline_entry,planka_entry,gitops_entry,gitea_atlas_entry,gitea_cassandra_entry,harbor_entry,hermes_entry,jenkins_entry,logs_entry,longhorn_entry,grafana_entry,quality_entry,keycloak_entry,vault_entry routeService + class matrix_api,call_entry,livekit_entry,turn_entry,moonlight_entry,pegasus_entry,mail_web_entry,mail_protocol_entry,office_entry,vaultwarden_entry,monero_entry,gitea_ssh_entry,harbor_registry_entry,alerts_entry routeDirect + class firefly_directory,wger_directory,jellyfin_directory routeAuth + class app_postgres,personal_postgres,platform_postgres,nextcloud_mail,climate_source routeData + class quality_metrics,sui_telemetry,ananke_telemetry,climate_telemetry routeTelemetry + class website_automation,hermes_ariadne,gitea_jenkins,gitea_flux,jenkins_gitea,jenkins_quality,jenkins_harbor,flux_gitea,flux_harbor,flux_commit,flux_apps,flux_personal,flux_platform routeControl + class website_model,hermes_chat_model,hermes_jenkins,hermes_metrics routeInternal + end + + classDef domain fill:#16324f,stroke:#6cb6ff,color:#ffffff + classDef user fill:#173f2b,stroke:#61d095,color:#ffffff + classDef support fill:#30343b,stroke:#aab2bf,color:#ffffff + classDef data fill:#3b2f52,stroke:#b79cff,color:#ffffff + classDef ephemeral fill:#4a301e,stroke:#f0a35b,color:#ffffff,stroke-dasharray:6 3 + classDef control fill:#3b354e,stroke:#c1a7ff,color:#ffffff + classDef access fill:#213a56,stroke:#6cb6ff,color:#ffffff + classDef oauth fill:#3b2f52,stroke:#c1a7ff,color:#ffffff + classDef directory fill:#344a2d,stroke:#8ed081,color:#ffffff + classDef external fill:#44301f,stroke:#ffb86c,color:#ffffff + + classDef linkIngress fill:#101820,stroke:#4ea1ff,color:#ffffff + classDef linkService fill:#101820,stroke:#61d095,color:#ffffff + classDef linkDirect fill:#101820,stroke:#e4bd55,color:#ffffff + classDef linkAuth fill:#101820,stroke:#c1a7ff,color:#ffffff + classDef linkData fill:#101820,stroke:#b79cff,color:#ffffff + classDef linkTelemetry fill:#101820,stroke:#55c8d3,color:#ffffff + classDef linkControl fill:#101820,stroke:#f0a35b,color:#ffffff + classDef linkInternal fill:#101820,stroke:#aab2bf,color:#ffffff + + classDef routeIngress stroke:#4ea1ff,stroke-width:2.5px,color:#4ea1ff + classDef routeService stroke:#61d095,stroke-width:2.5px,color:#61d095 + classDef routeDirect stroke:#e4bd55,stroke-width:2.5px,color:#e4bd55 + classDef routeAuth stroke:#c1a7ff,stroke-width:2.2px,color:#c1a7ff,stroke-dasharray:6 4 + classDef routeData stroke:#b79cff,stroke-width:2.2px,color:#b79cff,stroke-dasharray:6 4 + classDef routeTelemetry stroke:#55c8d3,stroke-width:2.2px,color:#55c8d3,stroke-dasharray:6 4 + classDef routeControl stroke:#f0a35b,stroke-width:2.2px,color:#f0a35b,stroke-dasharray:6 4 + classDef routeInternal stroke:#aab2bf,stroke-width:2px,color:#aab2bf,stroke-dasharray:5 4 + + style system fill:#000000,stroke:#8b95a5,stroke-width:2px + style legend fill:#050505,stroke:#6d7485,stroke-width:1.5px + style application_band fill:#02030d,stroke:#55c8d3,stroke-width:2px + style personal_band fill:#06030d,stroke:#8ed081,stroke-width:2px + style platform_band fill:#07040c,stroke:#c1a7ff,stroke-width:2px + style shared_plane fill:#080604,stroke:#e4bd55,stroke-width:2px + style external_systems fill:#080402,stroke:#ffb86c,stroke-width:2px diff --git a/mermaid/ServiceAccessArchitecture.svg b/mermaid/ServiceAccessArchitecture.svg new file mode 100644 index 000000000..59c1f622c --- /dev/null +++ b/mermaid/ServiceAccessArchitecture.svg @@ -0,0 +1,67 @@ +

Atlas Service Access Architecture

Host and Physical Systems

Shared Cluster Services

Platform Services

Personal Services

Applications and Communications

Legend

Additional Telemetry

maintenance

Shared State and Models

Ingress and Networking

Cluster Services

Keycloak Access

Domains

Cluster Services

atlas realm and directory access

Domains

Cluster Services

Keycloak Access

Domains

Links

Nodes

Identity and Security

Operations

Delivery and Automation

cassandra realm client

atlas realm operations

atlas realm delivery

Operations

Delivery

Mail and Productivity

Accounts, Health, and Media

Media and Productivity

Accounts

Media and Productivity

Accounts and Health

comms

cassandra

bstein-dev-home

cassandra realm

atlas realm

vault

sso

quality

monitoring

longhorn-system

logging

jenkins

hermes

harbor

gitea

flux-system

crypto

vaultwarden

Notes and Tasks

nextcloud

mailu-mailserver

jellyfin

hermes-chat

health

game-stream

finance

Ananke
titan-db and titan-24

Pyrphoros UPS

Statera UPS

AC Infinity
Controller and cloud

Typhon
Tent climate

SUI metrics

Ariadne

Metis proxy

Metis

Soteria proxy

Soteria

Node operations

Repair and migration jobs

Shared PostgreSQL
Application and platform databases

Ollama
Local model service

Zot access proxy

MetalLB
Service addresses

Traefik
Load balancer

Traefik
TLS and routing

cert-manager

Certificate webhooks

Vault

Vault cluster service

Secret injector

Configuration jobs

Shared access proxy

Keycloak

atlas realm

cassandra realm

OpenLDAP

Realm bootstrap and checks

SonarQube proxy

SonarQube

Quality exporter

Grafana

VictoriaMetrics

Cluster and node exporters

GPU exporters

Quality gateway

Postmark exporter

vmalert

Alertmanager

Longhorn proxy

Longhorn UI

Longhorn manager

Admission and recovery

Dashboard proxy

OpenSearch Dashboards

OpenSearch

Fluent Bit

OpenTelemetry

Data Prepper

Jenkins

Build agents

Hermes Agent

Model gate

gpt-oss:20b
Fallback model

Harbor portal

Harbor core

Job service

OCI registry

Redis

Gitea web

Gitea SSH

Source controller

Kustomize controller

Helm controller

Image reflector

Image automation

Notification controller

Webhook receiver

Weave GitOps

Flux delivery
Platform workloads

Shared PostgreSQL
Platform services

Gitea
OIDC

OpenSearch Dashboards
Access proxy

Longhorn
Admin proxy

Grafana
OIDC

SonarQube
Admin and developer proxy

Keycloak console
Realm admin

Vault
Admin OIDC

Weave GitOps
OIDC

Gitea
atlas realm

Harbor UI
OIDC

Hermes Agent
OIDC

Jenkins
OIDC

logs.bstein.dev

longhorn.bstein.dev

metrics.bstein.dev

alerts.bstein.dev

quality.bstein.dev

sso.bstein.dev

secret.bstein.dev

cd.bstein.dev

scm.bstein.dev

registry.bstein.dev

agent.bstein.dev

ci.bstein.dev

Wallet RPC

P2Pool

Monerod

Vaultwarden

Outline

Redis

Planka

Nextcloud

Collabora

Mail sync

Maintenance job

Mail web

Mail protocols

Admin

Dovecot

Postfix

Rspamd

ClamAV

Tika and oletools

Redis

Pegasus

Jellyfin

Hermes Chat

Wger

Account sync

Admin sync

Wolf access proxy

Wolf manager

Wolf API

Wolf

Gatekeeper

Moonlight

Actual Budget

Firefly

Account sync

Import job

Flux delivery
Personal workloads

Shared PostgreSQL
Personal services

Jellyfin
LDAP

Nextcloud
OIDC

Outline
OIDC

Planka
OIDC

Actual Budget
OpenID

Firefly
Synced account

Wolf
Access proxy

Wger
Synced account

Hermes Chat
OIDC

stream.bstein.dev

pegasus.bstein.dev

mail.bstein.dev

cloud.bstein.dev

office.bstein.dev

notes.bstein.dev

tasks.bstein.dev

vault.bstein.dev

monero.bstein.dev

budget.bstein.dev

money.bstein.dev

wolf.bstein.dev

moonlight.bstein.dev

health.bstein.dev

chat.bstein.dev

Element Web

Element Call

Matrix discovery

Guest registration

Matrix auth service

Synapse

Synapse replication

Redis

LiveKit token service

LiveKit

Coturn

Atlasbot

Cassandra UI

API

PostgreSQL

Generator

Simulation jobs

Retention job

Artifacts

Portal

Chat gateway

Backend

Flux delivery
Application workloads

Shared PostgreSQL
Portal and Matrix

Cassandra
OIDC

Portal API
OIDC token

Portal Chat
OIDC

Element Web
OIDC through MAS

Matrix
OIDC through MAS

bstein.dev

chat.ai.bstein.dev

cassandra.bstein.dev

live.bstein.dev

matrix.live.bstein.dev

call.live.bstein.dev

kit.live.bstein.dev

turn.live.bstein.dev

Blue
Domain to identity

Green
Identity to service

Gold
Direct access

Purple
Identity control

Violet
Data flow

Teal
Telemetry

Orange
Delivery and control

Gray
Internal flow

DNS name

OIDC access

Access proxy

LDAP or synced account

Entry service

Internal service

Data service

Job or worker

Outside Kubernetes

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

Matrix API

HTTPS

HTTPS and WebRTC

TURN

HTTPS

HTTPS

HTTPS

Streaming

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

Mail protocols

HTTPS

WOPI

HTTPS

HTTPS

HTTPS and local auth

RPC

HTTPS

HTTPS

HTTPS

SSH

HTTPS

OCI

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

HTTPS

SCM webhook

Flux webhook

Clone and fetch

Quality scan

Images and artifacts

Desired state

Image metadata

Manifest update

Mail delivery

 \ No newline at end of file diff --git a/mermaid/TestAutomation.mmd b/mermaid/TestAutomation.mmd new file mode 100644 index 000000000..2d6feaf2d --- /dev/null +++ b/mermaid/TestAutomation.mmd @@ -0,0 +1,213 @@ +%% Automatic test failure triage and response. +%% Hermes reads and recommends. Ariadne validates and performs every write. +%% Relationships carry the detail; node text stays concise. +%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 32, "rankSpacing": 48, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "LINEAR_SEGMENTS", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%% +flowchart TD + + subgraph system["Hermes Test Failure Triage"] + direction TB + + subgraph main_row["Main flow"] + direction TB + + subgraph upper_flow["Evidence and analysis"] + direction LR + + subgraph reference["Legend"] + direction TB + + key_build["Blue
Build and validation"]:::build + key_evidence["Green
Evidence"]:::evidence + key_ariadne["Teal
Ariadne orchestration"]:::ariadne + key_hermes["Purple
Hermes"]:::hermes + key_policy["Gold
Policy and action"]:::policy + key_human["Red
Human required"]:::human + key_limit["Dashed gray
Current boundary"]:::limited + end + + subgraph intake["Detect and gather"] + direction TB + + detector["Jenkins detector
Allowlisted jobs
Terminal failures only"]:::ariadne + evidence_sources["Evidence sources
Jenkins and OpenSearch
Metrics, Kubernetes, Flux
Gitea and Grafana"]:::evidence + collector["Console reader
Full console
2 MB cap"]:::ariadne + failure_ranker["Failure ranker
Strong evidence first
Tool output alone is neutral"]:::ariadne + context_filter["Context filter
Passing quotes removed
Repeats and overlaps merged"]:::ariadne + bundle["Incident bundle
Failures, logs, metrics
Health, revisions, links"]:::evidence + + detector --> evidence_sources + evidence_sources --> collector + collector --> failure_ranker + failure_ranker --> context_filter + context_filter --> bundle + end + + subgraph hermes_plane["Hermes analysis"] + direction TB + + hermes_access["Read-only access
Jenkins, metrics, Kubernetes
Flux, Gitea, Grafana
Investigation by incident ID"]:::hermes + model_gate["Model gate
openai-codex/gpt-5.6-terra
local gpt-oss:20b fallback"]:::model + skills["Triage skills
Jenkins evidence
Quality and cluster health
Flux and Git correlation"]:::skill + recommendation["Structured recommendation
Classification and confidence
Sourced facts
Action request or patch data"]:::hermes + tool_stream["Agent event stream
Evidence and tool calls"]:::audit + + hermes_access --> skills + model_gate -.-> skills + skills --> recommendation + hermes_access -.-> tool_stream + end + end + + subgraph lower_flow["Policy and response"] + direction LR + + subgraph controls["Ariadne policy gates"] + direction TB + + response_check["Response check
Schema, incident, sources
Confidence and action"]:::policy + repair_guard["Scoped repair guard
Classification and action
must match the owning job"]:::policy + marker_check["Transient marker check
DNS, connection, TLS
image pull, upstream 5xx
Disk full is excluded"]:::policy + authorizer["Action authorizer
Allowlist, evidence, confidence
One action, kill switch"]:::policy + + response_check -->|scoped repair| repair_guard + repair_guard --> authorizer + response_check -->|transient claim| marker_check + marker_check -->|marker confirmed| authorizer + response_check -->|other result| authorizer + end + + subgraph response["Ariadne response"] + direction LR + + subgraph response_paths["Response paths"] + direction TB + + route{"Policy result"}:::decision + + subgraph immediate_paths["Action or escalation"] + direction LR + + subgraph operational_path["Allowlisted action"] + direction TB + + registry["Action registry"]:::policy + repair_fixture["Scoped ConfigMap repair
In-process patch
No pod created"]:::action + retry_infra["Transient retry
One Jenkins rebuild
No cluster write"]:::action + operational_result["Action result
One attempt
One validation build"]:::action + + registry --> repair_fixture + registry --> retry_infra + repair_fixture --> operational_result + retry_infra --> operational_result + end + + subgraph escalation_path["Human-required response"] + direction TB + + human_required["Diagnosis and next checks
Incident stays human required"]:::human + gitea_issue["Ariadne opens an issue
Evidence and suggested fix"]:::human + + human_required --> gitea_issue + end + end + + route -->|authorized action| registry + route -->|human required| human_required + operational_result -->|action failed| human_required + end + + subgraph code_path["Optional source proposal"] + direction TB + + subgraph proposal_stage["Prepare and check"] + direction TB + + duplicate_guard["Open proposal check
Existing repair branch"]:::policy + candidate_files["Candidate files
Path and line hints
Read failing test"]:::evidence + source_target["Patch target
Follow test imports
Tests remain read only"]:::policy + patch_proposal["Hermes patch data
Path, anchor, replacement
Reason"]:::hermes + patch_validator["Patch validator
Allowed path and branch
20 lines, 4 KB
Exact anchor"]:::policy + + duplicate_guard --> candidate_files + candidate_files --> source_target + source_target --> patch_proposal + patch_proposal --> patch_validator + end + + subgraph pr_stage["Deliver for review"] + direction TB + + gitea_pr["Ariadne opens a pull request
hermes-repair branch
Ariadne holds the token"]:::git + branch_build["Branch validation
Configured repositories only"]:::limited + human_merge["Human review and merge
No automatic merge"]:::human + flux_delivery["Flux delivery
After human merge"]:::git + + gitea_pr -.->|where configured| branch_build + branch_build --> human_merge + gitea_pr -->|otherwise| human_merge + human_merge --> flux_delivery + end + + patch_validator -->|validated patch| gitea_pr + end + + response_paths -.->|eligible source defect| code_path + end + + subgraph outputs["Inspectable outputs"] + direction TB + + output_entry["Ariadne records every outcome"]:::audit + audit_events["Audit events
Incident, diagnosis, action
Patch proposal and agent IDs"]:::audit + metrics_output["Triage metrics
VictoriaMetrics and Grafana"]:::metrics + alert_output["Narrow alerts
Failed repair
Escalation untouched for six hours"]:::metrics + issue_output["Gitea issue
Human-required incident
Written by Ariadne"]:::git + pr_output["Gitea pull request
Source proposal
Written by Ariadne"]:::git + + output_entry --> audit_events + output_entry --> metrics_output + metrics_output --> alert_output + end + end + + reference ~~~ intake + intake ==>|incident bundle| hermes_plane + controls ==>|policy result| response + response ==>|records and artifacts| outputs + upper_flow ==>|recommendation| lower_flow + end + + end + + classDef build fill:#16324f,stroke:#6cb6ff,color:#ffffff + classDef evidence fill:#173f2b,stroke:#61d095,color:#ffffff + classDef ariadne fill:#24414a,stroke:#69c5d1,color:#ffffff + classDef hermes fill:#3c3155,stroke:#c1a7ff,color:#ffffff + classDef skill fill:#42345b,stroke:#c1a7ff,color:#ffffff + classDef model fill:#342b4f,stroke:#a78bfa,color:#ffffff + classDef policy fill:#3d3525,stroke:#e4bd55,color:#ffffff + classDef decision fill:#47391c,stroke:#e4bd55,color:#ffffff,stroke-width:3px + classDef action fill:#4a3b16,stroke:#e4bd55,color:#ffffff + classDef human fill:#4a2428,stroke:#ff6b78,color:#ffffff + classDef git fill:#253f35,stroke:#78c89a,color:#ffffff + classDef metrics fill:#243750,stroke:#65a9e8,color:#ffffff + classDef audit fill:#353125,stroke:#d9b55b,color:#ffffff + classDef limited fill:#1f2227,stroke:#9ba4b1,color:#c4c9d1,stroke-width:2px,stroke-dasharray:8 6 + + style system fill:#030303,stroke:#6b7280,stroke-width:3px + style main_row fill:#050607,stroke:#6b7280,stroke-width:2px + style upper_flow fill:#050607,stroke:#61d095,stroke-width:2px + style lower_flow fill:#070605,stroke:#e4bd55,stroke-width:2px + style reference fill:#07090c,stroke:#9ba4b1,stroke-width:2px + style intake fill:#070d09,stroke:#61d095,stroke-width:2px + style hermes_plane fill:#0c0912,stroke:#c1a7ff,stroke-width:3px + style controls fill:#0d0b07,stroke:#e4bd55,stroke-width:2px + style response fill:#090806,stroke:#e4bd55,stroke-width:3px + style response_paths fill:#070707,stroke:#6b7280,stroke-width:1px + style immediate_paths fill:#080706,stroke:#8b8170,stroke-width:1px + style operational_path fill:#0d0b07,stroke:#e4bd55,stroke-width:2px + style escalation_path fill:#0d090a,stroke:#ff6b78,stroke-width:2px + style code_path fill:#0b090d,stroke:#c1a7ff,stroke-width:2px + style proposal_stage fill:#09080b,stroke:#9f86da,stroke-width:1px + style pr_stage fill:#07100a,stroke:#78c89a,stroke-width:1px + style outputs fill:#07100d,stroke:#78c89a,stroke-width:2px diff --git a/mermaid/TestAutomation.svg b/mermaid/TestAutomation.svg new file mode 100644 index 000000000..c091556cd --- /dev/null +++ b/mermaid/TestAutomation.svg @@ -0,0 +1,67 @@ +

Hermes Test Failure Triage

Main flow

Policy and response

Evidence and analysis

Inspectable outputs

Ariadne response

Ariadne policy gates

Hermes analysis

Detect and gather

Legend

Optional source proposal

Response paths

Deliver for review

Prepare and check

Action or escalation

Human-required response

Allowlisted action

Ariadne records every outcome

Audit events
Incident, diagnosis, action
Patch proposal and agent IDs

Triage metrics
VictoriaMetrics and Grafana

Narrow alerts
Failed repair
Escalation untouched for six hours

Gitea issue
Human-required incident
Written by Ariadne

Gitea pull request
Source proposal
Written by Ariadne

Ariadne opens a pull request
hermes-repair branch
Ariadne holds the token

Branch validation
Configured repositories only

Human review and merge
No automatic merge

Flux delivery
After human merge

Open proposal check
Existing repair branch

Candidate files
Path and line hints
Read failing test

Patch target
Follow test imports
Tests remain read only

Hermes patch data
Path, anchor, replacement
Reason

Patch validator
Allowed path and branch
20 lines, 4 KB
Exact anchor

Diagnosis and next checks
Incident stays human required

Ariadne opens an issue
Evidence and suggested fix

Action registry

Scoped ConfigMap repair
In-process patch
No pod created

Transient retry
One Jenkins rebuild
No cluster write

Action result
One attempt
One validation build

Policy result

Response check
Schema, incident, sources
Confidence and action

Scoped repair guard
Classification and action
must match the owning job

Transient marker check
DNS, connection, TLS
image pull, upstream 5xx
Disk full is excluded

Action authorizer
Allowlist, evidence, confidence
One action, kill switch

Read-only access
Jenkins, metrics, Kubernetes
Flux, Gitea, Grafana
Investigation by incident ID

Model gate
openai-codex/gpt-5.6-terra
local gpt-oss:20b fallback

Triage skills
Jenkins evidence
Quality and cluster health
Flux and Git correlation

Structured recommendation
Classification and confidence
Sourced facts
Action request or patch data

Agent event stream
Evidence and tool calls

Jenkins detector
Allowlisted jobs
Terminal failures only

Evidence sources
Jenkins and OpenSearch
Metrics, Kubernetes, Flux
Gitea and Grafana

Console reader
Full console
2 MB cap

Failure ranker
Strong evidence first
Tool output alone is neutral

Context filter
Passing quotes removed
Repeats and overlaps merged

Incident bundle
Failures, logs, metrics
Health, revisions, links

Blue
Build and validation

Green
Evidence

Teal
Ariadne orchestration

Purple
Hermes

Gold
Policy and action

Red
Human required

Dashed gray
Current boundary

scoped repair

transient claim

marker confirmed

other result

authorized action

human required

action failed

where configured

otherwise

validated patch

eligible source defect

incident bundle

policy result

records and artifacts

recommendation

 \ No newline at end of file