%% Titan Lab physical hardware architecture.
%% Solid links are physical network, storage, power, or sensor paths.
%% Dashed links are administration, telemetry, or recovery control.
%%{init: {"flowchart": {"defaultRenderer": "elk", "curve": "stepAfter", "nodeSpacing": 30, "rankSpacing": 65, "useMaxWidth": false}, "elk": {"mergeEdges": true, "nodePlacementStrategy": "NETWORK_SIMPLEX", "forceNodeModelOrder": true, "considerModelOrder": "NODES_AND_EDGES"}, "themeVariables": {"background": "#000000"}, "themeCSS": "& { background-color: #000000 !important; }"}}%%
flowchart TB
    internet["Internet"]:::external
    ac_cloud["AC Infinity account"]:::external

    subgraph lab["Titan Lab"]
        direction TB

        subgraph link_legend["Link colors"]
            direction LR

            subgraph legend_network["Network"]
                direction TB
                legend_1g["1 Gbps"]:::legend1g
                legend_25g["2.5 Gbps"]:::legend25g
                legend_external["External network"]:::legendExternal
            end

            subgraph legend_resources["Resources"]
                direction TB
                legend_local["Local attachment"]:::legendLocal
                legend_shared["Shared storage"]:::legendShared
                legend_gpu["GPU"]:::legendGpu
            end

            subgraph legend_operations["Operations"]
                direction TB
                legend_power["Power and NUT"]:::legendPower
                legend_telemetry["Telemetry and environment"]:::legendTelemetry
                legend_control["Administration and recovery"]:::legendControl
                legend_safety["Protective shutdown"]:::legendSafety
            end
        end

        subgraph network["Network fabric"]
            direction LR

            router["ASUS RT-AX88U<br/>Router"]:::network
            cisco["Cisco CBS110-24T<br/>1 Gbps switch"]:::network
            trendnet["TRENDnet 9-port<br/>2.5 Gbps switch"]:::network

            router ==> cisco
            router ==> trendnet
        end

        subgraph atlas["Atlas Kubernetes cluster"]
            direction TB

            subgraph control_plane["High-availability control plane, ARM64"]
                direction LR

                subgraph titan_0a["titan-0a"]
                    direction TB
                    t0a_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
                    t0a_storage["Local<br/>500 GiB boot SSD"]:::localStorage
                    t0a_host --- t0a_storage
                end

                subgraph titan_0b["titan-0b"]
                    direction TB
                    t0b_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
                    t0b_storage["Local<br/>500 GiB boot SSD"]:::localStorage
                    t0b_host --- t0b_storage
                end

                subgraph titan_0c["titan-0c"]
                    direction TB
                    t0c_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::control
                    t0c_storage["Local<br/>500 GiB boot SSD"]:::localStorage
                    t0c_host --- t0c_storage
                end

                t0a_host --- t0b_host
                t0b_host --- t0c_host
            end

            subgraph workers["Workers"]
                direction TB

                subgraph pi5_workers["Newer workers, ARM64 Raspberry Pi 5"]
                    direction TB

                    subgraph pi5_row_a[" "]
                        direction LR

                        subgraph titan_04["titan-04"]
                            direction TB
                            t04_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
                            t04_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
                            t04_host --- t04_storage
                        end

                        subgraph titan_05["titan-05"]
                            direction TB
                            t05_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
                            t05_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
                            t05_host --- t05_storage
                        end

                        subgraph titan_06["titan-06"]
                            direction TB
                            t06_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
                            t06_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
                            t06_host --- t06_storage
                        end
                    end

                    subgraph pi5_row_b[" "]
                        direction LR

                        subgraph titan_07["titan-07"]
                            direction TB
                            t07_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
                            t07_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
                            t07_host --- t07_storage
                        end

                        subgraph titan_08["titan-08"]
                            direction TB
                            t08_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
                            t08_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
                            t08_host --- t08_storage
                        end

                        subgraph titan_11["titan-11"]
                            direction TB
                            t11_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::pi5
                            t11_storage["Local<br/>SD root<br/>64 GiB astraios"]:::localStorage
                            t11_host --- t11_storage
                        end
                    end
                end

                subgraph pi4_workers["Older workers, ARM64 Raspberry Pi 4"]
                    direction TB

                    subgraph pi4_row_a[" "]
                        direction LR

                        subgraph titan_12["titan-12"]
                            direction TB
                            t12_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t12_root["Local<br/>SD root"]:::localStorage
                            t12_host --- t12_root
                        end

                        subgraph titan_13["titan-13"]
                            direction TB
                            t13_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t13_root["Local<br/>SD root"]:::localStorage
                            t13_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
                            t13_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
                            t13_host --- t13_root
                            t13_host --- t13_astreae
                            t13_host --- t13_asteria
                        end

                        subgraph titan_14["titan-14"]
                            direction TB
                            t14_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t14_root["Local<br/>SD root"]:::localStorage
                            t14_host --- t14_root
                        end

                        subgraph titan_15["titan-15"]
                            direction TB
                            t15_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t15_root["Local<br/>SD root"]:::localStorage
                            t15_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
                            t15_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
                            t15_host --- t15_root
                            t15_host --- t15_astreae
                            t15_host --- t15_asteria
                        end
                    end

                    subgraph pi4_row_b[" "]
                        direction LR

                        subgraph titan_17["titan-17"]
                            direction TB
                            t17_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t17_root["Local<br/>SD root"]:::localStorage
                            t17_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
                            t17_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
                            t17_host --- t17_root
                            t17_host --- t17_astreae
                            t17_host --- t17_asteria
                        end

                        subgraph titan_18["titan-18"]
                            direction TB
                            t18_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t18_root["Local<br/>SD root"]:::localStorage
                            t18_host --- t18_root
                        end

                        subgraph titan_19["titan-19"]
                            direction TB
                            t19_host["Raspberry Pi 4<br/>8 GiB RAM<br/>Armbian"]:::pi4
                            t19_root["Local<br/>SD root"]:::localStorage
                            t19_astreae["Longhorn<br/>8 TiB Astreae"]:::sharedStorage
                            t19_asteria["Longhorn<br/>12 TiB Asteria"]:::sharedStorage
                            t19_host --- t19_root
                            t19_host --- t19_astreae
                            t19_host --- t19_asteria
                        end
                    end
                end

                subgraph accelerator_workers["Accelerators and high-capacity workers"]
                    direction TB

                    subgraph jetson_workers["ARM64 NVIDIA Jetson"]
                        direction LR

                        subgraph titan_20["titan-20"]
                            direction TB
                            t20_host["Jetson Xavier NX<br/>16 GiB RAM<br/>Ubuntu 20.04<br/>Accelerator"]:::accelerator
                            t20_gpu["GPU<br/>NVIDIA Xavier"]:::gpu
                            t20_storage["Local<br/>256 GiB disk"]:::localStorage
                            t20_host --- t20_gpu
                            t20_host --- t20_storage
                        end

                        subgraph titan_21["titan-21"]
                            direction TB
                            t21_host["Jetson Xavier NX<br/>16 GiB RAM<br/>Ubuntu 20.04<br/>Accelerator"]:::accelerator
                            t21_gpu["GPU<br/>NVIDIA Xavier"]:::gpu
                            t21_storage["Local<br/>256 GiB disk"]:::localStorage
                            t21_host --- t21_gpu
                            t21_host --- t21_storage
                        end
                    end

                    subgraph x86_workers["x86_64 systems"]
                        direction LR

                        subgraph titan_22["titan-22"]
                            direction TB
                            t22_host["X830 mini PC<br/>Core i9-12900H<br/>32 GiB RAM<br/>Debian 13"]:::accelerator
                            t22_gpu["GPU<br/>NVIDIA RTX 3050 Ti"]:::gpu
                            t22_igpu["Intel Iris Xe<br/>Not pooled"]:::localGpu
                            t22_storage["Local<br/>1 TiB disk"]:::localStorage
                            t22_host --- t22_gpu
                            t22_host --- t22_igpu
                            t22_host --- t22_storage
                        end

                        subgraph titan_23["titan-23, Oceanus"]
                            direction TB
                            t23_host["EPYC 74F3<br/>24 cores, 48 threads<br/>256 GiB RAM<br/>Debian 13"]:::server
                            t23_gpu["ASPEED graphics<br/>Not pooled"]:::localGpu
                            t23_root["Local<br/>1 TiB disk"]:::localStorage
                            t23_data["Local<br/>4 TiB Veles<br/>4 TiB SUI"]:::localStorage
                            t23_host --- t23_gpu
                            t23_host --- t23_root
                            t23_host --- t23_data
                        end

                        subgraph titan_24["titan-24, Tethys"]
                            direction TB
                            t24_host["Ryzen 9 3900X<br/>24 threads, 64 GiB RAM<br/>Debian 13"]:::accelerator
                            t24_gpu["GPU<br/>NVIDIA RTX 3080"]:::gpu
                            t24_root["Local<br/>500 GiB disk"]:::localStorage
                            t24_data["Local<br/>500 GiB home<br/>500 GiB temporary"]:::localStorage
                            t24_host --- t24_gpu
                            t24_host --- t24_root
                            t24_host --- t24_data
                        end
                    end
                end

                pi5_workers ~~~ pi4_workers
                pi4_workers ~~~ accelerator_workers
            end

            subgraph virtual_resources["Virtual resource pools"]
                direction LR

                subgraph cluster_storage["Longhorn storage classes"]
                    direction LR

                    astreae["Astreae<br/>4 x 8 TiB<br/>32 TiB"]:::sharedPool
                    asteria["Asteria<br/>4 x 12 TiB<br/>48 TiB"]:::sharedPool
                end

                subgraph cluster_gpus["Kubernetes GPU pool"]
                    direction LR

                    arm_gpu_pool["NVIDIA ARM64<br/>titan-20 and titan-21"]:::gpuPool
                    x86_gpu_pool["NVIDIA x86_64<br/>titan-22 and titan-24"]:::gpuPool
                end
            end

            subgraph cluster_telemetry["Cluster telemetry"]
                direction LR

                typhon["Typhon<br/>Kubernetes workload"]:::clusterService
                victoria["VictoriaMetrics"]:::telemetry
                grafana["Grafana"]:::telemetry

                typhon ==>|environment metrics| victoria
                victoria --> grafana
            end
        end

        subgraph outside_cluster["Lab systems outside the Atlas cluster"]
            direction TB

            subgraph support_hosts["Support and host-level services"]
                direction LR

                subgraph titan_db["titan-db"]
                    direction TB

                    tdb_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Ubuntu 24.04"]:::support
                    tdb_storage["Local<br/>500 GiB boot SSD"]:::localStorage
                    tdb_postgres["PostgreSQL<br/>HA control-plane database"]:::hostSoftware
                    ananke_db["NUT and Ananke<br/>Host services<br/>Automatic recovery"]:::hostService

                    tdb_host --- tdb_storage
                    tdb_host --- tdb_postgres
                    tdb_host --- ananke_db
                end

                subgraph titan_jh["titan-jh"]
                    direction TB

                    tjh_host["Raspberry Pi 5<br/>8 GiB RAM<br/>Arch Linux ARM<br/>Jump host"]:::support
                    tjh_storage["Local<br/>256 GiB boot SSD"]:::localStorage
                    lesavka["Lesavka<br/>Remote control for Tethys"]:::hostService

                    tjh_host --- tjh_storage
                    tjh_host --- lesavka
                end

                ananke_t24["NUT and Ananke on titan-24<br/>Host services outside Kubernetes<br/>UPS telemetry and safe shutdown"]:::hostService
            end

            subgraph failed_hardware["Boards awaiting replacement"]
                direction LR

                titan_09["titan-09<br/>Raspberry Pi 5"]:::failed
                titan_10["titan-10<br/>Raspberry Pi 5"]:::failed
                titan_16["titan-16<br/>Raspberry Pi 4"]:::failed
            end
        end

        subgraph facilities["Power and enclosure"]
            direction LR

            subgraph protected_power["UPS protection"]
                direction LR

                subgraph pyrphoros_power["Pyrphoros"]
                    direction TB

                    pyrphoros["CyberPower<br/>850 VA UPS"]:::power
                    pyrphoros_loads["Protected loads<br/>Router and Cisco switch<br/>Control plane and Pi workers<br/>Jetsons, titan-22, titan-db<br/>Cooling equipment"]:::powerLoad
                    pyrphoros --> pyrphoros_loads
                end

                subgraph statera_power["Statera"]
                    direction TB

                    statera["CyberPower<br/>1500PFCLCD"]:::power
                    statera_loads["Protected loads<br/>TRENDnet switch<br/>titan-23, titan-24, titan-jh"]:::powerLoad
                    statera --> statera_loads
                end
            end

            subgraph cooling["Enclosure cooling"]
                direction TB

                enclosure["Filtered equipment tent<br/>AC-cooled intake<br/>Controlled exhaust"]:::environment
                ac_controller["AC Infinity controller<br/>Temperature<br/>Humidity<br/>Pressure"]:::environment
                fans["Controlled intake<br/>and exhaust fans"]:::environment

                enclosure --- ac_controller
                ac_controller --> fans
            end
        end

        cisco ==> control_plane
        cisco ==> pi5_workers
        cisco ==> pi4_workers
        cisco ==> jetson_workers
        cisco ==> titan_db
        trendnet ==> titan_jh

        trendnet ==> x86_workers

        t0b_host -.->|HA PostgreSQL| tdb_postgres
        tjh_host -.->|administration| control_plane
        tjh_host -.->|administration| workers
        lesavka -.->|remote control| t24_host

        t13_astreae ==> astreae
        t15_astreae ==> astreae
        t17_astreae ==> astreae
        t19_astreae ==> astreae
        t13_asteria ==> asteria
        t15_asteria ==> asteria
        t17_asteria ==> asteria
        t19_asteria ==> asteria

        t20_gpu ==> arm_gpu_pool
        t21_gpu ==> arm_gpu_pool
        t22_gpu ==> x86_gpu_pool
        t24_gpu ==> x86_gpu_pool

        t24_host -.->|same physical host| ananke_t24
        pyrphoros -->|NUT| ananke_db
        statera -->|NUT| ananke_t24
        ananke_db -.->|power metrics| victoria
        ananke_t24 -.->|power metrics| victoria
        ananke_db -.->|database and cluster recovery| t0a_host
        ananke_t24 -.->|low-battery shutdown| control_plane

        ac_controller --> ac_cloud
        ac_cloud -.->|environment data| typhon
    end

    internet --> router

    classDef external fill:#292d33,stroke:#a0a8b3,color:#ffffff,stroke-width:2px
    classDef network fill:#17324a,stroke:#65b5e8,color:#ffffff,stroke-width:2px
    classDef control fill:#163e48,stroke:#65d2df,color:#ffffff,stroke-width:2px
    classDef pi5 fill:#173d2a,stroke:#68d391,color:#ffffff,stroke-width:2px
    classDef pi4 fill:#293b23,stroke:#9acb70,color:#ffffff,stroke-width:2px
    classDef accelerator fill:#382b50,stroke:#b49aef,color:#ffffff,stroke-width:2px
    classDef server fill:#3c3046,stroke:#c4a4dc,color:#ffffff,stroke-width:2px
    classDef support fill:#293845,stroke:#9fc5dc,color:#ffffff,stroke-width:2px
    classDef localStorage fill:#1d3040,stroke:#6f9fbd,color:#ffffff
    classDef sharedStorage fill:#493b18,stroke:#efc65b,color:#ffffff,stroke-width:2px
    classDef sharedPool fill:#594817,stroke:#ffd166,color:#ffffff,stroke-width:3px
    classDef gpu fill:#482c5f,stroke:#d0a6ff,color:#ffffff,stroke-width:2px
    classDef localGpu fill:#292d35,stroke:#9ba4b2,color:#ffffff
    classDef gpuPool fill:#573476,stroke:#d8b4fe,color:#ffffff,stroke-width:3px
    classDef clusterService fill:#153c38,stroke:#5fc9bc,color:#ffffff,stroke-width:2px
    classDef telemetry fill:#203651,stroke:#67a9e9,color:#ffffff,stroke-width:2px
    classDef hostSoftware fill:#243a45,stroke:#73b8d0,color:#ffffff,stroke-width:2px
    classDef hostService fill:#48351c,stroke:#e3ae59,color:#ffffff,stroke-width:2px
    classDef failed fill:#3d1f25,stroke:#f06b78,color:#ffffff,stroke-width:2px,stroke-dasharray:8 6
    classDef power fill:#49301c,stroke:#ec9f55,color:#ffffff,stroke-width:2px
    classDef powerLoad fill:#3b2d20,stroke:#c98e55,color:#ffffff
    classDef environment fill:#153d3a,stroke:#58c9bd,color:#ffffff,stroke-width:2px
    classDef legend1g fill:#0b1726,stroke:#4ea1ff,color:#ffffff,stroke-width:3px
    classDef legend25g fill:#082027,stroke:#22d3ee,color:#ffffff,stroke-width:3px
    classDef legendExternal fill:#17162c,stroke:#818cf8,color:#ffffff,stroke-width:3px
    classDef legendLocal fill:#171b20,stroke:#a8b3c2,color:#ffffff,stroke-width:3px
    classDef legendShared fill:#2f270d,stroke:#f6c453,color:#ffffff,stroke-width:3px
    classDef legendGpu fill:#271536,stroke:#c084fc,color:#ffffff,stroke-width:3px
    classDef legendPower fill:#301b0c,stroke:#fb923c,color:#ffffff,stroke-width:3px
    classDef legendTelemetry fill:#0b2917,stroke:#4ade80,color:#ffffff,stroke-width:3px
    classDef legendControl fill:#32132a,stroke:#f472b6,color:#ffffff,stroke-width:3px
    classDef legendSafety fill:#321217,stroke:#fb7185,color:#ffffff,stroke-width:3px

    style lab fill:#020202,stroke:#89919c,stroke-width:4px
    style link_legend fill:#050505,stroke:#89919c,stroke-width:2px
    style legend_network fill:#03070b,stroke:#4ea1ff,stroke-width:1px
    style legend_resources fill:#080706,stroke:#f6c453,stroke-width:1px
    style legend_operations fill:#080506,stroke:#f472b6,stroke-width:1px
    style network fill:#03070b,stroke:#65b5e8,stroke-width:3px
    style atlas fill:#020608,stroke:#65d2df,stroke-width:4px
    style control_plane fill:#061014,stroke:#65d2df,stroke-width:3px
    style workers fill:#030603,stroke:#72b879,stroke-width:3px
    style pi5_workers fill:#050d08,stroke:#68d391,stroke-width:2px
    style pi5_row_a fill:transparent,stroke:transparent
    style pi5_row_b fill:transparent,stroke:transparent
    style pi4_workers fill:#080d05,stroke:#9acb70,stroke-width:2px
    style pi4_row_a fill:transparent,stroke:transparent
    style pi4_row_b fill:transparent,stroke:transparent
    style accelerator_workers fill:#09060d,stroke:#b49aef,stroke-width:2px
    style jetson_workers fill:#0b0710,stroke:#b49aef,stroke-width:2px
    style x86_workers fill:#0b0710,stroke:#c4a4dc,stroke-width:2px
    style virtual_resources fill:#08080b,stroke:#a8a2b3,stroke-width:3px
    style cluster_storage fill:#100d05,stroke:#ffd166,stroke-width:2px
    style cluster_gpus fill:#0d0813,stroke:#d8b4fe,stroke-width:2px
    style cluster_telemetry fill:#04100f,stroke:#5fc9bc,stroke-width:2px
    style outside_cluster fill:#07090b,stroke:#9fc5dc,stroke-width:3px
    style support_hosts fill:#07090b,stroke:#9fc5dc,stroke-width:2px
    style failed_hardware fill:#0d0708,stroke:#f06b78,stroke-width:2px,stroke-dasharray:8 6
    style facilities fill:#070604,stroke:#ec9f55,stroke-width:3px
    style protected_power fill:#0e0905,stroke:#ec9f55,stroke-width:2px
    style pyrphoros_power fill:#100b06,stroke:#d99a58,stroke-width:2px
    style statera_power fill:#100b06,stroke:#d99a58,stroke-width:2px
    style cooling fill:#04100f,stroke:#58c9bd,stroke-width:2px

    linkStyle 0,54,55,56,57,58,59 stroke:#4ea1ff,color:#4ea1ff,stroke-width:3px
    linkStyle 1,60 stroke:#22d3ee,color:#22d3ee,stroke-width:4px
    linkStyle 86 stroke:#818cf8,color:#818cf8,stroke-width:3px
    linkStyle 2,3,4,7,8,9,10,11,12,13,14,17,18,21,24,25,29,31,34,36,37,39,40,45,48 stroke:#a8b3c2,color:#a8b3c2,stroke-width:2px
    linkStyle 15,16,19,20,22,23,26,27,65,66,67,68,69,70,71,72 stroke:#f6c453,color:#f6c453,stroke-width:3px
    linkStyle 28,30,32,33,35,38,73,74,75,76 stroke:#c084fc,color:#c084fc,stroke-width:3px
    linkStyle 50,51,78,79 stroke:#fb923c,color:#fb923c,stroke-width:3px
    linkStyle 43,44,52,53,80,81,84,85 stroke:#4ade80,color:#4ade80,stroke-width:3px
    linkStyle 5,6,46,47,49,61,62,63,64,77,82 stroke:#f472b6,color:#f472b6,stroke-width:2px
    linkStyle 83 stroke:#fb7185,color:#fb7185,stroke-width:4px