From 1ec3ca29a43a13ab9a64b1e6f8639081bc0f1c82 Mon Sep 17 00:00:00 2001
From: Brad Stein <Brad.Stein@gmail.com>
Date: Sat, 13 Dec 2025 19:04:22 -0300
Subject: [PATCH] Use OR-joined node ranks for plurality tie-break

---
 scripts/dashboards_render_atlas.py              | 7 ++++---
 services/monitoring/dashboards/atlas-pods.json  | 2 +-
 services/monitoring/grafana-dashboard-pods.yaml | 2 +-
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/scripts/dashboards_render_atlas.py b/scripts/dashboards_render_atlas.py
index 55ab9f2..237be34 100644
--- a/scripts/dashboards_render_atlas.py
+++ b/scripts/dashboards_render_atlas.py
@@ -1197,10 +1197,11 @@ def build_pods_dashboard():
         '(sum by (namespace,node) (kube_pod_info{pod!="" , node!=""}) '
         '/ on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=""}), 1) * 100)'
     )
-    rank_expr = " + ".join(
-        f"(sum by (node) (kube_node_info{{node=\"{node}\"}}) * 1e-3 * {idx})"
+    rank_terms = [
+        f"(sum by (node) (kube_node_info{{node=\"{node}\"}}) * 0 + {idx * 1e-3})"
         for idx, node in enumerate(CONTROL_ALL + WORKER_NODES, start=1)
-    )
+    ]
+    rank_expr = " or ".join(rank_terms)
     score_expr = f"{share_expr} + on(node) group_left() ({rank_expr})"
     mask_expr = (
         f"{score_expr} == bool on(namespace) group_left() "
diff --git a/services/monitoring/dashboards/atlas-pods.json b/services/monitoring/dashboards/atlas-pods.json
index ce6dea7..1684580 100644
--- a/services/monitoring/dashboards/atlas-pods.json
+++ b/services/monitoring/dashboards/atlas-pods.json
@@ -520,7 +520,7 @@
       },
       "targets": [
         {
-          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)))))",
+          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)))))",
           "refId": "A",
           "instant": true,
           "format": "table"
diff --git a/services/monitoring/grafana-dashboard-pods.yaml b/services/monitoring/grafana-dashboard-pods.yaml
index 8122866..d7cc0c1 100644
--- a/services/monitoring/grafana-dashboard-pods.yaml
+++ b/services/monitoring/grafana-dashboard-pods.yaml
@@ -529,7 +529,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)))))",
+              "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)))))",
               "refId": "A",
               "instant": true,
               "format": "table"