Use OR-joined node ranks for plurality tie-break

scripts/dashboards_render_atlas.py

@@ -1197,10 +1197,11 @@ def build_pods_dashboard():
         '(sum by (namespace,node) (kube_pod_info{pod!="" , node!=""}) '
         '/ on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=""}), 1) * 100)'
     )
-    rank_expr = " + ".join(
+    rank_terms = [
-        f"(sum by (node) (kube_node_info{{node=\"{node}\"}}) * 1e-3 * {idx})"
+        f"(sum by (node) (kube_node_info{{node=\"{node}\"}}) * 0 + {idx * 1e-3})"
         for idx, node in enumerate(CONTROL_ALL + WORKER_NODES, start=1)
-    )
+    ]
+    rank_expr = " or ".join(rank_terms)
     score_expr = f"{share_expr} + on(node) group_left() ({rank_expr})"
     mask_expr = (
         f"{score_expr} == bool on(namespace) group_left() "

services/monitoring/dashboards/atlas-pods.json

@@ -520,7 +520,7 @@
       },
       "targets": [
         {
-          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)))))",
+          "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)))))",
           "refId": "A",
           "instant": true,
           "format": "table"

services/monitoring/grafana-dashboard-pods.yaml

@@ -529,7 +529,7 @@ data:
           },
           "targets": [
             {
-              "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 1e-3 * 1) + (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 1e-3 * 2) + (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 1e-3 * 3) + (sum by (node) (kube_node_info{node=\"titan-db\"}) * 1e-3 * 4) + (sum by (node) (kube_node_info{node=\"titan-04\"}) * 1e-3 * 5) + (sum by (node) (kube_node_info{node=\"titan-05\"}) * 1e-3 * 6) + (sum by (node) (kube_node_info{node=\"titan-06\"}) * 1e-3 * 7) + (sum by (node) (kube_node_info{node=\"titan-07\"}) * 1e-3 * 8) + (sum by (node) (kube_node_info{node=\"titan-08\"}) * 1e-3 * 9) + (sum by (node) (kube_node_info{node=\"titan-09\"}) * 1e-3 * 10) + (sum by (node) (kube_node_info{node=\"titan-10\"}) * 1e-3 * 11) + (sum by (node) (kube_node_info{node=\"titan-11\"}) * 1e-3 * 12) + (sum by (node) (kube_node_info{node=\"titan-12\"}) * 1e-3 * 13) + (sum by (node) (kube_node_info{node=\"titan-13\"}) * 1e-3 * 14) + (sum by (node) (kube_node_info{node=\"titan-14\"}) * 1e-3 * 15) + (sum by (node) (kube_node_info{node=\"titan-15\"}) * 1e-3 * 16) + (sum by (node) (kube_node_info{node=\"titan-16\"}) * 1e-3 * 17) + (sum by (node) (kube_node_info{node=\"titan-17\"}) * 1e-3 * 18) + (sum by (node) (kube_node_info{node=\"titan-18\"}) * 1e-3 * 19) + (sum by (node) (kube_node_info{node=\"titan-19\"}) * 1e-3 * 20) + (sum by (node) (kube_node_info{node=\"titan-22\"}) * 1e-3 * 21) + (sum by (node) (kube_node_info{node=\"titan-24\"}) * 1e-3 * 22)))))",
+              "expr": "(sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) * on(namespace,node) group_left() ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)) == bool on(namespace) group_left() (max by (namespace) ((sum by (namespace,node) (kube_pod_info{pod!=\"\" , node!=\"\"}) / on(namespace) group_left() clamp_min(sum by (namespace) (kube_pod_info{pod!=\"\"}), 1) * 100) + on(node) group_left() ((sum by (node) (kube_node_info{node=\"titan-0a\"}) * 0 + 0.001) or (sum by (node) (kube_node_info{node=\"titan-0b\"}) * 0 + 0.002) or (sum by (node) (kube_node_info{node=\"titan-0c\"}) * 0 + 0.003) or (sum by (node) (kube_node_info{node=\"titan-db\"}) * 0 + 0.004) or (sum by (node) (kube_node_info{node=\"titan-04\"}) * 0 + 0.005) or (sum by (node) (kube_node_info{node=\"titan-05\"}) * 0 + 0.006) or (sum by (node) (kube_node_info{node=\"titan-06\"}) * 0 + 0.007) or (sum by (node) (kube_node_info{node=\"titan-07\"}) * 0 + 0.008) or (sum by (node) (kube_node_info{node=\"titan-08\"}) * 0 + 0.009000000000000001) or (sum by (node) (kube_node_info{node=\"titan-09\"}) * 0 + 0.01) or (sum by (node) (kube_node_info{node=\"titan-10\"}) * 0 + 0.011) or (sum by (node) (kube_node_info{node=\"titan-11\"}) * 0 + 0.012) or (sum by (node) (kube_node_info{node=\"titan-12\"}) * 0 + 0.013000000000000001) or (sum by (node) (kube_node_info{node=\"titan-13\"}) * 0 + 0.014) or (sum by (node) (kube_node_info{node=\"titan-14\"}) * 0 + 0.015) or (sum by (node) (kube_node_info{node=\"titan-15\"}) * 0 + 0.016) or (sum by (node) (kube_node_info{node=\"titan-16\"}) * 0 + 0.017) or (sum by (node) (kube_node_info{node=\"titan-17\"}) * 0 + 0.018000000000000002) or (sum by (node) (kube_node_info{node=\"titan-18\"}) * 0 + 0.019) or (sum by (node) (kube_node_info{node=\"titan-19\"}) * 0 + 0.02) or (sum by (node) (kube_node_info{node=\"titan-22\"}) * 0 + 0.021) or (sum by (node) (kube_node_info{node=\"titan-24\"}) * 0 + 0.022)))))",
               "refId": "A",
               "instant": true,
               "format": "table"