harbor/bootstrap: pin via dynamic host label managed by recovery script

scripts/bootstrap/recovery-config.env

@@ -4,6 +4,7 @@ STATE_SUBDIR=".local/share/ananke"
 HARBOR_BUNDLE_BASENAME="harbor-bootstrap-v2.14.1-arm64.tar.zst"
 HARBOR_TARGET_NODE=""
 HARBOR_CANARY_NODE=""
+HARBOR_HOST_LABEL_KEY="ananke.bstein.dev/harbor-bootstrap"
 HARBOR_CANARY_IMAGE="registry.bstein.dev/bstein/kubectl:1.35.0"
 NODE_HELPER_IMAGE="registry.bstein.dev/bstein/ananke-node-helper:0.1.0"
 NODE_HELPER_NAMESPACE="maintenance"

scripts/cluster_power_recovery.sh

@@ -35,6 +35,7 @@ Options:
   --harbor-bundle-file <path>   Harbor bootstrap bundle on the control host
   --harbor-target-node <name>   Node that should host Harbor during bootstrap (default: auto)
   --harbor-canary-node <name>   Node used for Harbor pull canary (default: auto)
+  --harbor-host-label-key <key> Node label key used to pin Harbor bootstrap workloads (default: ${HARBOR_HOST_LABEL_KEY:-ananke.bstein.dev/harbor-bootstrap})
   --harbor-canary-image <image> Harbor-backed image used for pull canary (default: ${HARBOR_CANARY_IMAGE:-registry.bstein.dev/bstein/kubectl:1.35.0})
   --node-helper-image <image>   Privileged helper image used for host operations (default: ${NODE_HELPER_IMAGE:-registry.bstein.dev/bstein/ananke-node-helper:0.1.0})
   --bundle-http-port <port>     Temporary HTTP port used to serve bootstrap bundles (default: ${BUNDLE_HTTP_PORT:-8877})
@@ -92,6 +93,7 @@ RECOVERY_STATE_FILE="${STATE_ROOT}/cluster_power_recovery.state"
 HARBOR_BUNDLE_FILE="${STATE_ROOT}/bundles/${HARBOR_BUNDLE_BASENAME:-harbor-bootstrap-v2.14.1-arm64.tar.zst}"
 HARBOR_TARGET_NODE="${HARBOR_TARGET_NODE:-}"
 HARBOR_CANARY_NODE="${HARBOR_CANARY_NODE:-}"
+HARBOR_HOST_LABEL_KEY="${HARBOR_HOST_LABEL_KEY:-ananke.bstein.dev/harbor-bootstrap}"
 HARBOR_CANARY_IMAGE="${HARBOR_CANARY_IMAGE:-registry.bstein.dev/bstein/kubectl:1.35.0}"
 NODE_HELPER_IMAGE="${NODE_HELPER_IMAGE:-registry.bstein.dev/bstein/ananke-node-helper:0.1.0}"
 NODE_HELPER_NAMESPACE="${NODE_HELPER_NAMESPACE:-maintenance}"
@@ -176,6 +178,10 @@ while [[ $# -gt 0 ]]; do
       HARBOR_CANARY_NODE="${2:?missing harbor canary node}"
       shift 2
       ;;
+    --harbor-host-label-key)
+      HARBOR_HOST_LABEL_KEY="${2:?missing harbor host label key}"
+      shift 2
+      ;;
     --harbor-canary-image)
       HARBOR_CANARY_IMAGE="${2:?missing canary image}"
       shift 2
@@ -484,6 +490,18 @@ ensure_harbor_target_node() {
   HARBOR_TARGET_NODE="${fallback}"
 }
 
+ensure_harbor_host_label() {
+  [[ -n "${HARBOR_TARGET_NODE}" ]] || die "Harbor target node is not set."
+  local labeled node
+  labeled="$(kubectl get nodes -l "${HARBOR_HOST_LABEL_KEY}=true" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)"
+  while IFS= read -r node; do
+    [[ -z "${node}" ]] && continue
+    [[ "${node}" == "${HARBOR_TARGET_NODE}" ]] && continue
+    run kubectl label node "${node}" "${HARBOR_HOST_LABEL_KEY}-"
+  done <<< "${labeled}"
+  run kubectl label node "${HARBOR_TARGET_NODE}" "${HARBOR_HOST_LABEL_KEY}=true" --overwrite
+}
+
 as_array_from_csv() {
   local csv="$1"
   local out_var="$2"
@@ -855,6 +873,7 @@ seed_harbor_images() {
   local images_text control_ip bundle_name script_content seed_rc=0
   [[ -f "${HARBOR_BUNDLE_FILE}" ]] || die "Harbor bundle not found at ${HARBOR_BUNDLE_FILE}"
   ensure_harbor_target_node
+  ensure_harbor_host_label
   images_text="$(sed '/^[[:space:]]*#/d;/^[[:space:]]*$/d' "${BOOTSTRAP_DIR}/harbor-bootstrap-images.txt")"
   [[ -n "${images_text}" ]] || die "No Harbor images listed in ${BOOTSTRAP_DIR}/harbor-bootstrap-images.txt"
   bundle_name="$(basename "${HARBOR_BUNDLE_FILE}")"
@@ -933,6 +952,7 @@ resume_flux_and_reconcile() {
 status_report() {
   local battery flux_ready harbor_code workers
   local effective_target effective_canary
+  local labeled_nodes
   battery="$(read_ups_battery || true)"
   flux_ready="$(kubectl -n flux-system get gitrepository flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
   harbor_code="$(curl -ksS -o /dev/null -w '%{http_code}' https://registry.bstein.dev/v2/ || true)"
@@ -951,6 +971,10 @@ status_report() {
   echo "node_helper_image=${NODE_HELPER_IMAGE}"
   echo "harbor_target_node=${effective_target:-unknown}"
   echo "harbor_canary_node=${effective_canary:-unknown}"
+  labeled_nodes="$(kubectl get nodes -l "${HARBOR_HOST_LABEL_KEY}=true" -o jsonpath='{range .items[*]}{.metadata.name}{","}{end}' 2>/dev/null || true)"
+  labeled_nodes="${labeled_nodes%,}"
+  echo "harbor_host_label_key=${HARBOR_HOST_LABEL_KEY}"
+  echo "harbor_host_label_nodes=${labeled_nodes:-none}"
   echo "workers=${workers}"
   echo "recovery_pending=${RECOVERY_PENDING}"
   echo "startup_attempted=${STARTUP_ATTEMPTED_DURING_OUTAGE}"
@@ -1053,6 +1077,10 @@ startup_flow() {
   fi
   mark_checkpoint startup_api_ready
 
+  ensure_harbor_target_node
+  ensure_harbor_host_label
+  mark_checkpoint startup_harbor_host_labeled
+
   if [[ -n "${FORCE_FLUX_BRANCH}" ]]; then
     run kubectl -n flux-system patch gitrepository flux-system --type=merge -p "{\"spec\":{\"ref\":{\"branch\":\"${FORCE_FLUX_BRANCH}\"}}}"
     mark_checkpoint startup_flux_branch_forced
@@ -1105,6 +1133,9 @@ startup_flow() {
 
 prepare_flow() {
   [[ -f "${HARBOR_BUNDLE_FILE}" ]] || die "Harbor bundle missing at ${HARBOR_BUNDLE_FILE}. Build and copy it to the canonical control host first."
+  ensure_harbor_target_node
+  ensure_harbor_host_label
+  mark_checkpoint prepare_harbor_host_labeled
   if [[ "${SKIP_HELPER_PREWARM}" -eq 0 ]]; then
     prewarm_node_helper_image
     mark_checkpoint prepare_helper_prewarmed
@@ -1131,6 +1162,7 @@ log "bundle-file=${HARBOR_BUNDLE_FILE}"
 log "node-helper-image=${NODE_HELPER_IMAGE}"
 log "harbor-target-node-config=${HARBOR_TARGET_NODE:-auto}"
 log "harbor-canary-node-config=${HARBOR_CANARY_NODE:-auto}"
+log "harbor-host-label-key=${HARBOR_HOST_LABEL_KEY}"
 report_flux_source_state
 
 case "${MODE}" in

services/harbor/helmrelease.yaml

@@ -75,6 +75,8 @@ spec:
     redis:
       type: internal
       internal:
+        nodeSelector:
+          ananke.bstein.dev/harbor-bootstrap: "true"
         image:
           repository: registry.bstein.dev/infra/harbor-redis
           tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-redis:tag"}
@@ -109,6 +111,8 @@ spec:
     existingSecretAdminPasswordKey: harbor_admin_password
     existingSecretSecretKey: harbor-core
     core:
+      nodeSelector:
+        ananke.bstein.dev/harbor-bootstrap: "true"
       image:
         repository: registry.bstein.dev/infra/harbor-core
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-core:tag"}
@@ -168,6 +172,8 @@ spec:
                     operator: In
                     values: ["rpi4"]
     jobservice:
+      nodeSelector:
+        ananke.bstein.dev/harbor-bootstrap: "true"
       image:
         repository: registry.bstein.dev/infra/harbor-jobservice
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-jobservice:tag"}
@@ -208,6 +214,8 @@ spec:
                     operator: In
                     values: ["rpi4"]
     portal:
+      nodeSelector:
+        ananke.bstein.dev/harbor-bootstrap: "true"
       image:
         repository: registry.bstein.dev/infra/harbor-portal
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-portal:tag"}
@@ -233,6 +241,8 @@ spec:
                     operator: In
                     values: ["rpi4"]
     registry:
+      nodeSelector:
+        ananke.bstein.dev/harbor-bootstrap: "true"
       registry:
         image:
           repository: registry.bstein.dev/infra/harbor-registry
@@ -309,6 +319,8 @@ spec:
                     operator: In
                     values: ["rpi4"]
     nginx:
+      nodeSelector:
+        ananke.bstein.dev/harbor-bootstrap: "true"
       image:
         repository: registry.bstein.dev/infra/harbor-nginx
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-nginx:tag"}