From 525a0f9e71f6255d3bd67b4918d86ecfd7f3f4b4 Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Mon, 6 Apr 2026 21:32:43 -0300 Subject: [PATCH] harbor/bootstrap: pin via dynamic host label managed by recovery script --- scripts/bootstrap/recovery-config.env | 1 + scripts/cluster_power_recovery.sh | 32 +++++++++++++++++++++++++++ services/harbor/helmrelease.yaml | 12 ++++++++++ 3 files changed, 45 insertions(+) diff --git a/scripts/bootstrap/recovery-config.env b/scripts/bootstrap/recovery-config.env index 91227fbf..f5cdcd7a 100644 --- a/scripts/bootstrap/recovery-config.env +++ b/scripts/bootstrap/recovery-config.env @@ -4,6 +4,7 @@ STATE_SUBDIR=".local/share/ananke" HARBOR_BUNDLE_BASENAME="harbor-bootstrap-v2.14.1-arm64.tar.zst" HARBOR_TARGET_NODE="" HARBOR_CANARY_NODE="" +HARBOR_HOST_LABEL_KEY="ananke.bstein.dev/harbor-bootstrap" HARBOR_CANARY_IMAGE="registry.bstein.dev/bstein/kubectl:1.35.0" NODE_HELPER_IMAGE="registry.bstein.dev/bstein/ananke-node-helper:0.1.0" NODE_HELPER_NAMESPACE="maintenance" diff --git a/scripts/cluster_power_recovery.sh b/scripts/cluster_power_recovery.sh index 62be81e9..5304e63a 100755 --- a/scripts/cluster_power_recovery.sh +++ b/scripts/cluster_power_recovery.sh @@ -35,6 +35,7 @@ Options: --harbor-bundle-file Harbor bootstrap bundle on the control host --harbor-target-node Node that should host Harbor during bootstrap (default: auto) --harbor-canary-node Node used for Harbor pull canary (default: auto) + --harbor-host-label-key Node label key used to pin Harbor bootstrap workloads (default: ${HARBOR_HOST_LABEL_KEY:-ananke.bstein.dev/harbor-bootstrap}) --harbor-canary-image Harbor-backed image used for pull canary (default: ${HARBOR_CANARY_IMAGE:-registry.bstein.dev/bstein/kubectl:1.35.0}) --node-helper-image Privileged helper image used for host operations (default: ${NODE_HELPER_IMAGE:-registry.bstein.dev/bstein/ananke-node-helper:0.1.0}) --bundle-http-port Temporary HTTP port used to serve bootstrap bundles (default: ${BUNDLE_HTTP_PORT:-8877}) @@ -92,6 +93,7 @@ RECOVERY_STATE_FILE="${STATE_ROOT}/cluster_power_recovery.state" HARBOR_BUNDLE_FILE="${STATE_ROOT}/bundles/${HARBOR_BUNDLE_BASENAME:-harbor-bootstrap-v2.14.1-arm64.tar.zst}" HARBOR_TARGET_NODE="${HARBOR_TARGET_NODE:-}" HARBOR_CANARY_NODE="${HARBOR_CANARY_NODE:-}" +HARBOR_HOST_LABEL_KEY="${HARBOR_HOST_LABEL_KEY:-ananke.bstein.dev/harbor-bootstrap}" HARBOR_CANARY_IMAGE="${HARBOR_CANARY_IMAGE:-registry.bstein.dev/bstein/kubectl:1.35.0}" NODE_HELPER_IMAGE="${NODE_HELPER_IMAGE:-registry.bstein.dev/bstein/ananke-node-helper:0.1.0}" NODE_HELPER_NAMESPACE="${NODE_HELPER_NAMESPACE:-maintenance}" @@ -176,6 +178,10 @@ while [[ $# -gt 0 ]]; do HARBOR_CANARY_NODE="${2:?missing harbor canary node}" shift 2 ;; + --harbor-host-label-key) + HARBOR_HOST_LABEL_KEY="${2:?missing harbor host label key}" + shift 2 + ;; --harbor-canary-image) HARBOR_CANARY_IMAGE="${2:?missing canary image}" shift 2 @@ -484,6 +490,18 @@ ensure_harbor_target_node() { HARBOR_TARGET_NODE="${fallback}" } +ensure_harbor_host_label() { + [[ -n "${HARBOR_TARGET_NODE}" ]] || die "Harbor target node is not set." + local labeled node + labeled="$(kubectl get nodes -l "${HARBOR_HOST_LABEL_KEY}=true" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)" + while IFS= read -r node; do + [[ -z "${node}" ]] && continue + [[ "${node}" == "${HARBOR_TARGET_NODE}" ]] && continue + run kubectl label node "${node}" "${HARBOR_HOST_LABEL_KEY}-" + done <<< "${labeled}" + run kubectl label node "${HARBOR_TARGET_NODE}" "${HARBOR_HOST_LABEL_KEY}=true" --overwrite +} + as_array_from_csv() { local csv="$1" local out_var="$2" @@ -855,6 +873,7 @@ seed_harbor_images() { local images_text control_ip bundle_name script_content seed_rc=0 [[ -f "${HARBOR_BUNDLE_FILE}" ]] || die "Harbor bundle not found at ${HARBOR_BUNDLE_FILE}" ensure_harbor_target_node + ensure_harbor_host_label images_text="$(sed '/^[[:space:]]*#/d;/^[[:space:]]*$/d' "${BOOTSTRAP_DIR}/harbor-bootstrap-images.txt")" [[ -n "${images_text}" ]] || die "No Harbor images listed in ${BOOTSTRAP_DIR}/harbor-bootstrap-images.txt" bundle_name="$(basename "${HARBOR_BUNDLE_FILE}")" @@ -933,6 +952,7 @@ resume_flux_and_reconcile() { status_report() { local battery flux_ready harbor_code workers local effective_target effective_canary + local labeled_nodes battery="$(read_ups_battery || true)" flux_ready="$(kubectl -n flux-system get gitrepository flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" harbor_code="$(curl -ksS -o /dev/null -w '%{http_code}' https://registry.bstein.dev/v2/ || true)" @@ -951,6 +971,10 @@ status_report() { echo "node_helper_image=${NODE_HELPER_IMAGE}" echo "harbor_target_node=${effective_target:-unknown}" echo "harbor_canary_node=${effective_canary:-unknown}" + labeled_nodes="$(kubectl get nodes -l "${HARBOR_HOST_LABEL_KEY}=true" -o jsonpath='{range .items[*]}{.metadata.name}{","}{end}' 2>/dev/null || true)" + labeled_nodes="${labeled_nodes%,}" + echo "harbor_host_label_key=${HARBOR_HOST_LABEL_KEY}" + echo "harbor_host_label_nodes=${labeled_nodes:-none}" echo "workers=${workers}" echo "recovery_pending=${RECOVERY_PENDING}" echo "startup_attempted=${STARTUP_ATTEMPTED_DURING_OUTAGE}" @@ -1053,6 +1077,10 @@ startup_flow() { fi mark_checkpoint startup_api_ready + ensure_harbor_target_node + ensure_harbor_host_label + mark_checkpoint startup_harbor_host_labeled + if [[ -n "${FORCE_FLUX_BRANCH}" ]]; then run kubectl -n flux-system patch gitrepository flux-system --type=merge -p "{\"spec\":{\"ref\":{\"branch\":\"${FORCE_FLUX_BRANCH}\"}}}" mark_checkpoint startup_flux_branch_forced @@ -1105,6 +1133,9 @@ startup_flow() { prepare_flow() { [[ -f "${HARBOR_BUNDLE_FILE}" ]] || die "Harbor bundle missing at ${HARBOR_BUNDLE_FILE}. Build and copy it to the canonical control host first." + ensure_harbor_target_node + ensure_harbor_host_label + mark_checkpoint prepare_harbor_host_labeled if [[ "${SKIP_HELPER_PREWARM}" -eq 0 ]]; then prewarm_node_helper_image mark_checkpoint prepare_helper_prewarmed @@ -1131,6 +1162,7 @@ log "bundle-file=${HARBOR_BUNDLE_FILE}" log "node-helper-image=${NODE_HELPER_IMAGE}" log "harbor-target-node-config=${HARBOR_TARGET_NODE:-auto}" log "harbor-canary-node-config=${HARBOR_CANARY_NODE:-auto}" +log "harbor-host-label-key=${HARBOR_HOST_LABEL_KEY}" report_flux_source_state case "${MODE}" in diff --git a/services/harbor/helmrelease.yaml b/services/harbor/helmrelease.yaml index 371bd4fd..10f08fd1 100644 --- a/services/harbor/helmrelease.yaml +++ b/services/harbor/helmrelease.yaml @@ -75,6 +75,8 @@ spec: redis: type: internal internal: + nodeSelector: + ananke.bstein.dev/harbor-bootstrap: "true" image: repository: registry.bstein.dev/infra/harbor-redis tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-redis:tag"} @@ -109,6 +111,8 @@ spec: existingSecretAdminPasswordKey: harbor_admin_password existingSecretSecretKey: harbor-core core: + nodeSelector: + ananke.bstein.dev/harbor-bootstrap: "true" image: repository: registry.bstein.dev/infra/harbor-core tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-core:tag"} @@ -168,6 +172,8 @@ spec: operator: In values: ["rpi4"] jobservice: + nodeSelector: + ananke.bstein.dev/harbor-bootstrap: "true" image: repository: registry.bstein.dev/infra/harbor-jobservice tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-jobservice:tag"} @@ -208,6 +214,8 @@ spec: operator: In values: ["rpi4"] portal: + nodeSelector: + ananke.bstein.dev/harbor-bootstrap: "true" image: repository: registry.bstein.dev/infra/harbor-portal tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-portal:tag"} @@ -233,6 +241,8 @@ spec: operator: In values: ["rpi4"] registry: + nodeSelector: + ananke.bstein.dev/harbor-bootstrap: "true" registry: image: repository: registry.bstein.dev/infra/harbor-registry @@ -309,6 +319,8 @@ spec: operator: In values: ["rpi4"] nginx: + nodeSelector: + ananke.bstein.dev/harbor-bootstrap: "true" image: repository: registry.bstein.dev/infra/harbor-nginx tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-nginx:tag"}