harbor/bootstrap: pin via dynamic host label managed by recovery script
This commit is contained in:
parent
d168f02c7f
commit
525a0f9e71
@ -4,6 +4,7 @@ STATE_SUBDIR=".local/share/ananke"
|
||||
HARBOR_BUNDLE_BASENAME="harbor-bootstrap-v2.14.1-arm64.tar.zst"
|
||||
HARBOR_TARGET_NODE=""
|
||||
HARBOR_CANARY_NODE=""
|
||||
HARBOR_HOST_LABEL_KEY="ananke.bstein.dev/harbor-bootstrap"
|
||||
HARBOR_CANARY_IMAGE="registry.bstein.dev/bstein/kubectl:1.35.0"
|
||||
NODE_HELPER_IMAGE="registry.bstein.dev/bstein/ananke-node-helper:0.1.0"
|
||||
NODE_HELPER_NAMESPACE="maintenance"
|
||||
|
||||
@ -35,6 +35,7 @@ Options:
|
||||
--harbor-bundle-file <path> Harbor bootstrap bundle on the control host
|
||||
--harbor-target-node <name> Node that should host Harbor during bootstrap (default: auto)
|
||||
--harbor-canary-node <name> Node used for Harbor pull canary (default: auto)
|
||||
--harbor-host-label-key <key> Node label key used to pin Harbor bootstrap workloads (default: ${HARBOR_HOST_LABEL_KEY:-ananke.bstein.dev/harbor-bootstrap})
|
||||
--harbor-canary-image <image> Harbor-backed image used for pull canary (default: ${HARBOR_CANARY_IMAGE:-registry.bstein.dev/bstein/kubectl:1.35.0})
|
||||
--node-helper-image <image> Privileged helper image used for host operations (default: ${NODE_HELPER_IMAGE:-registry.bstein.dev/bstein/ananke-node-helper:0.1.0})
|
||||
--bundle-http-port <port> Temporary HTTP port used to serve bootstrap bundles (default: ${BUNDLE_HTTP_PORT:-8877})
|
||||
@ -92,6 +93,7 @@ RECOVERY_STATE_FILE="${STATE_ROOT}/cluster_power_recovery.state"
|
||||
HARBOR_BUNDLE_FILE="${STATE_ROOT}/bundles/${HARBOR_BUNDLE_BASENAME:-harbor-bootstrap-v2.14.1-arm64.tar.zst}"
|
||||
HARBOR_TARGET_NODE="${HARBOR_TARGET_NODE:-}"
|
||||
HARBOR_CANARY_NODE="${HARBOR_CANARY_NODE:-}"
|
||||
HARBOR_HOST_LABEL_KEY="${HARBOR_HOST_LABEL_KEY:-ananke.bstein.dev/harbor-bootstrap}"
|
||||
HARBOR_CANARY_IMAGE="${HARBOR_CANARY_IMAGE:-registry.bstein.dev/bstein/kubectl:1.35.0}"
|
||||
NODE_HELPER_IMAGE="${NODE_HELPER_IMAGE:-registry.bstein.dev/bstein/ananke-node-helper:0.1.0}"
|
||||
NODE_HELPER_NAMESPACE="${NODE_HELPER_NAMESPACE:-maintenance}"
|
||||
@ -176,6 +178,10 @@ while [[ $# -gt 0 ]]; do
|
||||
HARBOR_CANARY_NODE="${2:?missing harbor canary node}"
|
||||
shift 2
|
||||
;;
|
||||
--harbor-host-label-key)
|
||||
HARBOR_HOST_LABEL_KEY="${2:?missing harbor host label key}"
|
||||
shift 2
|
||||
;;
|
||||
--harbor-canary-image)
|
||||
HARBOR_CANARY_IMAGE="${2:?missing canary image}"
|
||||
shift 2
|
||||
@ -484,6 +490,18 @@ ensure_harbor_target_node() {
|
||||
HARBOR_TARGET_NODE="${fallback}"
|
||||
}
|
||||
|
||||
ensure_harbor_host_label() {
|
||||
[[ -n "${HARBOR_TARGET_NODE}" ]] || die "Harbor target node is not set."
|
||||
local labeled node
|
||||
labeled="$(kubectl get nodes -l "${HARBOR_HOST_LABEL_KEY}=true" -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)"
|
||||
while IFS= read -r node; do
|
||||
[[ -z "${node}" ]] && continue
|
||||
[[ "${node}" == "${HARBOR_TARGET_NODE}" ]] && continue
|
||||
run kubectl label node "${node}" "${HARBOR_HOST_LABEL_KEY}-"
|
||||
done <<< "${labeled}"
|
||||
run kubectl label node "${HARBOR_TARGET_NODE}" "${HARBOR_HOST_LABEL_KEY}=true" --overwrite
|
||||
}
|
||||
|
||||
as_array_from_csv() {
|
||||
local csv="$1"
|
||||
local out_var="$2"
|
||||
@ -855,6 +873,7 @@ seed_harbor_images() {
|
||||
local images_text control_ip bundle_name script_content seed_rc=0
|
||||
[[ -f "${HARBOR_BUNDLE_FILE}" ]] || die "Harbor bundle not found at ${HARBOR_BUNDLE_FILE}"
|
||||
ensure_harbor_target_node
|
||||
ensure_harbor_host_label
|
||||
images_text="$(sed '/^[[:space:]]*#/d;/^[[:space:]]*$/d' "${BOOTSTRAP_DIR}/harbor-bootstrap-images.txt")"
|
||||
[[ -n "${images_text}" ]] || die "No Harbor images listed in ${BOOTSTRAP_DIR}/harbor-bootstrap-images.txt"
|
||||
bundle_name="$(basename "${HARBOR_BUNDLE_FILE}")"
|
||||
@ -933,6 +952,7 @@ resume_flux_and_reconcile() {
|
||||
status_report() {
|
||||
local battery flux_ready harbor_code workers
|
||||
local effective_target effective_canary
|
||||
local labeled_nodes
|
||||
battery="$(read_ups_battery || true)"
|
||||
flux_ready="$(kubectl -n flux-system get gitrepository flux-system -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
||||
harbor_code="$(curl -ksS -o /dev/null -w '%{http_code}' https://registry.bstein.dev/v2/ || true)"
|
||||
@ -951,6 +971,10 @@ status_report() {
|
||||
echo "node_helper_image=${NODE_HELPER_IMAGE}"
|
||||
echo "harbor_target_node=${effective_target:-unknown}"
|
||||
echo "harbor_canary_node=${effective_canary:-unknown}"
|
||||
labeled_nodes="$(kubectl get nodes -l "${HARBOR_HOST_LABEL_KEY}=true" -o jsonpath='{range .items[*]}{.metadata.name}{","}{end}' 2>/dev/null || true)"
|
||||
labeled_nodes="${labeled_nodes%,}"
|
||||
echo "harbor_host_label_key=${HARBOR_HOST_LABEL_KEY}"
|
||||
echo "harbor_host_label_nodes=${labeled_nodes:-none}"
|
||||
echo "workers=${workers}"
|
||||
echo "recovery_pending=${RECOVERY_PENDING}"
|
||||
echo "startup_attempted=${STARTUP_ATTEMPTED_DURING_OUTAGE}"
|
||||
@ -1053,6 +1077,10 @@ startup_flow() {
|
||||
fi
|
||||
mark_checkpoint startup_api_ready
|
||||
|
||||
ensure_harbor_target_node
|
||||
ensure_harbor_host_label
|
||||
mark_checkpoint startup_harbor_host_labeled
|
||||
|
||||
if [[ -n "${FORCE_FLUX_BRANCH}" ]]; then
|
||||
run kubectl -n flux-system patch gitrepository flux-system --type=merge -p "{\"spec\":{\"ref\":{\"branch\":\"${FORCE_FLUX_BRANCH}\"}}}"
|
||||
mark_checkpoint startup_flux_branch_forced
|
||||
@ -1105,6 +1133,9 @@ startup_flow() {
|
||||
|
||||
prepare_flow() {
|
||||
[[ -f "${HARBOR_BUNDLE_FILE}" ]] || die "Harbor bundle missing at ${HARBOR_BUNDLE_FILE}. Build and copy it to the canonical control host first."
|
||||
ensure_harbor_target_node
|
||||
ensure_harbor_host_label
|
||||
mark_checkpoint prepare_harbor_host_labeled
|
||||
if [[ "${SKIP_HELPER_PREWARM}" -eq 0 ]]; then
|
||||
prewarm_node_helper_image
|
||||
mark_checkpoint prepare_helper_prewarmed
|
||||
@ -1131,6 +1162,7 @@ log "bundle-file=${HARBOR_BUNDLE_FILE}"
|
||||
log "node-helper-image=${NODE_HELPER_IMAGE}"
|
||||
log "harbor-target-node-config=${HARBOR_TARGET_NODE:-auto}"
|
||||
log "harbor-canary-node-config=${HARBOR_CANARY_NODE:-auto}"
|
||||
log "harbor-host-label-key=${HARBOR_HOST_LABEL_KEY}"
|
||||
report_flux_source_state
|
||||
|
||||
case "${MODE}" in
|
||||
|
||||
@ -75,6 +75,8 @@ spec:
|
||||
redis:
|
||||
type: internal
|
||||
internal:
|
||||
nodeSelector:
|
||||
ananke.bstein.dev/harbor-bootstrap: "true"
|
||||
image:
|
||||
repository: registry.bstein.dev/infra/harbor-redis
|
||||
tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-redis:tag"}
|
||||
@ -109,6 +111,8 @@ spec:
|
||||
existingSecretAdminPasswordKey: harbor_admin_password
|
||||
existingSecretSecretKey: harbor-core
|
||||
core:
|
||||
nodeSelector:
|
||||
ananke.bstein.dev/harbor-bootstrap: "true"
|
||||
image:
|
||||
repository: registry.bstein.dev/infra/harbor-core
|
||||
tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-core:tag"}
|
||||
@ -168,6 +172,8 @@ spec:
|
||||
operator: In
|
||||
values: ["rpi4"]
|
||||
jobservice:
|
||||
nodeSelector:
|
||||
ananke.bstein.dev/harbor-bootstrap: "true"
|
||||
image:
|
||||
repository: registry.bstein.dev/infra/harbor-jobservice
|
||||
tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-jobservice:tag"}
|
||||
@ -208,6 +214,8 @@ spec:
|
||||
operator: In
|
||||
values: ["rpi4"]
|
||||
portal:
|
||||
nodeSelector:
|
||||
ananke.bstein.dev/harbor-bootstrap: "true"
|
||||
image:
|
||||
repository: registry.bstein.dev/infra/harbor-portal
|
||||
tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-portal:tag"}
|
||||
@ -233,6 +241,8 @@ spec:
|
||||
operator: In
|
||||
values: ["rpi4"]
|
||||
registry:
|
||||
nodeSelector:
|
||||
ananke.bstein.dev/harbor-bootstrap: "true"
|
||||
registry:
|
||||
image:
|
||||
repository: registry.bstein.dev/infra/harbor-registry
|
||||
@ -309,6 +319,8 @@ spec:
|
||||
operator: In
|
||||
values: ["rpi4"]
|
||||
nginx:
|
||||
nodeSelector:
|
||||
ananke.bstein.dev/harbor-bootstrap: "true"
|
||||
image:
|
||||
repository: registry.bstein.dev/infra/harbor-nginx
|
||||
tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-nginx:tag"}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user