04a737e156
Some checks failed
Tests / Declarative: Post Actions failed: 43, skipped: 89, passed: 4272
194 lines
8.5 KiB
YAML
194 lines
8.5 KiB
YAML
# services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml
|
|
apiVersion: apps/v1
|
|
kind: DaemonSet
|
|
metadata:
|
|
name: titan-22-link-keeper
|
|
namespace: maintenance
|
|
labels:
|
|
app: titan-22-link-keeper
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: titan-22-link-keeper
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: titan-22-link-keeper
|
|
spec:
|
|
hostNetwork: true
|
|
hostPID: true
|
|
nodeSelector:
|
|
kubernetes.io/hostname: titan-22
|
|
tolerations:
|
|
- key: node.kubernetes.io/unschedulable
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.kubernetes.io/not-ready
|
|
operator: Exists
|
|
effect: NoExecute
|
|
- key: node.kubernetes.io/unreachable
|
|
operator: Exists
|
|
effect: NoExecute
|
|
- key: node.kubernetes.io/unreachable
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.kubernetes.io/disk-pressure
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.kubernetes.io/memory-pressure
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.kubernetes.io/pid-pressure
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
- key: node.kubernetes.io/network-unavailable
|
|
operator: Exists
|
|
effect: NoSchedule
|
|
containers:
|
|
- name: link-keeper
|
|
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
|
|
imagePullPolicy: IfNotPresent
|
|
securityContext:
|
|
privileged: true
|
|
runAsUser: 0
|
|
command:
|
|
- /bin/bash
|
|
- -lc
|
|
- |
|
|
set -u
|
|
while true; do
|
|
nsenter -t 1 -m -u -i -n -p -- /bin/sh -lc '
|
|
usb_if="enx4cea41672a87"
|
|
usb_mac="4c:ea:41:67:2a:87"
|
|
old_if="enp5s0"
|
|
canonical_ip="192.168.22.22"
|
|
gateway="192.168.22.1"
|
|
k3s_env="/etc/default/k3s-agent"
|
|
|
|
date -Is
|
|
if [ ! -d "/sys/class/net/${usb_if}" ]; then
|
|
echo "USB Ethernet interface ${usb_if} is missing"
|
|
exit 0
|
|
fi
|
|
|
|
detected_mac="$(cat "/sys/class/net/${usb_if}/address")"
|
|
if [ "${detected_mac}" != "${usb_mac}" ]; then
|
|
echo "refusing to configure ${usb_if}: unexpected MAC ${detected_mac}"
|
|
exit 0
|
|
fi
|
|
|
|
# Keep the RTL8153 firmware patch available for subsequent cold boots.
|
|
if ! dpkg-query -W -f="\${Status}" firmware-realtek 2>/dev/null | grep -q "install ok installed"; then
|
|
echo "installing firmware-realtek for ${usb_if}"
|
|
apt-get update
|
|
DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firmware-realtek
|
|
fi
|
|
|
|
stale_swap_uuid="ccab2cd7-43fe-480d-b0a6-e567e5d7310a"
|
|
if [ -f /swapfile ] && grep -q -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab; then
|
|
echo "removing stale swap UUID ${stale_swap_uuid} from /etc/fstab"
|
|
fstab_tmp="$(mktemp)"
|
|
grep -v -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab >"${fstab_tmp}"
|
|
install -m 0644 "${fstab_tmp}" /etc/fstab
|
|
rm -f "${fstab_tmp}"
|
|
fi
|
|
|
|
if nmcli connection show "Wired connection 1" >/dev/null 2>&1; then
|
|
nmcli connection modify "Wired connection 1" connection.autoconnect no || true
|
|
fi
|
|
nmcli device disconnect "${old_if}" >/dev/null 2>&1 || true
|
|
ip link set "${old_if}" down || true
|
|
|
|
connection="$(nmcli -g GENERAL.CONNECTION device show "${usb_if}" 2>/dev/null || true)"
|
|
if [ -z "${connection}" ] || [ "${connection}" = "--" ]; then
|
|
connection="titan-22-usb-primary"
|
|
if ! nmcli connection show "${connection}" >/dev/null 2>&1; then
|
|
nmcli connection add type ethernet ifname "${usb_if}" con-name "${connection}"
|
|
fi
|
|
fi
|
|
|
|
nmcli connection modify "${connection}" \
|
|
connection.interface-name "${usb_if}" \
|
|
connection.autoconnect yes \
|
|
connection.autoconnect-priority 100 \
|
|
ipv4.method manual \
|
|
ipv4.addresses "${canonical_ip}/24" \
|
|
ipv4.gateway "${gateway}" \
|
|
ipv4.dns "${gateway}" \
|
|
ipv4.dns-search titan \
|
|
ipv6.method auto
|
|
|
|
if ! ip -4 address show dev "${usb_if}" | grep -q " ${canonical_ip}/24"; then
|
|
echo "moving the canonical node address to ${usb_if}"
|
|
nmcli connection up "${connection}" ifname "${usb_if}"
|
|
fi
|
|
|
|
ip link set "${usb_if}" up
|
|
ethtool --set-eee "${usb_if}" eee off || true
|
|
|
|
mkdir -p "$(dirname "${k3s_env}")"
|
|
env_tmp="$(mktemp)"
|
|
if [ -f "${k3s_env}" ]; then
|
|
grep -v -E "^K3S_(NODE_IP|FLANNEL_IFACE)=" "${k3s_env}" >"${env_tmp}" || true
|
|
fi
|
|
printf "K3S_NODE_IP=%s\\nK3S_FLANNEL_IFACE=%s\\n" \
|
|
"${canonical_ip}" "${usb_if}" >>"${env_tmp}"
|
|
if ! cmp -s "${env_tmp}" "${k3s_env}"; then
|
|
install -m 0644 "${env_tmp}" "${k3s_env}"
|
|
echo "updated k3s network environment"
|
|
fi
|
|
rm -f "${env_tmp}"
|
|
|
|
# k3s embeds kubelet; the standalone Debian unit only crash-loops.
|
|
systemctl disable --now kubelet.service >/dev/null 2>&1 || true
|
|
|
|
recover_overlay() {
|
|
# A USB disconnect removes its VXLAN child without stopping k3s.
|
|
# Host /run survives helper restarts but resets on the next boot.
|
|
state_dir="$1"
|
|
now="$2"
|
|
mkdir -p "${state_dir}"
|
|
if ip link show flannel.1 >/dev/null 2>&1; then
|
|
rm -f "${state_dir}/missing-since"
|
|
return
|
|
fi
|
|
systemctl is-active --quiet k3s-agent || return 0
|
|
[ "${now}" -ge 300 ] || return 0
|
|
if [ ! -f "${state_dir}/missing-since" ]; then
|
|
printf "%s\n" "${now}" >"${state_dir}/missing-since"
|
|
echo "pod overlay missing; checking again before recovery"
|
|
return
|
|
fi
|
|
missing_since="$(cat "${state_dir}/missing-since")"
|
|
[ "$((now - missing_since))" -ge 60 ] || return 0
|
|
attempts=0
|
|
last_restart=0
|
|
[ ! -f "${state_dir}/attempts" ] || attempts="$(cat "${state_dir}/attempts")"
|
|
[ ! -f "${state_dir}/last-restart" ] || last_restart="$(cat "${state_dir}/last-restart")"
|
|
if [ "${attempts}" -ge 3 ]; then
|
|
echo "pod overlay recovery exhausted for this boot; operator review required"
|
|
return
|
|
fi
|
|
if [ "${attempts}" -gt 0 ] && [ "$((now - last_restart))" -lt 900 ]; then
|
|
return
|
|
fi
|
|
# Record before restarting so helper interruption cannot cause a loop.
|
|
printf "%s\n" "$((attempts + 1))" >"${state_dir}/attempts"
|
|
printf "%s\n" "${now}" >"${state_dir}/last-restart"
|
|
echo "restarting k3s-agent to restore missing pod overlay"
|
|
systemctl --no-block try-restart k3s-agent
|
|
}
|
|
|
|
if [ "$(cat "/sys/class/net/${usb_if}/carrier" 2>/dev/null)" = "1" ] &&
|
|
ip -4 address show dev "${usb_if}" | grep -q " ${canonical_ip}/24"; then
|
|
read -r uptime_seconds unused </proc/uptime
|
|
recover_overlay /run/titan-22-link-keeper "${uptime_seconds%%.*}"
|
|
fi
|
|
|
|
ethtool "${usb_if}" | sed -n "1,45p" || true
|
|
ip -br address show "${usb_if}" || true
|
|
ip -d link show flannel.1 | sed -n "1,3p" || true
|
|
'
|
|
sleep 60
|
|
done
|