# services/maintenance/node-ops/titan-22-link-keeper-daemonset.yaml apiVersion: apps/v1 kind: DaemonSet metadata: name: titan-22-link-keeper namespace: maintenance labels: app: titan-22-link-keeper spec: selector: matchLabels: app: titan-22-link-keeper template: metadata: labels: app: titan-22-link-keeper spec: hostNetwork: true hostPID: true nodeSelector: kubernetes.io/hostname: titan-22 tolerations: - key: node.kubernetes.io/unschedulable operator: Exists effect: NoSchedule - key: node.kubernetes.io/not-ready operator: Exists effect: NoExecute - key: node.kubernetes.io/unreachable operator: Exists effect: NoExecute - key: node.kubernetes.io/unreachable operator: Exists effect: NoSchedule - key: node.kubernetes.io/disk-pressure operator: Exists effect: NoSchedule - key: node.kubernetes.io/memory-pressure operator: Exists effect: NoSchedule - key: node.kubernetes.io/pid-pressure operator: Exists effect: NoSchedule - key: node.kubernetes.io/network-unavailable operator: Exists effect: NoSchedule containers: - name: link-keeper image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131 imagePullPolicy: IfNotPresent securityContext: privileged: true runAsUser: 0 command: - /bin/bash - -lc - | set -u while true; do nsenter -t 1 -m -u -i -n -p -- /bin/sh -lc ' usb_if="enx4cea41672a87" usb_mac="4c:ea:41:67:2a:87" old_if="enp5s0" canonical_ip="192.168.22.22" gateway="192.168.22.1" k3s_env="/etc/default/k3s-agent" date -Is if [ ! -d "/sys/class/net/${usb_if}" ]; then echo "USB Ethernet interface ${usb_if} is missing" exit 0 fi detected_mac="$(cat "/sys/class/net/${usb_if}/address")" if [ "${detected_mac}" != "${usb_mac}" ]; then echo "refusing to configure ${usb_if}: unexpected MAC ${detected_mac}" exit 0 fi # Keep the RTL8153 firmware patch available for subsequent cold boots. if ! dpkg-query -W -f="\${Status}" firmware-realtek 2>/dev/null | grep -q "install ok installed"; then echo "installing firmware-realtek for ${usb_if}" apt-get update DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends firmware-realtek fi stale_swap_uuid="ccab2cd7-43fe-480d-b0a6-e567e5d7310a" if [ -f /swapfile ] && grep -q -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab; then echo "removing stale swap UUID ${stale_swap_uuid} from /etc/fstab" fstab_tmp="$(mktemp)" grep -v -E "^UUID=${stale_swap_uuid}[[:space:]]" /etc/fstab >"${fstab_tmp}" install -m 0644 "${fstab_tmp}" /etc/fstab rm -f "${fstab_tmp}" fi if nmcli connection show "Wired connection 1" >/dev/null 2>&1; then nmcli connection modify "Wired connection 1" connection.autoconnect no || true fi nmcli device disconnect "${old_if}" >/dev/null 2>&1 || true ip link set "${old_if}" down || true connection="$(nmcli -g GENERAL.CONNECTION device show "${usb_if}" 2>/dev/null || true)" if [ -z "${connection}" ] || [ "${connection}" = "--" ]; then connection="titan-22-usb-primary" if ! nmcli connection show "${connection}" >/dev/null 2>&1; then nmcli connection add type ethernet ifname "${usb_if}" con-name "${connection}" fi fi nmcli connection modify "${connection}" \ connection.interface-name "${usb_if}" \ connection.autoconnect yes \ connection.autoconnect-priority 100 \ ipv4.method manual \ ipv4.addresses "${canonical_ip}/24" \ ipv4.gateway "${gateway}" \ ipv4.dns "${gateway}" \ ipv4.dns-search titan \ ipv6.method auto if ! ip -4 address show dev "${usb_if}" | grep -q " ${canonical_ip}/24"; then echo "moving the canonical node address to ${usb_if}" nmcli connection up "${connection}" ifname "${usb_if}" fi ip link set "${usb_if}" up ethtool --set-eee "${usb_if}" eee off || true mkdir -p "$(dirname "${k3s_env}")" env_tmp="$(mktemp)" if [ -f "${k3s_env}" ]; then grep -v -E "^K3S_(NODE_IP|FLANNEL_IFACE)=" "${k3s_env}" >"${env_tmp}" || true fi printf "K3S_NODE_IP=%s\\nK3S_FLANNEL_IFACE=%s\\n" \ "${canonical_ip}" "${usb_if}" >>"${env_tmp}" if ! cmp -s "${env_tmp}" "${k3s_env}"; then install -m 0644 "${env_tmp}" "${k3s_env}" echo "updated k3s network environment" fi rm -f "${env_tmp}" # k3s embeds kubelet; the standalone Debian unit only crash-loops. systemctl disable --now kubelet.service >/dev/null 2>&1 || true recover_overlay() { # A USB disconnect removes its VXLAN child without stopping k3s. # Host /run survives helper restarts but resets on the next boot. state_dir="$1" now="$2" mkdir -p "${state_dir}" if ip link show flannel.1 >/dev/null 2>&1; then rm -f "${state_dir}/missing-since" return fi systemctl is-active --quiet k3s-agent || return 0 [ "${now}" -ge 300 ] || return 0 if [ ! -f "${state_dir}/missing-since" ]; then printf "%s\n" "${now}" >"${state_dir}/missing-since" echo "pod overlay missing; checking again before recovery" return fi missing_since="$(cat "${state_dir}/missing-since")" [ "$((now - missing_since))" -ge 60 ] || return 0 attempts=0 last_restart=0 [ ! -f "${state_dir}/attempts" ] || attempts="$(cat "${state_dir}/attempts")" [ ! -f "${state_dir}/last-restart" ] || last_restart="$(cat "${state_dir}/last-restart")" if [ "${attempts}" -ge 3 ]; then echo "pod overlay recovery exhausted for this boot; operator review required" return fi if [ "${attempts}" -gt 0 ] && [ "$((now - last_restart))" -lt 900 ]; then return fi # Record before restarting so helper interruption cannot cause a loop. printf "%s\n" "$((attempts + 1))" >"${state_dir}/attempts" printf "%s\n" "${now}" >"${state_dir}/last-restart" echo "restarting k3s-agent to restore missing pod overlay" systemctl --no-block try-restart k3s-agent } if [ "$(cat "/sys/class/net/${usb_if}/carrier" 2>/dev/null)" = "1" ] && ip -4 address show dev "${usb_if}" | grep -q " ${canonical_ip}/24"; then read -r uptime_seconds unused