Compare commits
1 Commits
master
...
hermes-rep
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
354e9d48fa |
40
Jenkinsfile
vendored
40
Jenkinsfile
vendored
@ -1,8 +1,6 @@
|
|||||||
pipeline {
|
pipeline {
|
||||||
agent {
|
agent {
|
||||||
kubernetes {
|
kubernetes {
|
||||||
// Keep build I/O off the node's runtime USB drive.
|
|
||||||
workspaceVolume dynamicPVC(accessModes: 'ReadWriteOnce', requestsSize: '40Gi', storageClassName: 'ci-scratch')
|
|
||||||
defaultContainer 'builder'
|
defaultContainer 'builder'
|
||||||
retries 2
|
retries 2
|
||||||
yaml """
|
yaml """
|
||||||
@ -12,9 +10,6 @@ metadata:
|
|||||||
labels:
|
labels:
|
||||||
app: metis
|
app: metis
|
||||||
spec:
|
spec:
|
||||||
securityContext:
|
|
||||||
fsGroup: 1000
|
|
||||||
fsGroupChangePolicy: OnRootMismatch
|
|
||||||
nodeSelector:
|
nodeSelector:
|
||||||
kubernetes.io/arch: arm64
|
kubernetes.io/arch: arm64
|
||||||
node-role.kubernetes.io/worker: "true"
|
node-role.kubernetes.io/worker: "true"
|
||||||
@ -23,9 +18,12 @@ spec:
|
|||||||
requiredDuringSchedulingIgnoredDuringExecution:
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
nodeSelectorTerms:
|
nodeSelectorTerms:
|
||||||
- matchExpressions:
|
- matchExpressions:
|
||||||
- {key: hardware, operator: In, values: [rpi5, rpi4]}
|
- key: kubernetes.io/hostname
|
||||||
- {key: node-role.kubernetes.io/worker, operator: In, values: ["true"]}
|
operator: NotIn
|
||||||
- {key: kubernetes.io/hostname, operator: NotIn, values: [titan-04, titan-05, titan-06, titan-08, titan-11, titan-12, titan-13, titan-14, titan-15, titan-17, titan-18, titan-19]}
|
values:
|
||||||
|
- titan-06
|
||||||
|
- titan-12
|
||||||
|
- titan-14
|
||||||
preferredDuringSchedulingIgnoredDuringExecution:
|
preferredDuringSchedulingIgnoredDuringExecution:
|
||||||
- weight: 100
|
- weight: 100
|
||||||
preference:
|
preference:
|
||||||
@ -61,17 +59,12 @@ spec:
|
|||||||
- --host=tcp://0.0.0.0:2375
|
- --host=tcp://0.0.0.0:2375
|
||||||
- --tls=false
|
- --tls=false
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: workspace-volume
|
- name: dind-storage
|
||||||
mountPath: /var/lib/docker
|
mountPath: /var/lib/docker
|
||||||
subPath: .cache/docker
|
|
||||||
- name: builder
|
- name: builder
|
||||||
image: registry.bstein.dev/bstein/docker:27
|
image: registry.bstein.dev/bstein/docker:27
|
||||||
command: ["cat"]
|
command: ["cat"]
|
||||||
tty: true
|
tty: true
|
||||||
# The build runs in dind; this container is its client.
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "128Mi"
|
|
||||||
env:
|
env:
|
||||||
- name: DOCKER_HOST
|
- name: DOCKER_HOST
|
||||||
value: tcp://localhost:2375
|
value: tcp://localhost:2375
|
||||||
@ -97,9 +90,6 @@ spec:
|
|||||||
image: registry.bstein.dev/bstein/python:3.12-slim
|
image: registry.bstein.dev/bstein/python:3.12-slim
|
||||||
command: ["cat"]
|
command: ["cat"]
|
||||||
tty: true
|
tty: true
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
memory: "128Mi"
|
|
||||||
volumeMounts:
|
volumeMounts:
|
||||||
- name: workspace-volume
|
- name: workspace-volume
|
||||||
mountPath: /home/jenkins/agent
|
mountPath: /home/jenkins/agent
|
||||||
@ -111,8 +101,12 @@ spec:
|
|||||||
- name: workspace-volume
|
- name: workspace-volume
|
||||||
mountPath: /home/jenkins/agent
|
mountPath: /home/jenkins/agent
|
||||||
volumes:
|
volumes:
|
||||||
|
- name: workspace-volume
|
||||||
|
emptyDir: {}
|
||||||
- name: docker-config-writable
|
- name: docker-config-writable
|
||||||
emptyDir: {}
|
emptyDir: {}
|
||||||
|
- name: dind-storage
|
||||||
|
emptyDir: {}
|
||||||
- name: harbor-config
|
- name: harbor-config
|
||||||
secret:
|
secret:
|
||||||
secretName: harbor-robot-pipeline
|
secretName: harbor-robot-pipeline
|
||||||
@ -123,13 +117,6 @@ spec:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
environment {
|
environment {
|
||||||
PIP_CACHE_DIR = '/home/jenkins/agent/.cache/pip'
|
|
||||||
NPM_CONFIG_CACHE = '/home/jenkins/agent/.cache/npm'
|
|
||||||
SONAR_USER_HOME = '/home/jenkins/agent/.cache/sonar'
|
|
||||||
TMPDIR = '/home/jenkins/agent/.cache/tmp'
|
|
||||||
GOCACHE = '/home/jenkins/agent/.cache/go-build'
|
|
||||||
GOMODCACHE = '/home/jenkins/agent/.cache/go-mod'
|
|
||||||
GOTMPDIR = '/home/jenkins/agent/.cache/go-tmp'
|
|
||||||
REGISTRY = 'registry.bstein.dev/bstein'
|
REGISTRY = 'registry.bstein.dev/bstein'
|
||||||
IMAGE = "${REGISTRY}/metis"
|
IMAGE = "${REGISTRY}/metis"
|
||||||
SENTINEL_IMAGE = "${REGISTRY}/metis-sentinel"
|
SENTINEL_IMAGE = "${REGISTRY}/metis-sentinel"
|
||||||
@ -162,11 +149,6 @@ spec:
|
|||||||
pollSCM('H/5 * * * *')
|
pollSCM('H/5 * * * *')
|
||||||
}
|
}
|
||||||
stages {
|
stages {
|
||||||
stage('Prepare build scratch') {
|
|
||||||
steps {
|
|
||||||
sh 'mkdir -p /home/jenkins/agent/.cache/tmp /home/jenkins/agent/.cache/go-tmp'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stage('Checkout') {
|
stage('Checkout') {
|
||||||
steps {
|
steps {
|
||||||
checkout scm
|
checkout scm
|
||||||
|
|||||||
@ -35,11 +35,6 @@ if [[ "${1:-}" == "-R" ]]; then
|
|||||||
set -- $2
|
set -- $2
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
stat)
|
stat)
|
||||||
link="$(awk -v path="${2:-}" '$1=="symlink" && $2==path {print $3}' "${state}" | tail -n1)"
|
|
||||||
if [ -n "${link}" ]; then
|
|
||||||
printf 'Type: symlink\nFast link dest: "%s"\n' "${link}"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
||||||
mode="${mode: -4}"
|
mode="${mode: -4}"
|
||||||
printf 'Mode: %s\n' "${mode}"
|
printf 'Mode: %s\n' "${mode}"
|
||||||
|
|||||||
@ -73,11 +73,6 @@ if [[ "${1:-}" == "-R" ]]; then
|
|||||||
set -- $2
|
set -- $2
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
stat)
|
stat)
|
||||||
link="$(awk -v path="${2:-}" '$1=="symlink" && $2==path {print $3}' "${state}" | tail -n1)"
|
|
||||||
if [ -n "${link}" ]; then
|
|
||||||
printf 'Type: symlink\nFast link dest: "%s"\n' "${link}"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
||||||
mode="${mode: -4}"
|
mode="${mode: -4}"
|
||||||
printf 'Mode: %s\n' "${mode}"
|
printf 'Mode: %s\n' "${mode}"
|
||||||
@ -171,11 +166,6 @@ if [[ "${1:-}" == "-R" ]]; then
|
|||||||
set -- $2
|
set -- $2
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
stat)
|
stat)
|
||||||
link="$(awk -v path="${2:-}" '$1=="symlink" && $2==path {print $3}' "${state}" | tail -n1)"
|
|
||||||
if [ -n "${link}" ]; then
|
|
||||||
printf 'Type: symlink\nFast link dest: "%s"\n' "${link}"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
||||||
mode="${mode: -4}"
|
mode="${mode: -4}"
|
||||||
printf 'Mode: %s\n' "${mode}"
|
printf 'Mode: %s\n' "${mode}"
|
||||||
|
|||||||
@ -402,11 +402,6 @@ if [[ "${1:-}" == "-R" ]]; then
|
|||||||
set -- $2
|
set -- $2
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
stat)
|
stat)
|
||||||
link="$(awk -v path="${2:-}" '$1=="symlink" && $2==path {print $3}' "${state}" | tail -n1)"
|
|
||||||
if [ -n "${link}" ]; then
|
|
||||||
printf 'Type: symlink\nFast link dest: "%s"\n' "${link}"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
||||||
mode="${mode: -4}"
|
mode="${mode: -4}"
|
||||||
printf 'Mode: %s\n' "${mode}"
|
printf 'Mode: %s\n' "${mode}"
|
||||||
|
|||||||
@ -1,54 +0,0 @@
|
|||||||
# Node administration after recovery
|
|
||||||
|
|
||||||
Use the normal node SSH account and password-protected sudo. Passwords live in
|
|
||||||
Vault at `kv/atlas/nodes/<hostname>` under `atlas_password` and `root_password`.
|
|
||||||
The SSH key remains the normal login mechanism; a console root password does
|
|
||||||
not require allowing root password login over SSH.
|
|
||||||
|
|
||||||
Recovery injects a native `metis-node-identity.service`, enabled in the image.
|
|
||||||
It applies passwords even when cloud-init is absent. Each new injection creates
|
|
||||||
`/etc/metis/node-identity.pending`, so an old completion marker in the base image
|
|
||||||
cannot suppress the new identity. Cloud-init and the native unit serialize on
|
|
||||||
one lock. Failed credential setup remains a failed unit with the pending marker
|
|
||||||
intact; the Longhorn first-boot helper no longer ignores that failure.
|
|
||||||
|
|
||||||
Both administrator passwords must be supplied. The generated first-boot file is
|
|
||||||
root-readable only. After applying passwords, the script replaces it with the
|
|
||||||
non-secret boot metadata. This is removal from the active filesystem, not a
|
|
||||||
claim of forensic erasure from flash media or old image copies.
|
|
||||||
|
|
||||||
Default Metis passwordless command grants are removed. Existing sudo-group
|
|
||||||
administration remains password protected. Do not provision a substitute
|
|
||||||
NOPASSWD rule when password setup fails.
|
|
||||||
|
|
||||||
## Existing-node repair, October 4, 2026
|
|
||||||
|
|
||||||
Titan-12/13/19 retained root SSH keys but had locked atlas passwords; their root
|
|
||||||
passwords also differed from Vault. Both accounts were restored to their existing
|
|
||||||
Vault credentials without rebooting. A separate fresh SSH session authenticated
|
|
||||||
sudo with the Vault atlas password and reached UID 0 on each node. Titan-20/21's
|
|
||||||
existing unlocked root passwords were also restored to their Vault values.
|
|
||||||
|
|
||||||
The Atlas repository contains `scripts/node_admin_access.py` for an explicit
|
|
||||||
hostname-checked audit/repair. It accepts credentials only on stdin and emits
|
|
||||||
booleans, never passwords or hashes. Consult Atlas's cluster operator guide for
|
|
||||||
the current verification record and unavailable nodes. Existing native Ubuntu
|
|
||||||
root-account locks do not prevent full administration through atlas and sudo.
|
|
||||||
|
|
||||||
Do not apply a recovered node's old firstboot.env blindly to a live node: fetch
|
|
||||||
that node's current Vault record and verify the hostname first. Verify independent
|
|
||||||
password-backed sudo before retiring a legacy Metis grant.
|
|
||||||
|
|
||||||
## Checks after burning a replacement image
|
|
||||||
|
|
||||||
1. `systemctl status metis-node-identity.service`
|
|
||||||
2. Confirm `/etc/metis/node-identity.pending` is absent.
|
|
||||||
3. Log in over a separate SSH connection and run `sudo -k -v`, using the stored
|
|
||||||
atlas password; then `sudo id -u` must print `0`.
|
|
||||||
4. Confirm `firstboot.env` contains no password variables, without printing the
|
|
||||||
file into shared logs. Preserve the existing authorized keys.
|
|
||||||
5. Check host/storage health before uncordoning the Kubernetes node through Flux.
|
|
||||||
|
|
||||||
The image and script tests exercise password failure, successful retry, repeated
|
|
||||||
execution, ext4 symlink enablement, and credential-file permissions. These tests
|
|
||||||
are not a physical image boot validation; do that on the next replacement medium.
|
|
||||||
@ -5,6 +5,7 @@ marker="/var/lib/metis/rpi4-longhorn-firstboot.done"
|
|||||||
env_file="/etc/metis/firstboot.env"
|
env_file="/etc/metis/firstboot.env"
|
||||||
key_file="/etc/metis/authorized_keys"
|
key_file="/etc/metis/authorized_keys"
|
||||||
fstab_append="/etc/metis/fstab.append"
|
fstab_append="/etc/metis/fstab.append"
|
||||||
|
sudoers_file="/etc/metis/sudoers-hecate"
|
||||||
default_groups=(tty disk dialout sudo audio video plugdev games users systemd-journal input render netdev)
|
default_groups=(tty disk dialout sudo audio video plugdev games users systemd-journal input render netdev)
|
||||||
|
|
||||||
exec > >(tee -a /var/log/metis-rpi4-longhorn-firstboot.log) 2>&1
|
exec > >(tee -a /var/log/metis-rpi4-longhorn-firstboot.log) 2>&1
|
||||||
@ -142,8 +143,19 @@ if [ -s "${key_file}" ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
if [ -s "${sudoers_file}" ]; then
|
||||||
|
install -d -m 755 /etc/sudoers.d
|
||||||
|
install -m 440 "${sudoers_file}" /etc/sudoers.d/90-hecate-atlas
|
||||||
|
if command -v visudo >/dev/null 2>&1; then
|
||||||
|
if ! visudo -cf /etc/sudoers.d/90-hecate-atlas >/dev/null 2>&1; then
|
||||||
|
echo "WARN: invalid /etc/sudoers.d/90-hecate-atlas generated by metis; removing it."
|
||||||
|
rm -f /etc/sudoers.d/90-hecate-atlas
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
if [ -x /usr/local/sbin/metis-apply-node-identity.sh ]; then
|
if [ -x /usr/local/sbin/metis-apply-node-identity.sh ]; then
|
||||||
/usr/local/sbin/metis-apply-node-identity.sh
|
/usr/local/sbin/metis-apply-node-identity.sh || true
|
||||||
fi
|
fi
|
||||||
|
|
||||||
rm -f /root/.not_logged_in_yet
|
rm -f /root/.not_logged_in_yet
|
||||||
|
|||||||
@ -270,13 +270,6 @@ func writeExt4Files(fsPath string, files []inject.FileSpec) error {
|
|||||||
destPath := "/" + strings.TrimPrefix(filepath.ToSlash(f.Path), "/")
|
destPath := "/" + strings.TrimPrefix(filepath.ToSlash(f.Path), "/")
|
||||||
localPath := filepath.Join(stageDir, filepath.FromSlash(f.Path))
|
localPath := filepath.Join(stageDir, filepath.FromSlash(f.Path))
|
||||||
commands = append(commands, fmt.Sprintf("rm %s", destPath))
|
commands = append(commands, fmt.Sprintf("rm %s", destPath))
|
||||||
if f.Symlink != "" {
|
|
||||||
if filepath.IsAbs(f.Symlink) || strings.ContainsAny(f.Symlink, "\n\r\t \"") {
|
|
||||||
return fmt.Errorf("invalid relative link target for %s", destPath)
|
|
||||||
}
|
|
||||||
commands = append(commands, fmt.Sprintf("symlink %s %s", destPath, f.Symlink))
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
commands = append(commands, fmt.Sprintf("write %s %s", localPath, destPath))
|
commands = append(commands, fmt.Sprintf("write %s %s", localPath, destPath))
|
||||||
commands = append(commands, fmt.Sprintf("sif %s mode 0%o", destPath, uint32(0o100000|f.Mode.Perm())))
|
commands = append(commands, fmt.Sprintf("sif %s mode 0%o", destPath, uint32(0o100000|f.Mode.Perm())))
|
||||||
}
|
}
|
||||||
@ -347,13 +340,6 @@ func verifyExt4File(fsPath string, file inject.FileSpec, workDir string) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("verify %s: %w: %s", destPath, err, string(statOut))
|
return fmt.Errorf("verify %s: %w: %s", destPath, err, string(statOut))
|
||||||
}
|
}
|
||||||
if file.Symlink != "" {
|
|
||||||
if !strings.Contains(string(statOut), "Type: symlink") ||
|
|
||||||
!strings.Contains(string(statOut), fmt.Sprintf("Fast link dest: \"%s\"", file.Symlink)) {
|
|
||||||
return fmt.Errorf("verify %s symlink mismatch", destPath)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
expectedMode := fmt.Sprintf("Mode: %04o", file.Mode.Perm())
|
expectedMode := fmt.Sprintf("Mode: %04o", file.Mode.Perm())
|
||||||
if !strings.Contains(string(statOut), expectedMode) {
|
if !strings.Contains(string(statOut), expectedMode) {
|
||||||
return fmt.Errorf("verify %s mode: expected %s in %s", destPath, expectedMode, string(statOut))
|
return fmt.Errorf("verify %s mode: expected %s in %s", destPath, expectedMode, string(statOut))
|
||||||
|
|||||||
@ -1,44 +0,0 @@
|
|||||||
package image
|
|
||||||
|
|
||||||
import (
|
|
||||||
"metis/pkg/inject"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Use a disposable filesystem image to verify actual debugfs symlink semantics.
|
|
||||||
func TestSystemdSymlinkOnExt4(t *testing.T) {
|
|
||||||
for _, tool := range []string{"mkfs.ext4", "debugfs"} {
|
|
||||||
if _, err := exec.LookPath(tool); err != nil {
|
|
||||||
t.Skip("filesystem tools unavailable")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
fs := filepath.Join(t.TempDir(), "test.ext4")
|
|
||||||
f, err := os.Create(fs)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err = f.Truncate(16 * 1024 * 1024); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err = f.Close(); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if out, err := exec.Command("mkfs.ext4", "-q", "-F", fs).CombinedOutput(); err != nil {
|
|
||||||
t.Fatalf("mkfs: %v %s", err, out)
|
|
||||||
}
|
|
||||||
files := []inject.FileSpec{
|
|
||||||
{Path: "etc/systemd/system/metis-node-identity.service", Content: []byte("[Unit]\nDescription=Test\n"), Mode: 0644, RootFS: true},
|
|
||||||
{Path: "etc/systemd/system/multi-user.target.wants/metis-node-identity.service", Symlink: "../metis-node-identity.service", RootFS: true},
|
|
||||||
}
|
|
||||||
for i := 0; i < 2; i++ {
|
|
||||||
if err := writeExt4Files(fs, files); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if err := verifyExt4File(fs, inject.FileSpec{Path: files[1].Path, Symlink: "../wrong.service"}, t.TempDir()); err == nil {
|
|
||||||
t.Fatal("wrong target accepted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -6,8 +6,6 @@ import (
|
|||||||
"path/filepath"
|
"path/filepath"
|
||||||
)
|
)
|
||||||
|
|
||||||
var setFileMode = os.Chmod
|
|
||||||
|
|
||||||
// Injector writes node config into a mounted image (boot/root paths supplied by caller).
|
// Injector writes node config into a mounted image (boot/root paths supplied by caller).
|
||||||
type Injector struct {
|
type Injector struct {
|
||||||
BootPath string
|
BootPath string
|
||||||
@ -19,8 +17,7 @@ type FileSpec struct {
|
|||||||
Path string
|
Path string
|
||||||
Content []byte
|
Content []byte
|
||||||
Mode os.FileMode
|
Mode os.FileMode
|
||||||
RootFS bool // if true, write under root path; else boot path
|
RootFS bool // if true, write under root path; else boot path
|
||||||
Symlink string // relative target for native systemd enablement
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write materializes the requested files under the boot or root mount because
|
// Write materializes the requested files under the boot or root mount because
|
||||||
@ -35,24 +32,9 @@ func (i *Injector) Write(files []FileSpec) error {
|
|||||||
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
|
||||||
return fmt.Errorf("mkdir %s: %w", filepath.Dir(target), err)
|
return fmt.Errorf("mkdir %s: %w", filepath.Dir(target), err)
|
||||||
}
|
}
|
||||||
if f.Symlink != "" {
|
|
||||||
if current, err := os.Readlink(target); err == nil && current == f.Symlink {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
// Refuse to replace unrelated files in the mounted image.
|
|
||||||
if err := os.Symlink(f.Symlink, target); err != nil {
|
|
||||||
return fmt.Errorf("link %s: %w", target, err)
|
|
||||||
}
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(target, f.Content, f.Mode); err != nil {
|
if err := os.WriteFile(target, f.Content, f.Mode); err != nil {
|
||||||
return fmt.Errorf("write %s: %w", target, err)
|
return fmt.Errorf("write %s: %w", target, err)
|
||||||
}
|
}
|
||||||
// WriteFile retains an existing file's permissions, including loose
|
|
||||||
// permissions inherited from a recovery image.
|
|
||||||
if err := setFileMode(target, f.Mode); err != nil {
|
|
||||||
return fmt.Errorf("chmod %s: %w", target, err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@ -39,46 +39,3 @@ func TestWriteReturnsFilesystemErrors(t *testing.T) {
|
|||||||
t.Fatal("expected write error for root path file")
|
t.Fatal("expected write error for root path file")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestWriteSystemdEnablementAndSecretPermissions(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
target := filepath.Join(root, "secret")
|
|
||||||
if err := os.WriteFile(target, []byte("old"), 0644); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
files := []FileSpec{
|
|
||||||
{Path: "secret", Content: []byte("synthetic"), Mode: 0600, RootFS: true},
|
|
||||||
{Path: "system/wants/unit", Symlink: "../unit", RootFS: true},
|
|
||||||
}
|
|
||||||
inj := Injector{RootPath: root}
|
|
||||||
for n := 0; n < 2; n++ {
|
|
||||||
if err := inj.Write(files); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
info, err := os.Stat(target)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if info.Mode().Perm() != 0600 {
|
|
||||||
t.Fatal("old loose permissions retained")
|
|
||||||
}
|
|
||||||
link, err := os.Readlink(filepath.Join(root, "system/wants/unit"))
|
|
||||||
if err != nil || link != "../unit" {
|
|
||||||
t.Fatal(link, err)
|
|
||||||
}
|
|
||||||
if err := inj.Write([]FileSpec{{Path: "secret", Symlink: "../unit", RootFS: true}}); err == nil {
|
|
||||||
t.Fatal("overwrote existing regular file")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestSecretPermissionFailureIsReported(t *testing.T) {
|
|
||||||
previous := setFileMode
|
|
||||||
setFileMode = func(string, os.FileMode) error { return os.ErrPermission }
|
|
||||||
defer func() { setFileMode = previous }()
|
|
||||||
inj := Injector{RootPath: t.TempDir()}
|
|
||||||
err := inj.Write([]FileSpec{{Path: "secret", Content: []byte("synthetic"), Mode: 0600, RootFS: true}})
|
|
||||||
if err == nil {
|
|
||||||
t.Fatal("secret permission failure was ignored")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@ -107,9 +107,6 @@ func Inject(inv *inventory.Inventory, nodeName, boot, root string) error {
|
|||||||
|
|
||||||
func buildFiles(cfg *config.NodeConfig, sec *secrets.NodeSecrets) ([]inject.FileSpec, error) {
|
func buildFiles(cfg *config.NodeConfig, sec *secrets.NodeSecrets) ([]inject.FileSpec, error) {
|
||||||
files := []inject.FileSpec{
|
files := []inject.FileSpec{
|
||||||
{Path: "etc/metis/node-identity.pending", Content: []byte("Apply the injected node identity before accepting recovery as complete.\n"), Mode: 0o600, RootFS: true},
|
|
||||||
{Path: "etc/systemd/system/metis-node-identity.service", Content: []byte(nodeIdentityUnitContent()), Mode: 0o644, RootFS: true},
|
|
||||||
{Path: "etc/systemd/system/multi-user.target.wants/metis-node-identity.service", Symlink: "../metis-node-identity.service", RootFS: true},
|
|
||||||
{Path: "etc/hostname", Content: []byte(cfg.Hostname + "\n"), Mode: 0o644, RootFS: true},
|
{Path: "etc/hostname", Content: []byte(cfg.Hostname + "\n"), Mode: 0o644, RootFS: true},
|
||||||
{Path: "etc/hosts", Content: []byte(hostsContent(cfg.Hostname)), Mode: 0o644, RootFS: true},
|
{Path: "etc/hosts", Content: []byte(hostsContent(cfg.Hostname)), Mode: 0o644, RootFS: true},
|
||||||
{Path: "etc/rancher/k3s/config.yaml", Content: []byte(k3sConfigContent(cfg)), Mode: 0o644, RootFS: true},
|
{Path: "etc/rancher/k3s/config.yaml", Content: []byte(k3sConfigContent(cfg)), Mode: 0o644, RootFS: true},
|
||||||
@ -160,6 +157,21 @@ func buildFiles(cfg *config.NodeConfig, sec *secrets.NodeSecrets) ([]inject.File
|
|||||||
RootFS: true,
|
RootFS: true,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
if cfg.SSHUser == "atlas" {
|
||||||
|
sudoers := hecateSudoersContent(cfg.SSHUser)
|
||||||
|
files = append(files, inject.FileSpec{
|
||||||
|
Path: "etc/sudoers.d/90-hecate-atlas",
|
||||||
|
Content: []byte(sudoers),
|
||||||
|
Mode: 0o440,
|
||||||
|
RootFS: true,
|
||||||
|
})
|
||||||
|
files = append(files, inject.FileSpec{
|
||||||
|
Path: "etc/metis/sudoers-hecate",
|
||||||
|
Content: []byte(sudoers),
|
||||||
|
Mode: 0o440,
|
||||||
|
RootFS: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
if len(cfg.Fstab) > 0 {
|
if len(cfg.Fstab) > 0 {
|
||||||
files = append(files, inject.FileSpec{
|
files = append(files, inject.FileSpec{
|
||||||
Path: "etc/metis/fstab.append",
|
Path: "etc/metis/fstab.append",
|
||||||
@ -334,6 +346,13 @@ func fstabAppendContent(cfg *config.NodeConfig) string {
|
|||||||
return strings.Join(lines, "\n") + "\n"
|
return strings.Join(lines, "\n") + "\n"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func hecateSudoersContent(user string) string {
|
||||||
|
return fmt.Sprintf(
|
||||||
|
"%s ALL=(ALL) NOPASSWD: /usr/bin/systemctl, /usr/sbin/poweroff, /sbin/poweroff, /usr/local/bin/hecate, /usr/local/bin/k3s, /usr/bin/k3s\n",
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
func collectOverlays(class *inventory.NodeClass) ([]inject.FileSpec, error) {
|
func collectOverlays(class *inventory.NodeClass) ([]inject.FileSpec, error) {
|
||||||
var files []inject.FileSpec
|
var files []inject.FileSpec
|
||||||
if class == nil {
|
if class == nil {
|
||||||
|
|||||||
@ -147,7 +147,7 @@ func TestSecretsWrite(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestBuildFilesDoesNotGrantPasswordlessSudo(t *testing.T) {
|
func TestBuildFilesAddsHecateSudoersForAtlas(t *testing.T) {
|
||||||
cfg := &config.NodeConfig{
|
cfg := &config.NodeConfig{
|
||||||
Hostname: "n1",
|
Hostname: "n1",
|
||||||
IP: "10.0.0.10",
|
IP: "10.0.0.10",
|
||||||
@ -165,10 +165,13 @@ func TestBuildFilesDoesNotGrantPasswordlessSudo(t *testing.T) {
|
|||||||
for _, f := range files {
|
for _, f := range files {
|
||||||
pathMap[f.Path] = string(f.Content)
|
pathMap[f.Path] = string(f.Content)
|
||||||
}
|
}
|
||||||
for _, name := range []string{"etc/sudoers.d/90-hecate-atlas", "etc/metis/sudoers-hecate"} {
|
sudoers, ok := pathMap["etc/sudoers.d/90-hecate-atlas"]
|
||||||
if _, ok := pathMap[name]; ok {
|
if !ok || !strings.Contains(sudoers, "atlas ALL=(ALL) NOPASSWD: /usr/bin/systemctl") {
|
||||||
t.Fatalf("unexpected passwordless sudo grant: %s", name)
|
t.Fatalf("sudoers file missing/incorrect: %s", sudoers)
|
||||||
}
|
}
|
||||||
|
backup, ok := pathMap["etc/metis/sudoers-hecate"]
|
||||||
|
if !ok || backup != sudoers {
|
||||||
|
t.Fatalf("metis sudoers backup missing/incorrect: %s", backup)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -70,20 +70,15 @@ set -euo pipefail
|
|||||||
|
|
||||||
marker="/var/lib/metis/node-identity-applied.done"
|
marker="/var/lib/metis/node-identity-applied.done"
|
||||||
env_file="/etc/metis/firstboot.env"
|
env_file="/etc/metis/firstboot.env"
|
||||||
pending="/etc/metis/node-identity.pending"
|
|
||||||
key_file="/etc/metis/authorized_keys"
|
key_file="/etc/metis/authorized_keys"
|
||||||
|
sudoers_file="/etc/metis/sudoers-hecate"
|
||||||
default_groups=(adm sudo tty disk dialout audio video plugdev games users systemd-journal input render netdev)
|
default_groups=(adm sudo tty disk dialout audio video plugdev games users systemd-journal input render netdev)
|
||||||
|
|
||||||
if [ -f "${marker}" ] && [ ! -f "${pending}" ]; then
|
if [ -f "${marker}" ]; then
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
mkdir -p /var/lib/metis
|
mkdir -p /var/lib/metis
|
||||||
umask 077
|
|
||||||
# Cloud-init and native first boot can arrive together. Only one applies identity.
|
|
||||||
exec 9>/var/lib/metis/node-identity.lock
|
|
||||||
flock -x 9
|
|
||||||
if [ -f "${marker}" ] && [ ! -f "${pending}" ]; then exit 0; fi
|
|
||||||
if [ -f "${env_file}" ]; then
|
if [ -f "${env_file}" ]; then
|
||||||
# shellcheck disable=SC1090
|
# shellcheck disable=SC1090
|
||||||
. "${env_file}"
|
. "${env_file}"
|
||||||
@ -93,13 +88,6 @@ atlas_user="${METIS_ATLAS_USER:-atlas}"
|
|||||||
ssh_user="${METIS_SSH_USER:-${atlas_user}}"
|
ssh_user="${METIS_SSH_USER:-${atlas_user}}"
|
||||||
atlas_password="${METIS_ATLAS_PASSWORD:-}"
|
atlas_password="${METIS_ATLAS_PASSWORD:-}"
|
||||||
root_password="${METIS_ROOT_PASSWORD:-}"
|
root_password="${METIS_ROOT_PASSWORD:-}"
|
||||||
if [ -z "${atlas_password}" ] || [ -z "${root_password}" ]; then
|
|
||||||
echo "Metis identity requires both administrator passwords; no completion marker written" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
case "${atlas_password}${root_password}" in
|
|
||||||
*$'\n'*|*$'\r'*) echo "Invalid password record" >&2; exit 1 ;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
group_list=()
|
group_list=()
|
||||||
for group_name in "${default_groups[@]}"; do
|
for group_name in "${default_groups[@]}"; do
|
||||||
@ -123,7 +111,7 @@ ensure_user() {
|
|||||||
useradd -m -s /bin/bash "${user_name}"
|
useradd -m -s /bin/bash "${user_name}"
|
||||||
fi
|
fi
|
||||||
elif [ -n "${group_csv}" ]; then
|
elif [ -n "${group_csv}" ]; then
|
||||||
usermod -a -G "${group_csv}" "${user_name}"
|
usermod -a -G "${group_csv}" "${user_name}" || true
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -173,19 +161,16 @@ if [ -s "${key_file}" ]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Remove only the obsolete Metis-owned passwordless command grants.
|
if [ -s "${sudoers_file}" ]; then
|
||||||
rm -f /etc/sudoers.d/90-hecate-atlas /etc/metis/sudoers-hecate
|
install -d -m 755 /etc/sudoers.d
|
||||||
visudo -c >/dev/null
|
install -m 440 "${sudoers_file}" /etc/sudoers.d/90-hecate-atlas
|
||||||
# Keep boot metadata needed by other first-boot helpers, without passwords.
|
if command -v visudo >/dev/null 2>&1; then
|
||||||
clean_env="$(mktemp /etc/metis/firstboot.env.XXXXXX)"
|
visudo -cf /etc/sudoers.d/90-hecate-atlas >/dev/null 2>&1 || rm -f /etc/sudoers.d/90-hecate-atlas
|
||||||
for variable in METIS_HOSTNAME METIS_SSH_USER METIS_ATLAS_USER METIS_K3S_VERSION; do
|
fi
|
||||||
printf '%s=%q\n' "${variable}" "${!variable-}" >> "${clean_env}"
|
fi
|
||||||
done
|
|
||||||
chmod 600 "${clean_env}"
|
systemctl restart ssh.service >/dev/null 2>&1 || systemctl restart sshd.service >/dev/null 2>&1 || systemctl restart ssh.socket >/dev/null 2>&1 || true
|
||||||
mv "${clean_env}" "${env_file}"
|
|
||||||
unset atlas_password root_password METIS_ATLAS_PASSWORD METIS_ROOT_PASSWORD
|
|
||||||
touch "${marker}"
|
touch "${marker}"
|
||||||
rm -f "${pending}"
|
|
||||||
`
|
`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -250,24 +235,3 @@ func shellQuote(value string) string {
|
|||||||
}
|
}
|
||||||
return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'"
|
return "'" + strings.ReplaceAll(value, "'", `'"'"'`) + "'"
|
||||||
}
|
}
|
||||||
|
|
||||||
// nodeIdentityUnitContent makes password setup independent of cloud-init support.
|
|
||||||
func nodeIdentityUnitContent() string {
|
|
||||||
return `[Unit]
|
|
||||||
Description=Apply Metis node identity once
|
|
||||||
After=local-fs.target cloud-config.service
|
|
||||||
Before=k3s-agent.service
|
|
||||||
ConditionPathExists=/etc/metis/firstboot.env
|
|
||||||
ConditionPathExists=/etc/metis/node-identity.pending
|
|
||||||
|
|
||||||
[Service]
|
|
||||||
Type=oneshot
|
|
||||||
UMask=0077
|
|
||||||
ExecStart=/usr/local/sbin/metis-apply-node-identity.sh
|
|
||||||
RemainAfterExit=yes
|
|
||||||
TimeoutStartSec=120
|
|
||||||
|
|
||||||
[Install]
|
|
||||||
WantedBy=multi-user.target
|
|
||||||
`
|
|
||||||
}
|
|
||||||
|
|||||||
@ -1,107 +0,0 @@
|
|||||||
package plan
|
|
||||||
|
|
||||||
import (
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
|
||||||
"testing"
|
|
||||||
|
|
||||||
"metis/pkg/config"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Exercise the generated script with synthetic credentials and fake host tools.
|
|
||||||
func TestIdentityFailureAndRetry(t *testing.T) {
|
|
||||||
root := t.TempDir()
|
|
||||||
etc := filepath.Join(root, "etc/metis")
|
|
||||||
state := filepath.Join(root, "var/lib/metis")
|
|
||||||
bin := filepath.Join(root, "bin")
|
|
||||||
for _, dir := range []string{etc, state, bin, filepath.Join(root, "etc/sudoers.d")} {
|
|
||||||
if err := os.MkdirAll(dir, 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
envPath := filepath.Join(etc, "firstboot.env")
|
|
||||||
synthetic := "METIS_HOSTNAME='test'\nMETIS_ATLAS_PASSWORD='synthetic-atlas'\nMETIS_ROOT_PASSWORD='synthetic-root'\n"
|
|
||||||
if err := os.WriteFile(envPath, []byte(synthetic), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
commands := map[string]string{
|
|
||||||
"id": "exit 0", "getent": "exit 1", "visudo": "exit 0",
|
|
||||||
"chpasswd": "cat >/dev/null\n[ \"${FAIL_PASSWORD:-0}\" = 0 ]",
|
|
||||||
}
|
|
||||||
for name, body := range commands {
|
|
||||||
if err := os.WriteFile(filepath.Join(bin, name), []byte("#!/bin/sh\n"+body+"\n"), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
script := strings.NewReplacer("/etc/metis", etc, "/var/lib/metis", state, "/etc/sudoers.d", filepath.Join(root, "etc/sudoers.d")).Replace(nodeIdentityScriptContent())
|
|
||||||
path := filepath.Join(root, "identity.sh")
|
|
||||||
if err := os.WriteFile(path, []byte(script), 0700); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
run := func(fail string) error {
|
|
||||||
cmd := exec.Command("bash", path)
|
|
||||||
cmd.Env = append(os.Environ(), "PATH="+bin+":"+os.Getenv("PATH"), "FAIL_PASSWORD="+fail)
|
|
||||||
out, err := cmd.CombinedOutput()
|
|
||||||
if strings.Contains(string(out), "synthetic-") {
|
|
||||||
t.Fatal("password in output")
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
if run("1") == nil {
|
|
||||||
t.Fatal("password failure must propagate")
|
|
||||||
}
|
|
||||||
marker := filepath.Join(state, "node-identity-applied.done")
|
|
||||||
if _, err := os.Stat(marker); !os.IsNotExist(err) {
|
|
||||||
t.Fatal("failed setup marked complete")
|
|
||||||
}
|
|
||||||
if err := run("0"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(marker); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
clean, err := os.ReadFile(envPath)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if strings.Contains(string(clean), "PASSWORD") || strings.Contains(string(clean), "synthetic-") {
|
|
||||||
t.Fatal("password persisted after application")
|
|
||||||
}
|
|
||||||
if err := run("1"); err != nil {
|
|
||||||
t.Fatal("completed setup should not reset passwords", err)
|
|
||||||
}
|
|
||||||
// Re-injection into a previously booted image must not trust its old marker.
|
|
||||||
if err := os.WriteFile(filepath.Join(etc, "node-identity.pending"), []byte("pending"), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if err := os.WriteFile(envPath, []byte(synthetic), 0600); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if run("1") == nil {
|
|
||||||
t.Fatal("old marker suppressed newly injected credentials")
|
|
||||||
}
|
|
||||||
if err := run("0"); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if _, err := os.Stat(filepath.Join(etc, "node-identity.pending")); !os.IsNotExist(err) {
|
|
||||||
t.Fatal("successful identity left a pending marker")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIdentityEnabledWithoutCloudInit(t *testing.T) {
|
|
||||||
files, err := buildFiles(&config.NodeConfig{Hostname: "test", SSHUser: "atlas"}, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
enabled := false
|
|
||||||
for _, f := range files {
|
|
||||||
if f.Path == "etc/systemd/system/multi-user.target.wants/metis-node-identity.service" {
|
|
||||||
enabled = f.Symlink == "../metis-node-identity.service"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if !enabled {
|
|
||||||
t.Fatal("identity service is not enabled in image")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -161,11 +161,6 @@ if [[ "${1:-}" == "-R" ]]; then
|
|||||||
set -- $2
|
set -- $2
|
||||||
case "${1:-}" in
|
case "${1:-}" in
|
||||||
stat)
|
stat)
|
||||||
link="$(awk -v path="${2:-}" '$1=="symlink" && $2==path {print $3}' "${state}" | tail -n1)"
|
|
||||||
if [ -n "${link}" ]; then
|
|
||||||
printf 'Type: symlink\nFast link dest: "%s"\n' "${link}"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
mode="$(awk -v path="${2:-}" '$1=="sif" && $2==path {print $4}' "${state}" | tail -n1)"
|
||||||
mode="${mode: -4}"
|
mode="${mode: -4}"
|
||||||
printf 'Mode: %s\n' "${mode}"
|
printf 'Mode: %s\n' "${mode}"
|
||||||
|
|||||||
@ -150,17 +150,7 @@ func statusForJobError(err error) int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) handleWatch(w http.ResponseWriter, r *http.Request) {
|
func (a *App) handleWatch(w http.ResponseWriter, r *http.Request) {
|
||||||
if r.Method != http.MethodPost {
|
a.handleInternalWatch(w, r)
|
||||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
event, err := a.WatchSentinel()
|
|
||||||
if err != nil {
|
|
||||||
a.metrics.RecordWatch("error")
|
|
||||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
writeJSON(w, http.StatusOK, event)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (a *App) handleIndex(w http.ResponseWriter, r *http.Request) {
|
func (a *App) handleIndex(w http.ResponseWriter, r *http.Request) {
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user