harbor: move secrets to vault sidecars

dockerfiles/Dockerfile.harbor-core-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-core:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-jobservice-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-jobservice:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-registry-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-registry:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/home/harbor/entrypoint.sh"]

dockerfiles/Dockerfile.harbor-registryctl-vault

@@ -0,0 +1,9 @@
+FROM registry.bstein.dev/infra/harbor-registryctl:v2.14.1-arm64
+
+USER root
+COPY dockerfiles/vault-entrypoint.sh /entrypoint.sh
+RUN chmod 0755 /entrypoint.sh
+USER harbor
+
+ENTRYPOINT ["/entrypoint.sh"]
+CMD ["/home/harbor/start.sh"]

dockerfiles/vault-entrypoint.sh

@@ -11,4 +11,25 @@ if [ -n "${VAULT_ENV_FILE:-}" ]; then
   fi
 fi
 
+if [ -n "${VAULT_COPY_FILES:-}" ]; then
+  old_ifs="$IFS"
+  IFS=','
+  set -- ${VAULT_COPY_FILES}
+  IFS="$old_ifs"
+  for pair in "$@"; do
+    src="${pair%%:*}"
+    dest="${pair#*:}"
+    if [ -z "${src}" ] || [ -z "${dest}" ]; then
+      echo "Vault copy entry malformed: ${pair}" >&2
+      exit 1
+    fi
+    if [ ! -f "${src}" ]; then
+      echo "Vault file not found: ${src}" >&2
+      exit 1
+    fi
+    mkdir -p "$(dirname "${dest}")"
+    cp "${src}" "${dest}"
+  done
+fi
+
 exec "$@"

services/harbor/helmrelease.yaml

@@ -112,21 +112,46 @@ spec:
     existingSecretSecretKey: harbor-core
     core:
       image:
-        repository: registry.bstein.dev/infra/harbor-core
+        repository: registry.bstein.dev/infra/harbor-core-vault
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-core:tag"}
       nodeSelector:
         kubernetes.io/hostname: titan-05
+      serviceAccountName: harbor-vault-sync
+      automountServiceAccountToken: true
       existingSecret: harbor-core
       existingXsrfSecret: harbor-core
       existingXsrfSecretKey: CSRF_KEY
-      # OIDC config is injected via CONFIG_OVERWRITE_JSON from the harbor-oidc secret.
-      extraEnvVars:
-        - name: CONFIG_OVERWRITE_JSON
-          valueFrom:
-            secretKeyRef:
-              name: harbor-oidc
-              key: CONFIG_OVERWRITE_JSON
-              optional: true
+      secretName: harbor-core
+      podAnnotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "harbor"
+        vault.hashicorp.com/agent-inject-secret-harbor-core-env.sh: "kv/data/atlas/harbor/harbor-core"
+        vault.hashicorp.com/agent-inject-template-harbor-core-env.sh: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          export CORE_SECRET="{{ .Data.data.secret }}"
+          export CSRF_KEY="{{ .Data.data.CSRF_KEY }}"
+          export HARBOR_ADMIN_PASSWORD="{{ .Data.data.harbor_admin_password }}"
+          export REGISTRY_CREDENTIAL_PASSWORD="{{ .Data.data.REGISTRY_CREDENTIAL_PASSWORD }}"
+          {{- end }}
+          {{- with secret "kv/data/atlas/harbor/harbor-jobservice" -}}
+          export JOBSERVICE_SECRET="{{ .Data.data.JOBSERVICE_SECRET }}"
+          {{- end }}
+          {{- with secret "kv/data/atlas/harbor/harbor-db" -}}
+          export POSTGRESQL_PASSWORD="{{ .Data.data.password }}"
+          {{- end }}
+          {{- with secret "kv/data/atlas/harbor/harbor-oidc" -}}
+          export CONFIG_OVERWRITE_JSON='{{ .Data.data.CONFIG_OVERWRITE_JSON }}'
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-harbor-core-secretKey: "kv/data/atlas/harbor/harbor-core"
+        vault.hashicorp.com/agent-inject-template-harbor-core-secretKey: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          {{ .Data.data.secretKey }}
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-harbor-core-tls-key: "kv/data/atlas/harbor/harbor-core"
+        vault.hashicorp.com/agent-inject-template-harbor-core-tls-key: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          {{ index .Data.data "tls.key" }}
+          {{- end }}
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -150,10 +175,25 @@ spec:
                     values: ["rpi4"]
     jobservice:
       image:
-        repository: registry.bstein.dev/infra/harbor-jobservice
+        repository: registry.bstein.dev/infra/harbor-jobservice-vault
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-jobservice:tag"}
       nodeSelector:
         kubernetes.io/hostname: titan-05
+      serviceAccountName: harbor-vault-sync
+      automountServiceAccountToken: true
+      existingSecret: harbor-jobservice
+      podAnnotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "harbor"
+        vault.hashicorp.com/agent-inject-secret-harbor-jobservice-env.sh: "kv/data/atlas/harbor/harbor-jobservice"
+        vault.hashicorp.com/agent-inject-template-harbor-jobservice-env.sh: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          export CORE_SECRET="{{ .Data.data.secret }}"
+          {{- end }}
+          {{- with secret "kv/data/atlas/harbor/harbor-jobservice" -}}
+          export JOBSERVICE_SECRET="{{ .Data.data.JOBSERVICE_SECRET }}"
+          export REGISTRY_CREDENTIAL_PASSWORD="{{ .Data.data.REGISTRY_CREDENTIAL_PASSWORD }}"
+          {{- end }}
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -205,12 +245,43 @@ spec:
     registry:
       registry:
         image:
-          repository: registry.bstein.dev/infra/harbor-registry
+          repository: registry.bstein.dev/infra/harbor-registry-vault
           tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-registry:tag"}
       controller:
         image:
-          repository: registry.bstein.dev/infra/harbor-registryctl
+          repository: registry.bstein.dev/infra/harbor-registryctl-vault
           tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-registryctl:tag"}
+      serviceAccountName: harbor-vault-sync
+      automountServiceAccountToken: true
+      existingSecret: harbor-registry
+      credentials:
+        existingSecret: harbor-registry
+      podAnnotations:
+        vault.hashicorp.com/agent-inject: "true"
+        vault.hashicorp.com/role: "harbor"
+        vault.hashicorp.com/agent-inject-secret-harbor-registry-env.sh: "kv/data/atlas/harbor/harbor-registry"
+        vault.hashicorp.com/agent-inject-template-harbor-registry-env.sh: |
+          {{- with secret "kv/data/atlas/harbor/harbor-registry" -}}
+          export REGISTRY_HTTP_SECRET="{{ .Data.data.REGISTRY_HTTP_SECRET }}"
+          export REGISTRY_REDIS_PASSWORD="{{ .Data.data.REGISTRY_REDIS_PASSWORD }}"
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-harbor-registryctl-env.sh: "kv/data/atlas/harbor/harbor-registry"
+        vault.hashicorp.com/agent-inject-template-harbor-registryctl-env.sh: |
+          {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+          export CORE_SECRET="{{ .Data.data.secret }}"
+          {{- end }}
+          {{- with secret "kv/data/atlas/harbor/harbor-jobservice" -}}
+          export JOBSERVICE_SECRET="{{ .Data.data.JOBSERVICE_SECRET }}"
+          {{- end }}
+          {{- with secret "kv/data/atlas/harbor/harbor-registry" -}}
+          export REGISTRY_HTTP_SECRET="{{ .Data.data.REGISTRY_HTTP_SECRET }}"
+          export REGISTRY_REDIS_PASSWORD="{{ .Data.data.REGISTRY_REDIS_PASSWORD }}"
+          {{- end }}
+        vault.hashicorp.com/agent-inject-secret-harbor-registry-htpasswd: "kv/data/atlas/harbor/harbor-registry-htpasswd"
+        vault.hashicorp.com/agent-inject-template-harbor-registry-htpasswd: |
+          {{- with secret "kv/data/atlas/harbor/harbor-registry-htpasswd" -}}
+          {{ .Data.data.REGISTRY_HTPASSWD }}
+          {{- end }}
       nodeSelector:
         kubernetes.io/hostname: titan-05
       affinity:
@@ -267,3 +338,202 @@ spec:
         tag: v2.14.1-arm64 # {"$imagepolicy": "harbor:harbor-prepare:tag"}
     updateStrategy:
       type: Recreate
+  postRenderers:
+    - kustomize:
+        patches:
+          - target:
+              kind: Deployment
+              name: harbor-core
+            patch: |-
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: harbor-core
+              spec:
+                template:
+                  spec:
+                    containers:
+                      - name: core
+                        env:
+                          - $patch: replace
+                          - name: VAULT_ENV_FILE
+                            value: /vault/secrets/harbor-core-env.sh
+                          - name: VAULT_COPY_FILES
+                            value: /vault/secrets/harbor-core-secretKey:/etc/core/key,/vault/secrets/harbor-core-tls-key:/etc/core/private_key.pem
+                        envFrom:
+                          - $patch: replace
+                          - configMapRef:
+                              name: harbor-core
+                        volumeMounts:
+                          - name: secret-key
+                            $patch: delete
+                          - name: token-service-private-key
+                            $patch: delete
+                          - name: core-writable
+                            mountPath: /etc/core
+                    volumes:
+                      - name: secret-key
+                        $patch: delete
+                      - name: token-service-private-key
+                        $patch: delete
+                      - name: core-writable
+                        emptyDir: {}
+          - target:
+              kind: Deployment
+              name: harbor-jobservice
+            patch: |-
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: harbor-jobservice
+              spec:
+                template:
+                  spec:
+                    containers:
+                      - name: jobservice
+                        env:
+                          - $patch: replace
+                          - name: VAULT_ENV_FILE
+                            value: /vault/secrets/harbor-jobservice-env.sh
+                        envFrom:
+                          - $patch: replace
+                          - configMapRef:
+                              name: harbor-jobservice-env
+          - target:
+              kind: Deployment
+              name: harbor-registry
+            patch: |-
+              apiVersion: apps/v1
+              kind: Deployment
+              metadata:
+                name: harbor-registry
+              spec:
+                template:
+                  spec:
+                    containers:
+                      - name: registry
+                        env:
+                          - $patch: replace
+                          - name: VAULT_ENV_FILE
+                            value: /vault/secrets/harbor-registry-env.sh
+                          - name: VAULT_COPY_FILES
+                            value: /vault/secrets/harbor-registry-htpasswd:/etc/registry/passwd
+                        envFrom:
+                          - $patch: replace
+                        volumeMounts:
+                          - name: registry-htpasswd
+                            $patch: delete
+                          - name: registry-writable
+                            mountPath: /etc/registry
+                      - name: registryctl
+                        env:
+                          - $patch: replace
+                          - name: VAULT_ENV_FILE
+                            value: /vault/secrets/harbor-registryctl-env.sh
+                        envFrom:
+                          - $patch: replace
+                          - configMapRef:
+                              name: harbor-registryctl
+                    volumes:
+                      - name: registry-htpasswd
+                        $patch: delete
+                      - name: registry-writable
+                        emptyDir: {}
+          - target:
+              kind: Job
+              name: migration-job
+            patch: |-
+              apiVersion: batch/v1
+              kind: Job
+              metadata:
+                name: migration-job
+              spec:
+                template:
+                  metadata:
+                    annotations:
+                      vault.hashicorp.com/agent-inject: "true"
+                      vault.hashicorp.com/role: "harbor"
+                      vault.hashicorp.com/agent-inject-secret-harbor-core-env.sh: "kv/data/atlas/harbor/harbor-core"
+                      vault.hashicorp.com/agent-inject-template-harbor-core-env.sh: |
+                        {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+                        export CORE_SECRET="{{ .Data.data.secret }}"
+                        export CSRF_KEY="{{ .Data.data.CSRF_KEY }}"
+                        export HARBOR_ADMIN_PASSWORD="{{ .Data.data.harbor_admin_password }}"
+                        export REGISTRY_CREDENTIAL_PASSWORD="{{ .Data.data.REGISTRY_CREDENTIAL_PASSWORD }}"
+                        {{- end }}
+                        {{- with secret "kv/data/atlas/harbor/harbor-jobservice" -}}
+                        export JOBSERVICE_SECRET="{{ .Data.data.JOBSERVICE_SECRET }}"
+                        {{- end }}
+                        {{- with secret "kv/data/atlas/harbor/harbor-db" -}}
+                        export POSTGRESQL_PASSWORD="{{ .Data.data.password }}"
+                        {{- end }}
+                        {{- with secret "kv/data/atlas/harbor/harbor-oidc" -}}
+                        export CONFIG_OVERWRITE_JSON='{{ .Data.data.CONFIG_OVERWRITE_JSON }}'
+                        {{- end }}
+                      vault.hashicorp.com/agent-inject-secret-harbor-core-secretKey: "kv/data/atlas/harbor/harbor-core"
+                      vault.hashicorp.com/agent-inject-template-harbor-core-secretKey: |
+                        {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+                        {{ .Data.data.secretKey }}
+                        {{- end }}
+                      vault.hashicorp.com/agent-inject-secret-harbor-core-tls-key: "kv/data/atlas/harbor/harbor-core"
+                      vault.hashicorp.com/agent-inject-template-harbor-core-tls-key: |
+                        {{- with secret "kv/data/atlas/harbor/harbor-core" -}}
+                        {{ index .Data.data "tls.key" }}
+                        {{- end }}
+                  spec:
+                    automountServiceAccountToken: true
+                    containers:
+                      - name: core-job
+                        env:
+                          - $patch: replace
+                          - name: VAULT_ENV_FILE
+                            value: /vault/secrets/harbor-core-env.sh
+                        envFrom:
+                          - $patch: replace
+                          - configMapRef:
+                              name: harbor-core
+          - target:
+              kind: Secret
+              name: harbor-core
+            patch: |-
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: harbor-core
+              $patch: delete
+          - target:
+              kind: Secret
+              name: harbor-jobservice
+            patch: |-
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: harbor-jobservice
+              $patch: delete
+          - target:
+              kind: Secret
+              name: harbor-registry
+            patch: |-
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: harbor-registry
+              $patch: delete
+          - target:
+              kind: Secret
+              name: harbor-registry-htpasswd
+            patch: |-
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: harbor-registry-htpasswd
+              $patch: delete
+          - target:
+              kind: Secret
+              name: harbor-registryctl
+            patch: |-
+              apiVersion: v1
+              kind: Secret
+              metadata:
+                name: harbor-registryctl
+              $patch: delete

services/harbor/image.yaml

@@ -5,7 +5,7 @@ metadata:
   name: harbor-core
   namespace: harbor
 spec:
-  image: registry.bstein.dev/infra/harbor-core
+  image: registry.bstein.dev/infra/harbor-core-vault
   interval: 5m0s
 ---
 apiVersion: image.toolkit.fluxcd.io/v1beta2
@@ -29,7 +29,7 @@ metadata:
   name: harbor-jobservice
   namespace: harbor
 spec:
-  image: registry.bstein.dev/infra/harbor-jobservice
+  image: registry.bstein.dev/infra/harbor-jobservice-vault
   interval: 5m0s
 ---
 apiVersion: image.toolkit.fluxcd.io/v1beta2
@@ -77,7 +77,7 @@ metadata:
   name: harbor-registry
   namespace: harbor
 spec:
-  image: registry.bstein.dev/infra/harbor-registry
+  image: registry.bstein.dev/infra/harbor-registry-vault
   interval: 5m0s
 ---
 apiVersion: image.toolkit.fluxcd.io/v1beta2
@@ -101,7 +101,7 @@ metadata:
   name: harbor-registryctl
   namespace: harbor
 spec:
-  image: registry.bstein.dev/infra/harbor-registryctl
+  image: registry.bstein.dev/infra/harbor-registryctl-vault
   interval: 5m0s
 ---
 apiVersion: image.toolkit.fluxcd.io/v1beta2

services/harbor/secretproviderclass.yaml

@@ -10,84 +10,10 @@ spec:
     vaultAddress: "http://vault.vault.svc.cluster.local:8200"
     roleName: "harbor"
     objects: |
-      - objectName: "harbor-core__CSRF_KEY"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "CSRF_KEY"
-      - objectName: "harbor-core__REGISTRY_CREDENTIAL_PASSWORD"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "REGISTRY_CREDENTIAL_PASSWORD"
-      - objectName: "harbor-core__harbor_admin_password"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "harbor_admin_password"
-      - objectName: "harbor-core__secret"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "secret"
-      - objectName: "harbor-core__secretKey"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "secretKey"
-      - objectName: "harbor-core__tls.crt"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "tls.crt"
-      - objectName: "harbor-core__tls.key"
-        secretPath: "kv/data/atlas/harbor/harbor-core"
-        secretKey: "tls.key"
-      - objectName: "harbor-db__database"
-        secretPath: "kv/data/atlas/harbor/harbor-db"
-        secretKey: "database"
-      - objectName: "harbor-db__host"
-        secretPath: "kv/data/atlas/harbor/harbor-db"
-        secretKey: "host"
-      - objectName: "harbor-db__password"
-        secretPath: "kv/data/atlas/harbor/harbor-db"
-        secretKey: "password"
-      - objectName: "harbor-db__port"
-        secretPath: "kv/data/atlas/harbor/harbor-db"
-        secretKey: "port"
-      - objectName: "harbor-db__username"
-        secretPath: "kv/data/atlas/harbor/harbor-db"
-        secretKey: "username"
-      - objectName: "harbor-oidc__CONFIG_OVERWRITE_JSON"
-        secretPath: "kv/data/atlas/harbor/harbor-oidc"
-        secretKey: "CONFIG_OVERWRITE_JSON"
       - objectName: "harbor-pull__dockerconfigjson"
         secretPath: "kv/data/atlas/harbor-pull/harbor"
         secretKey: "dockerconfigjson"
   secretObjects:
-    - secretName: harbor-core
-      type: Opaque
-      data:
-        - objectName: harbor-core__CSRF_KEY
-          key: CSRF_KEY
-        - objectName: harbor-core__REGISTRY_CREDENTIAL_PASSWORD
-          key: REGISTRY_CREDENTIAL_PASSWORD
-        - objectName: harbor-core__harbor_admin_password
-          key: harbor_admin_password
-        - objectName: harbor-core__secret
-          key: secret
-        - objectName: harbor-core__secretKey
-          key: secretKey
-        - objectName: harbor-core__tls.crt
-          key: tls.crt
-        - objectName: harbor-core__tls.key
-          key: tls.key
-    - secretName: harbor-db
-      type: Opaque
-      data:
-        - objectName: harbor-db__database
-          key: database
-        - objectName: harbor-db__host
-          key: host
-        - objectName: harbor-db__password
-          key: password
-        - objectName: harbor-db__port
-          key: port
-        - objectName: harbor-db__username
-          key: username
-    - secretName: harbor-oidc
-      type: Opaque
-      data:
-        - objectName: harbor-oidc__CONFIG_OVERWRITE_JSON
-          key: CONFIG_OVERWRITE_JSON
     - secretName: harbor-regcred
       type: kubernetes.io/dockerconfigjson
       data:

services/health/config/nginx.conf

@@ -5,6 +5,12 @@ upstream wger {
 server {
     listen 8080;
 
+    client_body_temp_path /tmp/nginx/client_body 1 2;
+    proxy_temp_path /tmp/nginx/proxy 1 2;
+    fastcgi_temp_path /tmp/nginx/fastcgi 1 2;
+    uwsgi_temp_path /tmp/nginx/uwsgi 1 2;
+    scgi_temp_path /tmp/nginx/scgi 1 2;
+
     location = /api/v2/register {
         return 404;
     }