bstein/titan-iac
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(cassandra): add parallel migration infrastructure

Browse Source
This commit is contained in:
jenkins 2026-07-25 00:18:48 -03:00
parent 74192399ad
commit ea334a2863
45 changed files with 2684 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
clusters/atlas/flux-system/applications/cassandra-auth/kustomization.yaml Normal file
View File

@ -0,0 +1,23 @@
# clusters/atlas/flux-system/applications/cassandra-auth/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cassandra-auth
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/cassandra-auth
targetNamespace: sso
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: keycloak
- name: vault
- name: vault-injector
wait: false
timeout: 10m

29
clusters/atlas/flux-system/applications/cassandra/image-automation.yaml Normal file
View File

@ -0,0 +1,29 @@
# clusters/atlas/flux-system/applications/cassandra/image-automation.yaml
# Staged for the first Cassandra image rollout. Add this file to the parent
# applications kustomization after the namespace exists and the Harbor repos
# have initial tags.
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageUpdateAutomation
metadata:
name: cassandra
namespace: cassandra
spec:
interval: 1m0s
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
git:
checkout:
ref:
branch: main
commit:
author:
email: ops@bstein.dev
name: flux-bot
messageTemplate: "chore(cassandra): automated image update"
push:
branch: main
update:
strategy: Setters
path: services/cassandra

28
clusters/atlas/flux-system/applications/cassandra/kustomization.yaml Normal file
View File

@ -0,0 +1,28 @@
# clusters/atlas/flux-system/applications/cassandra/kustomization.yaml
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: cassandra
namespace: flux-system
annotations:
kustomize.toolkit.fluxcd.io/ssa: IfNotPresent
spec:
interval: 10m
path: ./services/cassandra
targetNamespace: cassandra
prune: true
sourceRef:
kind: GitRepository
name: flux-system
namespace: flux-system
dependsOn:
- name: cert-manager
- name: core
- name: keycloak
- name: longhorn
- name: traefik
- name: vault
- name: vault-csi
- name: vault-injector
wait: false
timeout: 20m

2
clusters/atlas/flux-system/applications/kustomization.yaml
View File

@ -29,6 +29,8 @@ resources:
- openclaw/kustomization.yaml
- hermes/kustomization.yaml
- game-stream/kustomization.yaml
- cassandra-auth/kustomization.yaml
- cassandra/kustomization.yaml
- veles/kustomization.yaml
- typhon/kustomization.yaml
- nextcloud/kustomization.yaml

3
infrastructure/core/node-prefer-noschedule-cronjob.yaml
View File

@ -68,7 +68,10 @@ spec:
k label node titan-23 \
veles.bstein.dev/simulation=true \
veles.bstein.dev/node-pool=oceanus \
cassandra.bstein.dev/simulation=true \
cassandra.bstein.dev/node-pool=oceanus \
node-role.kubernetes.io/veles-sim=true \
node-role.kubernetes.io/cassandra-sim=true \
longhorn-host=true \
hardware=oceanus \
--overwrite=true || true

60
infrastructure/longhorn/core/cassandra-recurring-jobs.yaml Normal file
View File

@ -0,0 +1,60 @@
# infrastructure/longhorn/core/cassandra-recurring-jobs.yaml
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: cassandra-postgres-backup
namespace: longhorn-system
spec:
name: cassandra-postgres-backup
cron: "30 5 * * *"
task: backup
groups:
- cassandra
- cassandra-postgres
retain: 7
concurrency: 1
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: cassandra-postgres-snapshot
namespace: longhorn-system
spec:
name: cassandra-postgres-snapshot
cron: "*/30 * * * *"
task: snapshot
groups:
- cassandra
- cassandra-postgres
retain: 8
concurrency: 1
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: cassandra-artifacts-backup
namespace: longhorn-system
spec:
name: cassandra-artifacts-backup
cron: "45 5 * * *"
task: backup
groups:
- cassandra
- cassandra-artifacts
retain: 7
concurrency: 1
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: cassandra-artifacts-snapshot
namespace: longhorn-system
spec:
name: cassandra-artifacts-snapshot
cron: "15 */6 * * *"
task: snapshot
groups:
- cassandra
- cassandra-artifacts
retain: 8
concurrency: 1

1
infrastructure/longhorn/core/kustomization.yaml
View File

@ -7,6 +7,7 @@ resources:
- secretproviderclass.yaml
- vault-sync-deployment.yaml
- helmrelease.yaml
- cassandra-recurring-jobs.yaml
- veles-recurring-jobs.yaml
- longhorn-settings-ensure-job.yaml
- longhorn-csi-toleration-ensure-job.yaml

18
infrastructure/longhorn/core/scripts/longhorn_disk_tags_ensure.py
View File

@ -19,8 +19,14 @@ LONGHORN_NS = "longhorn-system"
LONGHORN_API = "/apis/longhorn.io/v1beta2/namespaces/{namespace}/nodes"
DESIRED_DISK_TAGS = {
"/mnt/astreae": ["astreae"],
"/mnt/asteria": ["asteria"],
"/mnt/veles": ["veles-oceanus", "veles-db", "veles-artifacts"],
"/mnt/asteria": ["asteria", "cassandra-db", "cassandra-artifacts"],
"/mnt/veles": [
"veles-oceanus",
"veles-db",
"veles-artifacts",
"cassandra-db",
"cassandra-artifacts",
],
"/mnt/veles-db": ["veles-oceanus", "veles-db"],
"/mnt/veles-artifacts": ["veles-oceanus", "veles-artifacts"],
}
@ -34,7 +40,13 @@ DESIRED_NODE_DISKS = {
"allowScheduling": True,
"evictionRequested": False,
"storageReserved": 0,
"tags": ["veles-oceanus", "veles-db", "veles-artifacts"],
"tags": [
"veles-oceanus",
"veles-db",
"veles-artifacts",
"cassandra-db",
"cassandra-artifacts",
],
}
}
}

30
infrastructure/longhorn/core/veles-recurring-jobs.yaml
View File

@ -28,3 +28,33 @@ spec:
- veles-postgres
retain: 8
concurrency: 1
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: veles-artifacts-backup
namespace: longhorn-system
spec:
name: veles-artifacts-backup
cron: "45 5 * * *"
task: backup
groups:
- veles
- veles-artifacts
retain: 7
concurrency: 1
---
apiVersion: longhorn.io/v1beta2
kind: RecurringJob
metadata:
name: veles-artifacts-snapshot
namespace: longhorn-system
spec:
name: veles-artifacts-snapshot
cron: "15 */6 * * *"
task: snapshot
groups:
- veles
- veles-artifacts
retain: 8
concurrency: 1

17
infrastructure/modules/base/priorityclass/cassandra.yaml Normal file
View File

@ -0,0 +1,17 @@
# infrastructure/modules/base/priorityclass/cassandra.yaml
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: cassandra-core
value: 500
globalDefault: false
description: "For Cassandra core database, API, and controller workloads"
---
apiVersion: scheduling.k8s.io/v1
kind: PriorityClass
metadata:
name: cassandra-sim
value: 50
globalDefault: false
preemptionPolicy: Never
description: "For Cassandra simulation jobs; lower than core and non-preempting"

1
infrastructure/modules/base/priorityclass/kustomization.yaml
View File

@ -3,4 +3,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- scavenger.yaml
- cassandra.yaml
- veles.yaml

20
infrastructure/modules/base/storageclass/cassandra-artifacts.yaml Normal file
View File

@ -0,0 +1,20 @@
# infrastructure/modules/base/storageclass/cassandra-artifacts.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: cassandra-artifacts
annotations:
cassandra.bstein.dev/allowed-namespace: cassandra
provisioner: driver.longhorn.io
parameters:
diskSelector: cassandra-artifacts
fromBackup: ""
numberOfReplicas: "2"
staleReplicaTimeout: "30"
fsType: ext4
replicaAutoBalance: least-effort
dataLocality: best-effort
recurringJobSelector: '[{"name":"cassandra-artifacts-backup","isGroup":false},{"name":"cassandra-artifacts-snapshot","isGroup":false}]'
reclaimPolicy: Retain
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer

20
infrastructure/modules/base/storageclass/cassandra-db.yaml Normal file
View File

@ -0,0 +1,20 @@
# infrastructure/modules/base/storageclass/cassandra-db.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: cassandra-db
annotations:
cassandra.bstein.dev/allowed-namespace: cassandra
provisioner: driver.longhorn.io
parameters:
diskSelector: cassandra-db
fromBackup: ""
numberOfReplicas: "2"
staleReplicaTimeout: "30"
fsType: ext4
replicaAutoBalance: least-effort
dataLocality: best-effort
recurringJobSelector: '[{"name":"cassandra-postgres-backup","isGroup":false},{"name":"cassandra-postgres-snapshot","isGroup":false}]'
reclaimPolicy: Retain
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer

25
infrastructure/modules/base/storageclass/cassandra-storage-policy.yaml Normal file
View File

@ -0,0 +1,25 @@
# infrastructure/modules/base/storageclass/cassandra-storage-policy.yaml
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
name: cassandra-storage-namespace
spec:
failurePolicy: Fail
matchConstraints:
resourceRules:
- apiGroups: [""]
apiVersions: ["v1"]
operations: ["CREATE", "UPDATE"]
resources: ["persistentvolumeclaims"]
validations:
- expression: "!has(object.spec.storageClassName) || !(object.spec.storageClassName in ['cassandra-db', 'cassandra-artifacts']) || object.metadata.namespace == 'cassandra'"
message: "Cassandra storage classes are reserved for namespace cassandra"
---
apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicyBinding
metadata:
name: cassandra-storage-namespace
spec:
policyName: cassandra-storage-namespace
validationActions:
- Deny

3
infrastructure/modules/base/storageclass/kustomization.yaml
View File

@ -5,6 +5,9 @@ resources:
- asteria.yaml
- asteria-encrypted.yaml
- astreae.yaml
- cassandra-db.yaml
- cassandra-artifacts.yaml
- cassandra-storage-policy.yaml
- veles-oceanus-db.yaml
- veles-oceanus-artifacts.yaml
- veles-oceanus-policy.yaml

54
services/cassandra-auth/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml Normal file
View File

@ -0,0 +1,54 @@
# services/keycloak/bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
# One-off job for sso/cassandra-gitea-oidc-secret-ensure-1.
# Purpose: create/update the Keycloak cassandra realm Gitea OIDC client, ensure the
# cassandra-tester group exists, and write the client secret to Vault for Gitea.
# Run by setting spec.suspend to false, reconcile, then set it back to true.
# Safe to delete the finished Job/pod; it should not run continuously.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-gitea-oidc-secret-ensure-1
namespace: sso
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "sso-secrets"
vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
{{ with secret "kv/data/atlas/shared/keycloak-admin" }}
export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
{{ end }}
spec:
serviceAccountName: mas-secrets-ensure
restartPolicy: Never
volumes:
- name: cassandra-gitea-oidc-secret-ensure-script
configMap:
name: cassandra-gitea-oidc-secret-ensure-script
defaultMode: 0555
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/arch
operator: In
values: ["arm64"]
- key: node-role.kubernetes.io/worker
operator: Exists
containers:
- name: apply
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command: ["/scripts/cassandra_gitea_oidc_secret_ensure.sh"]
volumeMounts:
- name: cassandra-gitea-oidc-secret-ensure-script
mountPath: /scripts
readOnly: true

401
services/cassandra-auth/bootstrap-jobs/cassandra-realm-ensure-job.yaml Normal file
View File

@ -0,0 +1,401 @@
# services/keycloak/bootstrap-jobs/cassandra-realm-ensure-job.yaml
# One-off job for sso/cassandra-realm-ensure-1.
# Purpose: create the Cassandra realm, groups, OIDC client, SMTP settings, and Vault client secret.
# Keep suspended until Cassandra Vault paths/policies have reconciled, then unsuspend once.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-realm-ensure-1
namespace: sso
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "sso-secrets"
vault.hashicorp.com/agent-inject-secret-keycloak-admin-env.sh: "kv/data/atlas/shared/keycloak-admin"
vault.hashicorp.com/agent-inject-template-keycloak-admin-env.sh: |
{{ with secret "kv/data/atlas/shared/keycloak-admin" }}
export KEYCLOAK_ADMIN="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_USER="{{ .Data.data.username }}"
export KEYCLOAK_ADMIN_PASSWORD="{{ .Data.data.password }}"
{{ end }}
{{ with secret "kv/data/atlas/shared/postmark-relay" }}
export KEYCLOAK_SMTP_USER="{{ index .Data.data "apikey" }}"
export KEYCLOAK_SMTP_PASSWORD="{{ index .Data.data "apikey" }}"
{{ end }}
spec:
serviceAccountName: mas-secrets-ensure
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: Exists
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/arch
operator: In
values: ["arm64"]
containers:
- name: configure
image: python:3.11-alpine
env:
- name: KEYCLOAK_SERVER
value: http://keycloak.sso.svc.cluster.local
- name: KEYCLOAK_REALM
value: cassandra
- name: KEYCLOAK_CLIENT_ID
value: cassandra-web
- name: KEYCLOAK_PUBLIC_ISSUER
value: https://sso.bstein.dev/realms/cassandra
- name: CASSANDRA_BASE_URL
value: https://cassandra.bstein.dev
- name: CASSANDRA_TESTER_GROUP
value: cassandra-tester
- name: KEYCLOAK_SMTP_HOST
value: mail.bstein.dev
- name: KEYCLOAK_SMTP_PORT
value: "587"
- name: KEYCLOAK_SMTP_FROM
value: no-reply-cassandra@bstein.dev
- name: KEYCLOAK_SMTP_FROM_NAME
value: Cassandra
- name: CASSANDRA_PUBLIC_REPLAY_USERS
value: cassandra-dev,daniel-test,viktor-test
command: ["/bin/sh", "-c"]
args:
- |
set -eu
. /vault/secrets/keycloak-admin-env.sh
python - <<'PY'
import json
import os
import time
import urllib.error
import urllib.parse
import urllib.request
base_url = os.environ["KEYCLOAK_SERVER"].rstrip("/")
realm = os.environ["KEYCLOAK_REALM"]
client_id = os.environ["KEYCLOAK_CLIENT_ID"]
issuer = os.environ["KEYCLOAK_PUBLIC_ISSUER"]
app_base_url = os.environ["CASSANDRA_BASE_URL"].rstrip("/")
admin_user = os.environ["KEYCLOAK_ADMIN_USER"]
admin_password = os.environ["KEYCLOAK_ADMIN_PASSWORD"]
def request(method, url, token=None, payload=None, headers=None, timeout=30):
data = None
req_headers = headers.copy() if headers else {}
if token:
req_headers["Authorization"] = f"Bearer {token}"
if payload is not None:
data = json.dumps(payload).encode()
req_headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, headers=req_headers, method=method)
try:
with urllib.request.urlopen(req, timeout=timeout) as resp:
body = resp.read()
if not body:
return resp.status, None
return resp.status, json.loads(body.decode())
except urllib.error.HTTPError as exc:
raw = exc.read()
if not raw:
return exc.code, None
try:
return exc.code, json.loads(raw.decode())
except Exception:
return exc.code, {"raw": raw.decode(errors="replace")}
token_body = None
form = urllib.parse.urlencode(
{
"grant_type": "password",
"client_id": "admin-cli",
"username": admin_user,
"password": admin_password,
}
).encode()
for attempt in range(1, 11):
req = urllib.request.Request(
f"{base_url}/realms/master/protocol/openid-connect/token",
data=form,
headers={"Content-Type": "application/x-www-form-urlencoded"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=10) as resp:
token_body = json.loads(resp.read().decode())
break
except urllib.error.URLError as exc:
if attempt == 10:
raise SystemExit(f"Keycloak token request failed after retries: {exc}")
time.sleep(attempt * 2)
token = token_body["access_token"]
smtp = {
"host": os.environ["KEYCLOAK_SMTP_HOST"],
"port": os.environ["KEYCLOAK_SMTP_PORT"],
"from": os.environ["KEYCLOAK_SMTP_FROM"],
"fromDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"],
"replyTo": os.environ["KEYCLOAK_SMTP_FROM"],
"replyToDisplayName": os.environ["KEYCLOAK_SMTP_FROM_NAME"],
"user": os.environ["KEYCLOAK_SMTP_USER"],
"password": os.environ["KEYCLOAK_SMTP_PASSWORD"],
"auth": "true",
"starttls": "true",
"ssl": "false",
}
status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token)
if status == 404:
create_payload = {
"realm": realm,
"enabled": True,
"registrationAllowed": False,
"resetPasswordAllowed": True,
"verifyEmail": True,
"loginWithEmailAllowed": True,
"duplicateEmailsAllowed": False,
"smtpServer": smtp,
}
status, body = request("POST", f"{base_url}/admin/realms", token, create_payload)
if status not in (201, 204, 409):
raise SystemExit(f"Realm create failed: status={status} body={body}")
status, realm_rep = request("GET", f"{base_url}/admin/realms/{realm}", token)
if status != 200 or not isinstance(realm_rep, dict):
raise SystemExit(f"Realm fetch failed: status={status}")
realm_rep.update(
{
"enabled": True,
"registrationAllowed": False,
"resetPasswordAllowed": True,
"verifyEmail": True,
"loginWithEmailAllowed": True,
"duplicateEmailsAllowed": False,
"smtpServer": smtp,
}
)
status, body = request("PUT", f"{base_url}/admin/realms/{realm}", token, realm_rep)
if status not in (200, 204):
raise SystemExit(f"Realm update failed: status={status} body={body}")
def find_group(name):
status, groups = request(
"GET",
f"{base_url}/admin/realms/{realm}/groups?search={urllib.parse.quote(name)}",
token,
)
if status != 200:
raise SystemExit(f"Group search failed for {name}: status={status}")
for group in groups or []:
if group.get("name") == name:
return group
return None
def ensure_group(name):
group = find_group(name)
if group:
return group
status, body = request("POST", f"{base_url}/admin/realms/{realm}/groups", token, {"name": name})
if status not in (201, 204, 409):
raise SystemExit(f"Group create failed for {name}: status={status} body={body}")
group = find_group(name)
if not group:
raise SystemExit(f"Group missing after create for {name}")
return group
def find_user(username):
encoded = urllib.parse.quote(username)
for query in (f"username={encoded}&exact=true", f"email={encoded}&exact=true", f"search={encoded}"):
status, users = request(
"GET",
f"{base_url}/admin/realms/{realm}/users?{query}",
token,
)
if status != 200:
raise SystemExit(f"User search failed for {username}: status={status}")
for user in users or []:
candidates = {
str(user.get("username") or "").casefold(),
str(user.get("email") or "").casefold(),
}
if username.casefold() in candidates:
return user
return None
def ensure_user_group(username, group):
user = find_user(username)
if not user:
print(f"WARNING: Cassandra public replay user {username!r} not found; skipping membership")
return
user_id = user.get("id")
group_id = group.get("id")
if not user_id or not group_id:
raise SystemExit(f"Missing id for user/group membership: user={username}")
status, memberships = request(
"GET",
f"{base_url}/admin/realms/{realm}/users/{user_id}/groups",
token,
)
if status != 200:
raise SystemExit(f"User group lookup failed for {username}: status={status}")
if any(item.get("id") == group_id for item in memberships or []):
return
status, body = request(
"PUT",
f"{base_url}/admin/realms/{realm}/users/{user_id}/groups/{group_id}",
token,
)
if status not in (200, 204):
raise SystemExit(f"User group update failed for {username}: status={status} body={body}")
ensure_group("alpha")
ensure_group("admin")
ensure_group(os.environ["CASSANDRA_TESTER_GROUP"])
public_replay_group = ensure_group("public-replay")
for username in os.environ.get("CASSANDRA_PUBLIC_REPLAY_USERS", "").split(","):
username = username.strip()
if username:
ensure_user_group(username, public_replay_group)
status, clients = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",
token,
)
if status != 200:
raise SystemExit(f"Client lookup failed: status={status}")
client_uuid = clients[0]["id"] if clients else None
client_payload = {
"clientId": client_id,
"enabled": True,
"protocol": "openid-connect",
"publicClient": False,
"standardFlowEnabled": True,
"implicitFlowEnabled": False,
"directAccessGrantsEnabled": False,
"serviceAccountsEnabled": False,
"redirectUris": [f"{app_base_url}/*"],
"webOrigins": [app_base_url],
"rootUrl": app_base_url,
"baseUrl": "/",
"attributes": {
"pkce.code.challenge.method": "S256",
"post.logout.redirect.uris": f"{app_base_url}/*",
},
}
if not client_uuid:
status, body = request("POST", f"{base_url}/admin/realms/{realm}/clients", token, client_payload)
if status not in (201, 204, 409):
raise SystemExit(f"Client create failed: status={status} body={body}")
status, clients = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients?clientId={urllib.parse.quote(client_id)}",
token,
)
client_uuid = clients[0]["id"] if clients else None
if not client_uuid:
raise SystemExit("Client cassandra-web not found after create")
status, body = request(
"PUT",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}",
token,
client_payload,
)
if status not in (200, 204):
raise SystemExit(f"Client update failed: status={status} body={body}")
mapper_payload = {
"name": "groups",
"protocol": "openid-connect",
"protocolMapper": "oidc-group-membership-mapper",
"consentRequired": False,
"config": {
"full.path": "false",
"id.token.claim": "true",
"access.token.claim": "true",
"userinfo.token.claim": "true",
"claim.name": "groups",
"jsonType.label": "String",
},
}
status, mappers = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models",
token,
)
if status != 200:
raise SystemExit(f"Mapper lookup failed: status={status}")
mapper_id = next((mapper.get("id") for mapper in mappers or [] if mapper.get("name") == "groups"), None)
if mapper_id:
mapper_update_payload = dict(mapper_payload)
mapper_update_payload["id"] = mapper_id
status, body = request(
"PUT",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models/{mapper_id}",
token,
mapper_update_payload,
)
else:
status, body = request(
"POST",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/protocol-mappers/models",
token,
mapper_payload,
)
if status not in (200, 201, 204):
raise SystemExit(f"Mapper ensure failed: status={status} body={body}")
status, secret = request(
"GET",
f"{base_url}/admin/realms/{realm}/clients/{client_uuid}/client-secret",
token,
)
client_secret = (secret or {}).get("value")
if status != 200 or not client_secret:
raise SystemExit(f"Client secret fetch failed: status={status}")
vault_addr = os.environ.get("VAULT_ADDR", "http://vault.vault.svc.cluster.local:8200")
jwt = open("/var/run/secrets/kubernetes.io/serviceaccount/token", encoding="utf-8").read().strip()
login_payload = json.dumps({"jwt": jwt, "role": os.environ.get("VAULT_ROLE", "sso-secrets")}).encode()
req = urllib.request.Request(
f"{vault_addr}/v1/auth/kubernetes/login",
data=login_payload,
headers={"Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=20) as resp:
vault_token = json.loads(resp.read().decode())["auth"]["client_token"]
payload = {
"data": {
"client_id": client_id,
"client_secret": client_secret,
"issuer": issuer,
"realm": realm,
"required_groups": "alpha,admin",
}
}
req = urllib.request.Request(
f"{vault_addr}/v1/kv/data/atlas/cassandra/cassandra-oidc",
data=json.dumps(payload).encode(),
headers={"X-Vault-Token": vault_token, "Content-Type": "application/json"},
method="POST",
)
with urllib.request.urlopen(req, timeout=20) as resp:
if resp.status not in (200, 204):
raise SystemExit(f"Vault write returned {resp.status}")
print("Cassandra Keycloak realm/client ready")
PY

13
services/cassandra-auth/kustomization.yaml Normal file
View File

@ -0,0 +1,13 @@
# services/cassandra-auth/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: sso
resources:
- bootstrap-jobs/cassandra-realm-ensure-job.yaml
- bootstrap-jobs/cassandra-gitea-oidc-secret-ensure-job.yaml
configMapGenerator:
- name: cassandra-gitea-oidc-secret-ensure-script
files:
- cassandra_gitea_oidc_secret_ensure.sh=scripts/cassandra_gitea_oidc_secret_ensure.sh
generatorOptions:
disableNameSuffixHash: true

157
services/cassandra-auth/scripts/cassandra_gitea_oidc_secret_ensure.sh Normal file
View File

@ -0,0 +1,157 @@
#!/usr/bin/env sh
set -euo pipefail
. /vault/secrets/keycloak-admin-env.sh
KC_URL="${KEYCLOAK_SERVER:-http://keycloak.sso.svc.cluster.local}"
REALM="${KEYCLOAK_REALM:-cassandra}"
CLIENT_ID="${KEYCLOAK_CLIENT_ID:-gitea}"
PUBLIC_BASE_URL="${GITEA_PUBLIC_BASE_URL:-https://scm.bstein.dev}"
AUTH_SOURCE_NAME="${GITEA_AUTH_SOURCE_NAME:-cassandra}"
TESTER_GROUP="${CASSANDRA_GITEA_TESTER_GROUP:-cassandra-tester}"
VAULT_SECRET_PATH="${VAULT_SECRET_PATH:-kv/data/atlas/gitea/gitea-cassandra-oidc}"
ACCESS_TOKEN=""
for attempt in 1 2 3 4 5 6 7 8 9 10; do
if curl -fsS "${KC_URL}/realms/master" >/dev/null 2>&1; then
break
fi
echo "Waiting for Keycloak to be reachable (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
for attempt in 1 2 3 4 5; do
TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
-H 'Content-Type: application/x-www-form-urlencoded' \
-d "grant_type=password" \
-d "client_id=admin-cli" \
-d "username=${KEYCLOAK_ADMIN}" \
-d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
break
fi
echo "Keycloak token request failed (attempt ${attempt})" >&2
sleep $((attempt * 2))
done
if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
echo "Failed to fetch Keycloak admin token" >&2
exit 1
fi
ensure_group() {
group_name="$1"
groups="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/groups?search=$(printf '%s' "${group_name}" | jq -sRr @uri)" || true)"
if echo "$groups" | jq -e --arg name "$group_name" '.[]? | select(.name == $name)' >/dev/null 2>&1; then
return
fi
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "$(jq -nc --arg name "$group_name" '{name:$name}')" \
"${KC_URL}/admin/realms/${REALM}/groups")"
if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
echo "Keycloak group create failed for ${group_name} (status ${status})" >&2
exit 1
fi
}
ensure_group "${TESTER_GROUP}"
CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)"
CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
client_payload="$(jq -nc \
--arg client_id "${CLIENT_ID}" \
--arg root_url "${PUBLIC_BASE_URL}" \
--arg callback "${PUBLIC_BASE_URL}/user/oauth2/${AUTH_SOURCE_NAME}/callback" \
'{clientId:$client_id,enabled:true,protocol:"openid-connect",publicClient:false,standardFlowEnabled:true,implicitFlowEnabled:false,directAccessGrantsEnabled:false,serviceAccountsEnabled:false,redirectUris:[$callback],webOrigins:[$root_url],rootUrl:$root_url,baseUrl:"/",attributes:{"pkce.code.challenge.method":"S256","post.logout.redirect.uris":($root_url + "/*")}}')"
if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${client_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients")"
if [ "$status" != "201" ] && [ "$status" != "204" ] && [ "$status" != "409" ]; then
echo "Keycloak client create failed for ${CLIENT_ID} (status ${status})" >&2
exit 1
fi
CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients?clientId=$(printf '%s' "${CLIENT_ID}" | jq -sRr @uri)" || true)"
CLIENT_UUID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
fi
if [ -z "$CLIENT_UUID" ] || [ "$CLIENT_UUID" = "null" ]; then
echo "Keycloak client ${CLIENT_ID} not found after create" >&2
exit 1
fi
status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${client_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}")"
if [ "$status" != "200" ] && [ "$status" != "204" ]; then
echo "Keycloak client update failed for ${CLIENT_ID} (status ${status})" >&2
exit 1
fi
mapper_payload="$(jq -nc \
'{name:"groups",protocol:"openid-connect",protocolMapper:"oidc-group-membership-mapper",consentRequired:false,config:{"full.path":"false","id.token.claim":"true","access.token.claim":"true","userinfo.token.claim":"true","claim.name":"groups","jsonType.label":"String"}}')"
mappers="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models" || true)"
MAPPER_ID="$(echo "$mappers" | jq -r '.[]? | select(.name == "groups") | .id' | head -n1 || true)"
if [ -n "$MAPPER_ID" ] && [ "$MAPPER_ID" != "null" ]; then
status="$(curl -sS -o /dev/null -w "%{http_code}" -X PUT \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${mapper_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models/${MAPPER_ID}")"
else
status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
-H "Authorization: Bearer ${ACCESS_TOKEN}" \
-H 'Content-Type: application/json' \
-d "${mapper_payload}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/protocol-mappers/models")"
fi
if [ "$status" != "200" ] && [ "$status" != "201" ] && [ "$status" != "204" ]; then
echo "Keycloak groups mapper ensure failed (status ${status})" >&2
exit 1
fi
CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
"${KC_URL}/admin/realms/${REALM}/clients/${CLIENT_UUID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
echo "Keycloak client secret not found for ${CLIENT_ID}" >&2
exit 1
fi
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-sso-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
payload="$(jq -nc \
--arg client_id "${CLIENT_ID}" \
--arg client_secret "${CLIENT_SECRET}" \
--arg discovery_url "${PUBLIC_BASE_URL}" \
--arg issuer "https://sso.bstein.dev/realms/${REALM}" \
--arg auto_discovery_url "https://sso.bstein.dev/realms/${REALM}/.well-known/openid-configuration" \
--arg auth_source_name "${AUTH_SOURCE_NAME}" \
--arg tester_group "${TESTER_GROUP}" \
--arg group_team_map "{\"${TESTER_GROUP}\":{\"cassandra-alpha\":[\"testers\"]}}" \
'{data:{client_id:$client_id,client_secret:$client_secret,issuer:$issuer,openid_auto_discovery_url:$auto_discovery_url,auth_source_name:$auth_source_name,required_claim_name:"groups",required_claim_value:$tester_group,group_claim_name:"groups",restricted_group:$tester_group,group_team_map:$group_team_map,gitea_base_url:$discovery_url}}')"
curl -sS -X POST -H "X-Vault-Token: ${vault_token}" \
-H 'Content-Type: application/json' \
-d "${payload}" "${vault_addr}/v1/${VAULT_SECRET_PATH}" >/dev/null
echo "Cassandra realm Gitea OIDC client ready"

144
services/cassandra/NOTES.md Normal file
View File

@ -0,0 +1,144 @@
# Cassandra Infrastructure Contract
This stack is staged as a parallel replacement for Veles. Do not remove or
mutate Veles data while Cassandra is being validated.
## Cluster Contract
- Namespace: `cassandra`
- Hostname: `https://cassandra.bstein.dev`
- Namespace: `cassandra`; no alternate alpha namespace is used.
- Backend service: `cassandra-backend.cassandra.svc.cluster.local:80`
- Frontend service: `cassandra-frontend.cassandra.svc.cluster.local:80`
- Postgres service: `cassandra-postgres.cassandra.svc.cluster.local:5432`
- Artifact PVC: `cassandra-artifacts`, mounted at `/data/cassandra-artifacts`
- Storage classes: `cassandra-db`, `cassandra-artifacts`
- Images:
- `registry.bstein.dev/cassandra/cassandra-backend`
- `registry.bstein.dev/cassandra/cassandra-frontend`
- `registry.bstein.dev/cassandra/cassandra-sim-worker`
- Backend `http` container port: `8796`
- Frontend `http` container port: `8080`
- Generator and retention workloads start disabled so copied data is not
consumed or pruned before the migration is verified.
## Auth Contract
Cassandra owns authorization in the app. The `cassandra` Ingress does not use oauth2-proxy or Traefik forward-auth, so no ingress/auth layer should strip OIDC token claims. The app should validate tokens from `https://sso.bstein.dev/realms/cassandra` and expect stable `sub`, `email`, `preferred_username`, `groups`, and `realm_access.roles` claims. Do not scale Cassandra for real user traffic until native OIDC login/session flow is implemented and smoke-tested.
The Keycloak realm setup creates both groups and realm roles named `alpha` and `admin`. Members of the `alpha` group receive the `alpha` realm role; members of `admin` receive both `alpha` and `admin`. Built-in/meta strategies can stay universal, while runs and user-created strategies should remain user-scoped in the Cassandra database.
## Runtime Env
Cassandra should consume:
- `CASSANDRA_PUBLIC_BASE_URL=https://cassandra.bstein.dev`
- `CASSANDRA_OIDC_ISSUER=https://sso.bstein.dev/realms/cassandra`
- `CASSANDRA_OIDC_CLIENT_ID=cassandra-web`
- `CASSANDRA_OIDC_REQUIRED_GROUPS=alpha,cassandra-tester,admin`
- `CASSANDRA_OIDC_GROUPS_CLAIM=groups`
- `CASSANDRA_OIDC_ROLES_CLAIM=realm_access.roles`
- `DATABASE_URL` from `kv/data/atlas/cassandra/cassandra-db`
- `CASSANDRA_SESSION_SECRET` from `kv/data/atlas/cassandra/app-secrets`
- `CASSANDRA_BYOK_ENCRYPTION_KEY` from `kv/data/atlas/cassandra/app-secrets`
User OpenAI API keys must stay in the Cassandra database encrypted with
`CASSANDRA_BYOK_ENCRYPTION_KEY`; do not store per-user BYOK secrets in Vault.
Backend runtime secrets are synced from Vault by `cassandra-vault` into the generated Kubernetes Secret `cassandra-runtime-secrets`; no secret values are committed. The backend consumes that secret with `envFrom`.
## Artifact Contract
`cassandra-artifacts` is an RWO Longhorn PVC mounted into backend pods at `/data/cassandra-artifacts`. Backend pods own artifact writes and serving. Simulation Jobs should not mount or write directly to this PVC unless they are explicitly scheduled on Oceanus with the Cassandra toleration and the app has chosen a same-node direct-write model. Queue-mediated upload/copy through the backend remains the safer default until the app contract settles.
Backend, simulation workers, and retention/cleanup workers must run on Oceanus/titan-23 when they need artifact access. Frontend pods must not mount `cassandra-artifacts`.
## Simulation Jobs
The backend service account can create, watch, and delete Jobs only inside the `cassandra` namespace. Simulation pods should use service account `cassandra-sim`, set `automountServiceAccountToken: false`, and use:
```yaml
priorityClassName: cassandra-sim
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
```
The toleration intentionally references the existing Veles taint while Veles and
Cassandra run side by side. The node carries Cassandra labels for selection, but
adding a Cassandra taint waits until Veles no longer needs the same node.
Retention/cleanup Jobs that touch artifacts should use the same node selector and toleration. If they do not need Kubernetes API access, use `cassandra-sim`; otherwise keep control-plane actions in the backend/controller and run artifact cleanup through a no-token worker.
## Staged Operator Steps
1. Join `titan-23`/Oceanus to Atlas as a worker.
2. Use Metis with `titan-23` in `METIS_FLASH_HOSTS`; the existing node secret placeholder uses `192.168.22.23`.
3. Confirm the node normalizer applies the Cassandra labels and taint.
4. Add Oceanus Longhorn disks at paths tagged by the Longhorn tag ensure job.
5. Let Vault policy reconciliation run, then unsuspend `cassandra-secrets-ensure-2`.
6. Unsuspend `cassandra-realm-ensure-1` in `services/keycloak` to create the realm/client secret, groups, and roles.
7. Create the Harbor `cassandra` project or robot access before image automation is enabled in production.
8. Let `cassandra-postgres` come up on the new `cassandra-db` PVC.
9. Unsuspend `cassandra-db-copy-from-veles-1` once to copy Veles Postgres into
Cassandra and run table/count/hash verification against the target.
10. Inspect new Cassandra Longhorn volumes and confirm each two-replica volume
has one titan-23 local replica plus one Asteria replica. If Longhorn places
replicas differently, fix placement before live use.
11. Copy artifacts from `veles-artifacts` to `cassandra-artifacts` and verify
file counts/bytes before enabling retention.
12. Smoke-test `https://cassandra.bstein.dev`, only then enable generator and
retention workers.
13. Redirect `veles.bstein.dev` only after Cassandra is validated. Do not delete
Veles resources until explicitly approved.
## Copy Commands
The DB copy job is suspended by default. After `cassandra-postgres` is ready:
```bash
kubectl -n cassandra patch job cassandra-db-copy-from-veles-1 \
--type=merge -p '{"spec":{"suspend":false}}'
kubectl -n cassandra logs job/cassandra-db-copy-from-veles-1 -f
```
The artifact copy is also opt-in. Start the read-only Veles exporter, run the
Cassandra importer, inspect its verification summary, then scale the exporter
back down:
```bash
kubectl -n veles scale deploy/veles-artifact-export-for-cassandra --replicas=1
kubectl -n veles rollout status deploy/veles-artifact-export-for-cassandra
kubectl -n cassandra patch job cassandra-artifact-copy-from-veles-1 \
--type=merge -p '{"spec":{"suspend":false}}'
kubectl -n cassandra logs job/cassandra-artifact-copy-from-veles-1 -f
kubectl -n veles scale deploy/veles-artifact-export-for-cassandra --replicas=0
```
The artifact importer writes verification details under
`/data/cassandra-artifacts/.cassandra-artifact-migration/`. If the Veles
artifact PVC changes during the copy, verification can fail correctly; rerun
after quiescing artifact writers or after a final delta-copy window.
## Assumptions
- `cassandra-artifacts` is RWO for alpha; simulation workers should either run
on Oceanus with the backend or stream logs to the backend, which owns writes.
- Cassandra storage classes request two Longhorn replicas with `best-effort`
data locality. This is intended to keep one replica local to titan-23 and one
replica on an Asteria disk, but actual placement must be inspected after PVC
provisioning.
- Longhorn default backup target is `s3://atlas-soteria@us-west-004/` with
credential secret `longhorn-backup-b2`. As of the migration prep, remote
backups are failing because the object-store cap is exceeded; local snapshots
still work, but remote backup protection is not healthy until that cap is
resolved.
- The Jenkins Cassandra job should publish:
`registry.bstein.dev/cassandra/cassandra-backend`,
`registry.bstein.dev/cassandra/cassandra-frontend`,
`registry.bstein.dev/cassandra/cassandra-sim-worker`, and
`registry.bstein.dev/cassandra/cassandra-generator-worker`.

32
services/cassandra/artifacts-pvc.yaml Normal file
View File

@ -0,0 +1,32 @@
# services/cassandra/artifacts-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cassandra-artifacts
namespace: cassandra
labels:
app.kubernetes.io/name: cassandra
app.kubernetes.io/component: artifacts
spec:
accessModes:
- ReadWriteOnce
storageClassName: cassandra-artifacts
resources:
requests:
storage: 500Gi
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: cassandra-engine-cache
namespace: cassandra
labels:
app.kubernetes.io/name: cassandra
app.kubernetes.io/component: engine-cache
spec:
accessModes:
- ReadWriteOnce
storageClassName: cassandra-artifacts
resources:
requests:
storage: 100Gi

139
services/cassandra/backend-deployment.yaml Normal file
View File

@ -0,0 +1,139 @@
# services/cassandra/backend-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: cassandra-backend
namespace: cassandra
labels:
app: cassandra-backend
spec:
replicas: 1
revisionHistoryLimit: 2
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app: cassandra-backend
template:
metadata:
labels:
app: cassandra-backend
annotations:
cassandra.bstein.dev/redeploy: "2026-07-24-0.7.65-hygiene-generator-worker"
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "cassandra"
vault.hashicorp.com/agent-inject-secret-cassandra-env.sh: "kv/data/atlas/cassandra/cassandra-db"
vault.hashicorp.com/agent-inject-template-cassandra-env.sh: |
{{- with secret "kv/data/atlas/cassandra/cassandra-db" }}
unset DATABASE_URL
export CASSANDRA_DATABASE_HOST="{{ .Data.data.POSTGRES_HOST }}"
export CASSANDRA_DATABASE_PORT="{{ .Data.data.POSTGRES_PORT }}"
export CASSANDRA_DATABASE_NAME="{{ .Data.data.POSTGRES_DB }}"
export CASSANDRA_DATABASE_USER="{{ .Data.data.POSTGRES_USER }}"
export CASSANDRA_DATABASE_PASSWORD="{{ .Data.data.POSTGRES_PASSWORD }}"
export CASSANDRA_DATABASE_SSLMODE="disable"
{{- end }}
{{- with secret "kv/data/atlas/cassandra/cassandra-oidc" }}
export CASSANDRA_OIDC_CLIENT_SECRET="{{ .Data.data.client_secret }}"
{{- end }}
{{- with secret "kv/data/atlas/cassandra/app-secrets" }}
export CASSANDRA_SESSION_SECRET="{{ .Data.data.CASSANDRA_SESSION_SECRET }}"
export CASSANDRA_BYOK_ENCRYPTION_KEY="{{ .Data.data.CASSANDRA_BYOK_ENCRYPTION_KEY }}"
{{- end }}
spec:
serviceAccountName: cassandra-backend
priorityClassName: cassandra-core
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
kubernetes.io/arch: amd64
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
securityContext:
fsGroup: 10001
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
initContainers:
- name: database-init
image: registry.bstein.dev/cassandra/cassandra-backend:0.7.65 # {"$imagepolicy": "cassandra:cassandra-backend"}
imagePullPolicy: Always
command: ["/bin/sh", "-c"]
args:
- |
set -eu
if [ -f /vault/secrets/cassandra-env.sh ]; then
. /vault/secrets/cassandra-env.sh
fi
exec cassandra-db --init --wait-seconds 120
envFrom:
- configMapRef:
name: cassandra-config
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 250m
memory: 512Mi
containers:
- name: backend
image: registry.bstein.dev/cassandra/cassandra-backend:0.7.65 # {"$imagepolicy": "cassandra:cassandra-backend"}
imagePullPolicy: Always
command: ["/bin/sh", "-c"]
args:
- |
set -eu
if [ -f /vault/secrets/cassandra-env.sh ]; then
. /vault/secrets/cassandra-env.sh
fi
exec cassandra-web --host 0.0.0.0 --port 8796 --no-resume-queued
ports:
- name: http
containerPort: 8796
protocol: TCP
envFrom:
- configMapRef:
name: cassandra-config
readinessProbe:
httpGet:
path: /api/v1/ready
port: http
initialDelaySeconds: 5
periodSeconds: 10
timeoutSeconds: 3
livenessProbe:
httpGet:
path: /api/v1/live
port: http
initialDelaySeconds: 20
periodSeconds: 20
timeoutSeconds: 3
resources:
requests:
cpu: 250m
memory: 2Gi
limits:
cpu: "1"
memory: 8Gi
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
volumeMounts:
- name: artifacts
mountPath: /data/cassandra-artifacts
volumes:
- name: artifacts
persistentVolumeClaim:
claimName: cassandra-artifacts

142
services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml Normal file
View File

@ -0,0 +1,142 @@
# services/cassandra/bootstrap-jobs/cassandra-secrets-ensure-job.yaml
# One-off job for cassandra/cassandra-secrets-ensure-2.
# Purpose: seed Cassandra Vault paths before app/Postgres pods are scaled up.
# Keep suspended until the cassandra Vault role has reconciled, then unsuspend once.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-secrets-ensure-2
namespace: cassandra
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 3600
template:
spec:
serviceAccountName: cassandra-secrets-ensure
restartPolicy: Never
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: Exists
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: kubernetes.io/arch
operator: In
values: ["arm64"]
containers:
- name: apply
image: bitnami/kubectl@sha256:554ab88b1858e8424c55de37ad417b16f2a0e65d1607aa0f3fe3ce9b9f10b131
command: ["/bin/bash", "-c"]
args:
- |
set -euo pipefail
vault_addr="${VAULT_ADDR:-http://vault.vault.svc.cluster.local:8200}"
vault_role="${VAULT_ROLE:-cassandra-secrets}"
jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)"
login_payload="$(jq -nc --arg jwt "${jwt}" --arg role "${vault_role}" '{jwt:$jwt, role:$role}')"
vault_token="$(curl -sS --request POST --data "${login_payload}" \
"${vault_addr}/v1/auth/kubernetes/login" | jq -r '.auth.client_token')"
if [ -z "${vault_token}" ] || [ "${vault_token}" = "null" ]; then
echo "vault login failed" >&2
exit 1
fi
read_secret() {
path="$1"
out="$2"
curl -sS -o "${out}" -w "%{http_code}" \
-H "X-Vault-Token: ${vault_token}" \
"${vault_addr}/v1/kv/data/atlas/${path}" || true
}
write_secret() {
path="$1"
payload="$2"
out="$(mktemp)"
status="$(curl -sS -o "${out}" -w "%{http_code}" -X POST \
-H "X-Vault-Token: ${vault_token}" \
-H "Content-Type: application/json" \
-d "${payload}" \
"${vault_addr}/v1/kv/data/atlas/${path}")"
if [ "${status}" != "200" ] && [ "${status}" != "204" ]; then
echo "Vault write failed for ${path} (status ${status})" >&2
cat "${out}" >&2 || true
exit 1
fi
}
rand_b64() {
bytes="$1"
openssl rand -base64 "${bytes}" | tr -d '\n'
}
status="$(read_secret cassandra/cassandra-db /tmp/cassandra-db.json)"
if [ "${status}" = "200" ]; then
db_password="$(jq -r '.data.data.POSTGRES_PASSWORD // empty' /tmp/cassandra-db.json)"
elif [ "${status}" = "404" ]; then
db_password=""
else
echo "Vault read failed for cassandra-db (status ${status})" >&2
cat /tmp/cassandra-db.json >&2 || true
exit 1
fi
if [ -z "${db_password}" ]; then
db_password="$(rand_b64 36)"
fi
db_payload="$(jq -nc \
--arg host "cassandra-postgres.cassandra.svc.cluster.local" \
--arg port "5432" \
--arg db "cassandra" \
--arg user "cassandra" \
--arg password "${db_password}" \
'{data:{POSTGRES_HOST:$host,POSTGRES_PORT:$port,POSTGRES_DB:$db,POSTGRES_USER:$user,POSTGRES_PASSWORD:$password,DATABASE_URL:("postgresql://"+($user|@uri)+":"+($password|@uri)+"@"+$host+":"+$port+"/"+($db|@uri)+"?sslmode=disable")}}')"
write_secret cassandra/cassandra-db "${db_payload}"
status="$(read_secret cassandra/app-secrets /tmp/app-secrets.json)"
if [ "${status}" = "200" ]; then
session_secret="$(jq -r '.data.data.CASSANDRA_SESSION_SECRET // empty' /tmp/app-secrets.json)"
byok_key="$(jq -r '.data.data.CASSANDRA_BYOK_ENCRYPTION_KEY // empty' /tmp/app-secrets.json)"
elif [ "${status}" = "404" ]; then
session_secret=""
byok_key=""
else
echo "Vault read failed for app-secrets (status ${status})" >&2
cat /tmp/app-secrets.json >&2 || true
exit 1
fi
if [ -z "${session_secret}" ]; then
session_secret="$(rand_b64 48)"
fi
if [ -z "${byok_key}" ]; then
byok_key="$(rand_b64 32)"
fi
app_payload="$(jq -nc \
--arg session_secret "${session_secret}" \
--arg byok_key "${byok_key}" \
'{data:{CASSANDRA_SESSION_SECRET:$session_secret,CASSANDRA_BYOK_ENCRYPTION_KEY:$byok_key}}')"
write_secret cassandra/app-secrets "${app_payload}"
postmark_status="$(read_secret shared/postmark-relay /tmp/postmark.json)"
if [ "${postmark_status}" = "200" ]; then
smtp_password="$(jq -r '.data.data.apikey // empty' /tmp/postmark.json)"
if [ -n "${smtp_password}" ]; then
smtp_payload="$(jq -nc \
--arg host "mail.bstein.dev" \
--arg port "587" \
--arg user "${smtp_password}" \
--arg password "${smtp_password}" \
--arg from "no-reply-cassandra@bstein.dev" \
--arg from_name "Cassandra" \
'{data:{SMTP_HOST:$host,SMTP_PORT:$port,SMTP_USER:$user,SMTP_PASSWORD:$password,SMTP_FROM:$from,SMTP_FROM_NAME:$from_name,SMTP_STARTTLS:"true"}}')"
write_secret cassandra/smtp "${smtp_payload}"
fi
fi
echo "Cassandra Vault paths ready: cassandra-db, app-secrets, smtp when Postmark relay exists"

45
services/cassandra/configmap.yaml Normal file
View File

@ -0,0 +1,45 @@
# services/cassandra/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: cassandra-config
namespace: cassandra
data:
CASSANDRA_PROFILE: cluster
CASSANDRA_AUTH_MODE: oidc
CASSANDRA_SIM_RUNNER: kubernetes-job
CASSANDRA_PUBLIC_URL: https://cassandra.bstein.dev
CASSANDRA_PUBLIC_BASE_URL: https://cassandra.bstein.dev
CASSANDRA_ALPHA_LABEL: ALPHA
CASSANDRA_OIDC_ISSUER_URL: https://sso.bstein.dev/realms/cassandra
CASSANDRA_OIDC_ISSUER: https://sso.bstein.dev/realms/cassandra
CASSANDRA_OIDC_CLIENT_ID: cassandra-web
CASSANDRA_OIDC_ALLOWED_GROUPS: alpha,cassandra-tester,admin
CASSANDRA_OIDC_REQUIRED_GROUPS: alpha,cassandra-tester,admin
CASSANDRA_OIDC_ADMIN_GROUPS: admin
CASSANDRA_NAMESPACE: cassandra
CASSANDRA_SIM_IMAGE: registry.bstein.dev/cassandra/cassandra-sim-worker:0.7.65 # {"$imagepolicy": "cassandra:cassandra-sim-worker"}
CASSANDRA_SIM_WORKER_VERSION: "0.7.65"
CASSANDRA_GENERATOR_WORKER_VERSION: "0.7.65"
CASSANDRA_SIM_SERVICE_ACCOUNT: cassandra-sim
CASSANDRA_SIM_PRIORITY_CLASS: cassandra-sim
CASSANDRA_SIM_NODE_SELECTOR: cassandra.bstein.dev/node-pool=oceanus,kubernetes.io/arch=amd64
CASSANDRA_SIM_TOLERATIONS: veles.bstein.dev/simulation=true:NoSchedule
CASSANDRA_SIM_ACTIVE_DEADLINE_SECONDS: "7200"
CASSANDRA_SIM_TTL_SECONDS: "3600"
CASSANDRA_SIM_CPU_REQUEST: 500m
CASSANDRA_SIM_CPU_LIMIT: "2"
CASSANDRA_SIM_MEMORY_REQUEST: 1Gi
CASSANDRA_SIM_MEMORY_LIMIT: 4Gi
CASSANDRA_SIM_FS_GROUP: "10001"
CASSANDRA_MAX_ACTIVE_SIMS_PER_USER: "32"
CASSANDRA_MAX_ACTIVE_SIMS_GLOBAL: "32"
CASSANDRA_RETENTION_DAYS: "30"
CASSANDRA_LOG_ROOT: /data/cassandra-artifacts/logs
CASSANDRA_REPORT_ROOT: /data/cassandra-artifacts/reports
CASSANDRA_ARTIFACT_ROOT: /data/cassandra-artifacts/artifacts
CASSANDRA_SIM_ARTIFACT_PVC: cassandra-artifacts
CASSANDRA_SIM_ARTIFACT_MOUNT_PATH: /data/cassandra-artifacts
CASSANDRA_SIM_CODEX_CLI_SECRET_PROVIDER_CLASS: cassandra-codex-cli
CASSANDRA_SIM_CODEX_CLI_MOUNT_PATH: /var/run/cassandra-codex-cli
CASSANDRA_SIM_CODEX_CLI_HOME: /home/cassandra/.codex

80
services/cassandra/frontend-deployment.yaml Normal file
View File

@ -0,0 +1,80 @@
# services/cassandra/frontend-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: cassandra-frontend
namespace: cassandra
labels:
app: cassandra-frontend
spec:
replicas: 2
revisionHistoryLimit: 2
selector:
matchLabels:
app: cassandra-frontend
template:
metadata:
labels:
app: cassandra-frontend
spec:
serviceAccountName: cassandra-frontend
priorityClassName: cassandra-core
automountServiceAccountToken: false
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: node-role.kubernetes.io/worker
operator: Exists
- key: hardware
operator: In
values: ["rpi5", "rpi4", "amd64"]
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 100
preference:
matchExpressions:
- key: atlas.bstein.dev/spillover
operator: DoesNotExist
- weight: 90
preference:
matchExpressions:
- key: hardware
operator: In
values: ["rpi5"]
securityContext:
fsGroup: 101
seccompProfile:
type: RuntimeDefault
containers:
- name: frontend
image: registry.bstein.dev/cassandra/cassandra-frontend:0.7.65 # {"$imagepolicy": "cassandra:cassandra-frontend"}
imagePullPolicy: Always
ports:
- name: http
containerPort: 8080
protocol: TCP
readinessProbe:
httpGet:
path: /
port: http
initialDelaySeconds: 3
periodSeconds: 10
livenessProbe:
httpGet:
path: /
port: http
initialDelaySeconds: 20
periodSeconds: 20
resources:
requests:
cpu: 50m
memory: 64Mi
limits:
cpu: 250m
memory: 256Mi
securityContext:
runAsNonRoot: true
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]

84
services/cassandra/generator-deployment.yaml Normal file
View File

@ -0,0 +1,84 @@
# services/cassandra/generator-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: cassandra-generator-worker
namespace: cassandra
labels:
app: cassandra-generator-worker
spec:
replicas: 0
revisionHistoryLimit: 2
selector:
matchLabels:
app: cassandra-generator-worker
template:
metadata:
labels:
app: cassandra-generator-worker
annotations:
cassandra.bstein.dev/redeploy: "2026-07-24-0.7.65-hygiene-generator-worker"
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "cassandra"
vault.hashicorp.com/agent-inject-secret-cassandra-env.sh: "kv/data/atlas/cassandra/cassandra-db"
vault.hashicorp.com/agent-inject-template-cassandra-env.sh: |
{{- with secret "kv/data/atlas/cassandra/cassandra-db" }}
unset DATABASE_URL
export CASSANDRA_DATABASE_HOST="{{ .Data.data.POSTGRES_HOST }}"
export CASSANDRA_DATABASE_PORT="{{ .Data.data.POSTGRES_PORT }}"
export CASSANDRA_DATABASE_NAME="{{ .Data.data.POSTGRES_DB }}"
export CASSANDRA_DATABASE_USER="{{ .Data.data.POSTGRES_USER }}"
export CASSANDRA_DATABASE_PASSWORD="{{ .Data.data.POSTGRES_PASSWORD }}"
export CASSANDRA_DATABASE_SSLMODE="disable"
{{- end }}
{{- with secret "kv/data/atlas/cassandra/app-secrets" }}
export CASSANDRA_SESSION_SECRET="{{ .Data.data.CASSANDRA_SESSION_SECRET }}"
export CASSANDRA_BYOK_ENCRYPTION_KEY="{{ .Data.data.CASSANDRA_BYOK_ENCRYPTION_KEY }}"
{{- end }}
spec:
serviceAccountName: cassandra-generator
priorityClassName: cassandra-core
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
kubernetes.io/arch: amd64
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
securityContext:
fsGroup: 10001
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
containers:
- name: generator
image: registry.bstein.dev/cassandra/cassandra-generator-worker:0.7.65 # {"$imagepolicy": "cassandra:cassandra-generator-worker"}
imagePullPolicy: Always
command: ["/bin/sh", "-c"]
args:
- |
set -eu
if [ -f /vault/secrets/cassandra-env.sh ]; then
. /vault/secrets/cassandra-env.sh
fi
exec cassandra-generation-worker --loop --idle-sleep-seconds 5
envFrom:
- configMapRef:
name: cassandra-config
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: "1"
memory: 2Gi
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]

92
services/cassandra/image.yaml Normal file
View File

@ -0,0 +1,92 @@
# services/cassandra/image.yaml
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageRepository
metadata:
name: cassandra-backend
namespace: cassandra
spec:
image: registry.bstein.dev/cassandra/cassandra-backend
interval: 1m0s
secretRef:
name: harbor-regcred
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImagePolicy
metadata:
name: cassandra-backend
namespace: cassandra
spec:
imageRepositoryRef:
name: cassandra-backend
policy:
semver:
range: ">=0.1.0-0"
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageRepository
metadata:
name: cassandra-frontend
namespace: cassandra
spec:
image: registry.bstein.dev/cassandra/cassandra-frontend
interval: 1m0s
secretRef:
name: harbor-regcred
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImagePolicy
metadata:
name: cassandra-frontend
namespace: cassandra
spec:
imageRepositoryRef:
name: cassandra-frontend
policy:
semver:
range: ">=0.1.0-0"
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageRepository
metadata:
name: cassandra-generator-worker
namespace: cassandra
spec:
image: registry.bstein.dev/cassandra/cassandra-generator-worker
interval: 1m0s
secretRef:
name: harbor-regcred
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImagePolicy
metadata:
name: cassandra-generator-worker
namespace: cassandra
spec:
imageRepositoryRef:
name: cassandra-generator-worker
policy:
semver:
range: ">=0.1.0-0"
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImageRepository
metadata:
name: cassandra-sim-worker
namespace: cassandra
spec:
image: registry.bstein.dev/cassandra/cassandra-sim-worker
interval: 1m0s
secretRef:
name: harbor-regcred
---
apiVersion: image.toolkit.fluxcd.io/v1
kind: ImagePolicy
metadata:
name: cassandra-sim-worker
namespace: cassandra
spec:
imageRepositoryRef:
name: cassandra-sim-worker
policy:
semver:
range: ">=0.1.0-0"

47
services/cassandra/ingress.yaml Normal file
View File

@ -0,0 +1,47 @@
# services/cassandra/ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: cassandra
namespace: cassandra
annotations:
cert-manager.io/cluster-issuer: letsencrypt
traefik.ingress.kubernetes.io/router.entrypoints: websecure
traefik.ingress.kubernetes.io/router.tls: "true"
spec:
ingressClassName: traefik
tls:
- hosts: ["cassandra.bstein.dev"]
secretName: cassandra-tls
rules:
- host: cassandra.bstein.dev
http:
paths:
- path: /api
pathType: Prefix
backend:
service:
name: cassandra-backend
port:
number: 80
- path: /events
pathType: Prefix
backend:
service:
name: cassandra-backend
port:
number: 80
- path: /ws
pathType: Prefix
backend:
service:
name: cassandra-backend
port:
number: 80
- path: /
pathType: Prefix
backend:
service:
name: cassandra-frontend
port:
number: 80

26
services/cassandra/kustomization.yaml Normal file
View File

@ -0,0 +1,26 @@
# services/cassandra/kustomization.yaml
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: cassandra
resources:
- namespace.yaml
- serviceaccounts.yaml
- secretproviderclass.yaml
- vault-sync-deployment.yaml
- resourcequota.yaml
- limitrange.yaml
- configmap.yaml
- rbac.yaml
- artifacts-pvc.yaml
- postgres-service.yaml
- postgres-statefulset.yaml
- services.yaml
- backend-deployment.yaml
- generator-deployment.yaml
- frontend-deployment.yaml
- retention-cronjob.yaml
- image.yaml
- ingress.yaml
- bootstrap-jobs/cassandra-secrets-ensure-job.yaml
- migration-jobs/db-copy-veles-to-cassandra-job.yaml
- migration-jobs/artifact-copy-veles-to-cassandra-job.yaml

21
services/cassandra/limitrange.yaml Normal file
View File

@ -0,0 +1,21 @@
# services/cassandra/limitrange.yaml
apiVersion: v1
kind: LimitRange
metadata:
name: cassandra-container-limits
namespace: cassandra
spec:
limits:
- type: Container
defaultRequest:
cpu: 100m
memory: 256Mi
default:
cpu: 500m
memory: 512Mi
min:
cpu: 10m
memory: 32Mi
max:
cpu: "16"
memory: 32Gi

150
services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml Normal file
View File

@ -0,0 +1,150 @@
# services/cassandra/migration-jobs/artifact-copy-veles-to-cassandra-job.yaml
# Suspended by default. Run only after scaling the Veles artifact exporter to 1.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-artifact-copy-from-veles-1
namespace: cassandra
spec:
suspend: true
backoffLimit: 0
activeDeadlineSeconds: 43200
ttlSecondsAfterFinished: 86400
template:
spec:
serviceAccountName: cassandra-artifact-migration
automountServiceAccountToken: false
restartPolicy: Never
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
containers:
- name: copy-artifacts
image: python:3.12-alpine
imagePullPolicy: IfNotPresent
env:
- name: SOURCE_URL
value: http://veles-artifact-export-for-cassandra.veles.svc.cluster.local:8765
- name: DEST_ROOT
value: /dest
- name: ALLOW_NONEMPTY_DEST
value: "0"
command: ["python", "-c"]
args:
- |
from __future__ import annotations
from pathlib import Path
import hashlib
import json
import os
import tarfile
import urllib.request
SOURCE_URL = os.environ["SOURCE_URL"].rstrip("/")
DEST = Path(os.environ.get("DEST_ROOT", "/dest")).resolve()
ALLOW_NONEMPTY = os.environ.get("ALLOW_NONEMPTY_DEST", "") in {"1", "true", "yes"}
EXCLUDE_TOP = {"lost+found", ".cassandra-artifact-migration"}
REPORT_DIR = DEST / ".cassandra-artifact-migration"
def visible_entries() -> list[Path]:
return [entry for entry in DEST.iterdir() if entry.name not in EXCLUDE_TOP]
def assert_safe_destination() -> None:
DEST.mkdir(parents=True, exist_ok=True)
entries = visible_entries()
if entries and not ALLOW_NONEMPTY:
names = ", ".join(entry.name for entry in entries[:10])
raise SystemExit(f"destination is not empty; refusing to merge into existing data: {names}")
REPORT_DIR.mkdir(parents=True, exist_ok=True)
def fetch_manifest() -> dict[str, object]:
with urllib.request.urlopen(f"{SOURCE_URL}/manifest.json", timeout=120) as response:
return json.loads(response.read().decode())
def safe_extract() -> None:
root_text = str(DEST)
with urllib.request.urlopen(f"{SOURCE_URL}/archive.tar", timeout=600) as response:
with tarfile.open(fileobj=response, mode="r|*") as archive:
for member in archive:
target = (DEST / member.name).resolve()
if target != DEST and not str(target).startswith(root_text + os.sep):
raise SystemExit(f"unsafe archive path: {member.name}")
archive.extract(member, path=DEST)
def digest_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def destination_manifest() -> dict[str, object]:
files = []
total_bytes = 0
for path in sorted(DEST.rglob("*")):
if not path.is_file():
continue
rel = path.relative_to(DEST).as_posix()
if rel.split("/", 1)[0] in EXCLUDE_TOP:
continue
size = path.stat().st_size
total_bytes += size
files.append({"path": rel, "size": size, "sha256": digest_file(path)})
manifest_text = "\n".join(
f"{item['sha256']} {item['size']} {item['path']}" for item in files
)
return {
"schema": "cassandra.artifact_manifest.v1",
"source": "cassandra-artifacts",
"file_count": len(files),
"total_bytes": total_bytes,
"manifest_sha256": hashlib.sha256(manifest_text.encode()).hexdigest(),
"files": files,
}
def comparable(manifest: dict[str, object]) -> dict[str, object]:
return {
"file_count": manifest.get("file_count"),
"total_bytes": manifest.get("total_bytes"),
"manifest_sha256": manifest.get("manifest_sha256"),
"files": manifest.get("files"),
}
assert_safe_destination()
source = fetch_manifest()
(REPORT_DIR / "source_manifest.json").write_text(json.dumps(source, indent=2, sort_keys=True))
safe_extract()
dest = destination_manifest()
(REPORT_DIR / "destination_manifest.json").write_text(json.dumps(dest, indent=2, sort_keys=True))
summary = {
"source_file_count": source.get("file_count"),
"destination_file_count": dest.get("file_count"),
"source_total_bytes": source.get("total_bytes"),
"destination_total_bytes": dest.get("total_bytes"),
"source_manifest_sha256": source.get("manifest_sha256"),
"destination_manifest_sha256": dest.get("manifest_sha256"),
"verified": comparable(source) == comparable(dest),
}
(REPORT_DIR / "copy_summary.json").write_text(json.dumps(summary, indent=2, sort_keys=True))
if not summary["verified"]:
raise SystemExit(f"artifact copy verification failed: {summary}")
print(json.dumps(summary, sort_keys=True), flush=True)
resources:
requests:
cpu: 500m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
volumeMounts:
- name: artifacts
mountPath: /dest
volumes:
- name: artifacts
persistentVolumeClaim:
claimName: cassandra-artifacts

120
services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml Normal file
View File

@ -0,0 +1,120 @@
# services/cassandra/migration-jobs/db-copy-veles-to-cassandra-job.yaml
# Suspended by default. Unsuspend after cassandra-postgres is ready and before
# enabling generator/retention workers. This copies from Veles to Cassandra and
# verifies table sets, row counts, and per-table content hashes.
apiVersion: batch/v1
kind: Job
metadata:
name: cassandra-db-copy-from-veles-1
namespace: cassandra
spec:
suspend: true
backoffLimit: 0
ttlSecondsAfterFinished: 86400
template:
metadata:
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-init-first: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "cassandra-migration"
vault.hashicorp.com/agent-inject-secret-source-db.sh: "kv/data/atlas/veles/veles-db"
vault.hashicorp.com/agent-inject-template-source-db.sh: |
{{- with secret "kv/data/atlas/veles/veles-db" }}
export SOURCE_DATABASE_URL="{{ .Data.data.DATABASE_URL }}"
{{- end }}
vault.hashicorp.com/agent-inject-secret-target-db.sh: "kv/data/atlas/cassandra/cassandra-db"
vault.hashicorp.com/agent-inject-template-target-db.sh: |
{{- with secret "kv/data/atlas/cassandra/cassandra-db" }}
export TARGET_DATABASE_URL="{{ .Data.data.DATABASE_URL }}"
{{- end }}
spec:
serviceAccountName: cassandra-db-migration
restartPolicy: Never
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
kubernetes.io/arch: amd64
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
containers:
- name: copy-and-verify
image: postgres:15
command: ["/bin/bash", "-c"]
args:
- |
set -euo pipefail
. /vault/secrets/source-db.sh
. /vault/secrets/target-db.sh
: "${SOURCE_DATABASE_URL:?missing source database url}"
: "${TARGET_DATABASE_URL:?missing target database url}"
export PGCONNECT_TIMEOUT=10
for attempt in $(seq 1 60); do
if pg_isready -d "${SOURCE_DATABASE_URL}" >/dev/null 2>&1 \
&& pg_isready -d "${TARGET_DATABASE_URL}" >/dev/null 2>&1; then
break
fi
if [ "${attempt}" = "60" ]; then
echo "source or target database did not become ready" >&2
exit 1
fi
sleep 5
done
dump_path=/tmp/veles-to-cassandra.dump
pg_dump --format=custom --no-owner --no-acl \
--dbname="${SOURCE_DATABASE_URL}" \
--file="${dump_path}"
pg_restore --clean --if-exists --no-owner --no-acl \
--dbname="${TARGET_DATABASE_URL}" \
"${dump_path}"
table_sql="select schemaname || E'\t' || tablename from pg_tables where schemaname not in ('pg_catalog','information_schema') order by 1"
psql -XAt --dbname="${SOURCE_DATABASE_URL}" --command="${table_sql}" > /tmp/source.tables
psql -XAt --dbname="${TARGET_DATABASE_URL}" --command="${table_sql}" > /tmp/target.tables
diff -u /tmp/source.tables /tmp/target.tables
checksum_sql() {
schema="$1"
table="$2"
printf '%s\n' \
"SELECT format(" \
" 'select count(*)::text || E''\t'' || coalesce(md5(string_agg(md5(row_to_json(t)::text), '''' order by md5(row_to_json(t)::text))), md5('''')) from %I.%I t'," \
" :'schema'," \
" :'table'" \
")\\gexec" > /tmp/checksum.sql
psql -XAt --set=ON_ERROR_STOP=1 \
--set=schema="${schema}" \
--set=table="${table}" \
--dbname="$3" \
--file=/tmp/checksum.sql
}
failures=0
while IFS="$(printf '\t')" read -r schema table; do
[ -n "${schema}" ] || continue
source_sum="$(checksum_sql "${schema}" "${table}" "${SOURCE_DATABASE_URL}")"
target_sum="$(checksum_sql "${schema}" "${table}" "${TARGET_DATABASE_URL}")"
if [ "${source_sum}" != "${target_sum}" ]; then
echo "checksum mismatch for ${schema}.${table}: source=${source_sum} target=${target_sum}" >&2
failures=$((failures + 1))
else
echo "verified ${schema}.${table}: ${source_sum}"
fi
done < /tmp/source.tables
if [ "${failures}" -ne 0 ]; then
echo "database copy integrity verification failed for ${failures} table(s)" >&2
exit 1
fi
echo "database copy integrity verification passed"
resources:
requests:
cpu: 500m
memory: 1Gi
limits:
cpu: "2"
memory: 4Gi

8
services/cassandra/namespace.yaml Normal file
View File

@ -0,0 +1,8 @@
# services/cassandra/namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: cassandra
labels:
app.kubernetes.io/name: cassandra
app.kubernetes.io/part-of: cassandra

17
services/cassandra/postgres-service.yaml Normal file
View File

@ -0,0 +1,17 @@
# services/cassandra/postgres-service.yaml
apiVersion: v1
kind: Service
metadata:
name: cassandra-postgres
namespace: cassandra
labels:
app: cassandra-postgres
spec:
clusterIP: None
ports:
- name: postgres
port: 5432
protocol: TCP
targetPort: 5432
selector:
app: cassandra-postgres

86
services/cassandra/postgres-statefulset.yaml Normal file
View File

@ -0,0 +1,86 @@
# services/cassandra/postgres-statefulset.yaml
apiVersion: apps/v1
kind: StatefulSet
metadata:
name: cassandra-postgres
namespace: cassandra
labels:
app: cassandra-postgres
spec:
serviceName: cassandra-postgres
replicas: 1
selector:
matchLabels:
app: cassandra-postgres
persistentVolumeClaimRetentionPolicy:
whenDeleted: Retain
whenScaled: Retain
updateStrategy:
type: RollingUpdate
template:
metadata:
labels:
app: cassandra-postgres
annotations:
vault.hashicorp.com/agent-inject: "true"
vault.hashicorp.com/agent-pre-populate-only: "true"
vault.hashicorp.com/role: "cassandra"
vault.hashicorp.com/agent-inject-secret-postgres-password: "kv/data/atlas/cassandra/cassandra-db"
vault.hashicorp.com/agent-inject-template-postgres-password: |
{{- with secret "kv/data/atlas/cassandra/cassandra-db" -}}
{{ .Data.data.POSTGRES_PASSWORD }}
{{- end -}}
spec:
serviceAccountName: cassandra-postgres
priorityClassName: cassandra-core
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
securityContext:
fsGroup: 999
seccompProfile:
type: RuntimeDefault
containers:
- name: postgres
image: postgres:15
ports:
- name: postgres
containerPort: 5432
protocol: TCP
env:
- name: PGDATA
value: /var/lib/postgresql/data/pgdata
- name: POSTGRES_USER
value: cassandra
- name: POSTGRES_PASSWORD_FILE
value: /vault/secrets/postgres-password
- name: POSTGRES_DB
value: cassandra
resources:
requests:
cpu: "2"
memory: 8Gi
limits:
cpu: "4"
memory: 16Gi
securityContext:
allowPrivilegeEscalation: false
volumeMounts:
- name: postgres-data
mountPath: /var/lib/postgresql/data
volumeClaimTemplates:
- metadata:
name: postgres-data
labels:
app: cassandra-postgres
cassandra.bstein.dev/backup: longhorn
spec:
accessModes: ["ReadWriteOnce"]
storageClassName: cassandra-db
resources:
requests:
storage: 100Gi

36
services/cassandra/rbac.yaml Normal file
View File

@ -0,0 +1,36 @@
# services/cassandra/rbac.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: cassandra-backend-jobs
namespace: cassandra
rules:
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["create", "delete", "deletecollection", "get", "list", "patch", "watch"]
- apiGroups: [""]
resources: ["pods"]
verbs: ["delete", "get", "list", "watch"]
- apiGroups: [""]
resources: ["pods/log"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["get", "list", "watch"]
- apiGroups: ["events.k8s.io"]
resources: ["events"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: cassandra-backend-jobs
namespace: cassandra
subjects:
- kind: ServiceAccount
name: cassandra-backend
namespace: cassandra
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: cassandra-backend-jobs

54
services/cassandra/resourcequota.yaml Normal file
View File

@ -0,0 +1,54 @@
# services/cassandra/resourcequota.yaml
apiVersion: v1
kind: ResourceQuota
metadata:
name: cassandra-namespace-quota
namespace: cassandra
spec:
hard:
requests.cpu: "32"
requests.memory: 64Gi
limits.cpu: "96"
limits.memory: 192Gi
pods: "96"
count/jobs.batch: "100"
persistentvolumeclaims: "8"
requests.storage: 1Ti
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: cassandra-core-quota
namespace: cassandra
spec:
hard:
requests.cpu: "4"
requests.memory: 16Gi
limits.cpu: "8"
limits.memory: 32Gi
pods: "12"
scopeSelector:
matchExpressions:
- scopeName: PriorityClass
operator: In
values:
- cassandra-core
---
apiVersion: v1
kind: ResourceQuota
metadata:
name: cassandra-sim-quota
namespace: cassandra
spec:
hard:
requests.cpu: "20"
requests.memory: 40Gi
limits.cpu: "80"
limits.memory: 160Gi
pods: "80"
scopeSelector:
matchExpressions:
- scopeName: PriorityClass
operator: In
values:
- cassandra-sim

68
services/cassandra/retention-cronjob.yaml Normal file
View File

@ -0,0 +1,68 @@
# services/cassandra/retention-cronjob.yaml
apiVersion: batch/v1
kind: CronJob
metadata:
name: cassandra-retention
namespace: cassandra
labels:
app: cassandra-retention
spec:
suspend: true
schedule: "17 4 * * *"
concurrencyPolicy: Forbid
successfulJobsHistoryLimit: 1
failedJobsHistoryLimit: 3
jobTemplate:
spec:
template:
metadata:
labels:
app: cassandra-retention
spec:
restartPolicy: Never
serviceAccountName: cassandra-retention
automountServiceAccountToken: false
priorityClassName: cassandra-core
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
kubernetes.io/arch: amd64
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
securityContext:
fsGroup: 10001
fsGroupChangePolicy: OnRootMismatch
seccompProfile:
type: RuntimeDefault
containers:
- name: retention
image: registry.bstein.dev/cassandra/cassandra-backend:0.7.65 # {"$imagepolicy": "cassandra:cassandra-backend"}
imagePullPolicy: IfNotPresent
command: ["cassandra-hygiene"]
args: ["--prune-artifacts"]
envFrom:
- configMapRef:
name: cassandra-config
volumeMounts:
- name: artifacts
mountPath: /data/cassandra-artifacts
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 250m
memory: 512Mi
securityContext:
runAsNonRoot: true
runAsUser: 10001
runAsGroup: 10001
allowPrivilegeEscalation: false
capabilities:
drop: ["ALL"]
volumes:
- name: artifacts
persistentVolumeClaim:
claimName: cassandra-artifacts

70
services/cassandra/secretproviderclass.yaml Normal file
View File

@ -0,0 +1,70 @@
# services/cassandra/secretproviderclass.yaml
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: cassandra-vault
namespace: cassandra
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "cassandra"
objects: |
- objectName: "harbor-pull__dockerconfigjson"
secretPath: "kv/data/atlas/shared/harbor-pull"
secretKey: "dockerconfigjson"
- objectName: "cassandra-db__database-url"
secretPath: "kv/data/atlas/cassandra/cassandra-db"
secretKey: "DATABASE_URL"
- objectName: "cassandra-db__user"
secretPath: "kv/data/atlas/cassandra/cassandra-db"
secretKey: "POSTGRES_USER"
- objectName: "cassandra-db__password"
secretPath: "kv/data/atlas/cassandra/cassandra-db"
secretKey: "POSTGRES_PASSWORD"
- objectName: "cassandra-oidc__client-secret"
secretPath: "kv/data/atlas/cassandra/cassandra-oidc"
secretKey: "client_secret"
- objectName: "cassandra-app__session-secret"
secretPath: "kv/data/atlas/cassandra/app-secrets"
secretKey: "CASSANDRA_SESSION_SECRET"
- objectName: "cassandra-app__byok-encryption-key"
secretPath: "kv/data/atlas/cassandra/app-secrets"
secretKey: "CASSANDRA_BYOK_ENCRYPTION_KEY"
secretObjects:
- secretName: harbor-regcred
type: kubernetes.io/dockerconfigjson
data:
- objectName: harbor-pull__dockerconfigjson
key: .dockerconfigjson
- secretName: cassandra-runtime-secrets
type: Opaque
data:
- objectName: cassandra-db__database-url
key: DATABASE_URL
- objectName: cassandra-db__user
key: CASSANDRA_DATABASE_USER
- objectName: cassandra-db__password
key: CASSANDRA_DATABASE_PASSWORD
- objectName: cassandra-oidc__client-secret
key: CASSANDRA_OIDC_CLIENT_SECRET
- objectName: cassandra-app__session-secret
key: CASSANDRA_SESSION_SECRET
- objectName: cassandra-app__byok-encryption-key
key: CASSANDRA_BYOK_ENCRYPTION_KEY
---
apiVersion: secrets-store.csi.x-k8s.io/v1
kind: SecretProviderClass
metadata:
name: cassandra-codex-cli
namespace: cassandra
spec:
provider: vault
parameters:
vaultAddress: "http://vault.vault.svc.cluster.local:8200"
roleName: "cassandra-sim"
objects: |
- objectName: "auth.json"
secretPath: "kv/data/atlas/cassandra/codex-cli"
secretKey: "auth.json"
filePermission: 292

76
services/cassandra/serviceaccounts.yaml Normal file
View File

@ -0,0 +1,76 @@
# services/cassandra/serviceaccounts.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-backend
namespace: cassandra
imagePullSecrets:
- name: harbor-regcred
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-generator
namespace: cassandra
imagePullSecrets:
- name: harbor-regcred
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-frontend
namespace: cassandra
automountServiceAccountToken: false
imagePullSecrets:
- name: harbor-regcred
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-postgres
namespace: cassandra
imagePullSecrets:
- name: harbor-regcred
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-retention
namespace: cassandra
automountServiceAccountToken: false
imagePullSecrets:
- name: harbor-regcred
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-vault-sync
namespace: cassandra
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-secrets-ensure
namespace: cassandra
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-db-migration
namespace: cassandra
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-artifact-migration
namespace: cassandra
automountServiceAccountToken: false
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: cassandra-sim
namespace: cassandra
automountServiceAccountToken: false
imagePullSecrets:
- name: harbor-regcred

32
services/cassandra/services.yaml Normal file
View File

@ -0,0 +1,32 @@
# services/cassandra/services.yaml
apiVersion: v1
kind: Service
metadata:
name: cassandra-backend
namespace: cassandra
labels:
app: cassandra-backend
spec:
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8796
selector:
app: cassandra-backend
---
apiVersion: v1
kind: Service
metadata:
name: cassandra-frontend
namespace: cassandra
labels:
app: cassandra-frontend
spec:
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
selector:
app: cassandra-frontend

43
services/cassandra/vault-sync-deployment.yaml Normal file
View File

@ -0,0 +1,43 @@
# services/cassandra/vault-sync-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: cassandra-vault-sync
namespace: cassandra
labels:
app: cassandra-vault-sync
spec:
replicas: 1
selector:
matchLabels:
app: cassandra-vault-sync
template:
metadata:
labels:
app: cassandra-vault-sync
spec:
serviceAccountName: cassandra-vault-sync
containers:
- name: sync
image: alpine:3.20
command: ["/bin/sh", "-c"]
args:
- "sleep infinity"
resources:
requests:
cpu: 10m
memory: 32Mi
limits:
cpu: 50m
memory: 64Mi
volumeMounts:
- name: vault-secrets
mountPath: /vault/secrets
readOnly: true
volumes:
- name: vault-secrets
csi:
driver: secrets-store.csi.k8s.io
readOnly: true
volumeAttributes:
secretProviderClass: cassandra-vault

11
services/vault/scripts/vault_k8s_auth_configure.sh
View File

@ -260,6 +260,15 @@ write_policy_and_role "veles-sim" "veles" "veles-sim" \
write_policy_and_role "veles-secrets" "veles" "veles-secrets-ensure" \
"shared/postmark-relay" \
"veles/*"
write_policy_and_role "cassandra" "cassandra" "cassandra-backend,cassandra-generator,cassandra-postgres,cassandra-vault-sync" \
"cassandra/* shared/harbor-pull shared/postmark-relay" ""
write_policy_and_role "cassandra-sim" "cassandra" "cassandra-sim" \
"cassandra/codex-cli" ""
write_policy_and_role "cassandra-secrets" "cassandra" "cassandra-secrets-ensure" \
"shared/postmark-relay" \
"cassandra/*"
write_policy_and_role "cassandra-migration" "cassandra" "cassandra-db-migration" \
"veles/veles-db cassandra/cassandra-db" ""
write_policy_and_role "maintenance" "maintenance" "ariadne,maintenance-vault-sync,metis" \
"maintenance/ariadne-db maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys maintenance/metis-runtime portal/atlas-portal-db portal/bstein-dev-home-keycloak-admin mailu/mailu-db-secret mailu/mailu-initial-account-secret nextcloud/nextcloud-db nextcloud/nextcloud-admin health/wger-admin finance/firefly-secrets comms/mas-admin-client-runtime comms/atlasbot-credentials-runtime comms/synapse-db comms/synapse-admin vault/vault-oidc-config shared/harbor-pull shared/soteria-restic harbor/harbor-core" "" \
'
@ -287,7 +296,7 @@ write_policy_and_role "vault" "vault" "vault" \
write_policy_and_role "sso-secrets" "sso" "mas-secrets-ensure" \
"shared/keycloak-admin shared/postmark-relay maintenance/metis-ssh-keys" \
"harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc gitea/gitea-veles-oidc" \
"harbor/harbor-oidc vault/vault-oidc-config comms/synapse-oidc logging/oauth2-proxy-logs-oidc finance/actual-oidc maintenance/metis-oidc maintenance/soteria-oidc maintenance/metis-ssh-keys veles/veles-oidc cassandra/cassandra-oidc gitea/gitea-veles-oidc gitea/gitea-cassandra-oidc" \
'
path "kv/data/atlas/nodes/*" {
capabilities = ["create", "update", "read"]

1
services/veles/kustomization.yaml
View File

@ -22,3 +22,4 @@ resources:
- image.yaml
- ingress.yaml
- bootstrap-jobs/veles-secrets-ensure-job.yaml
- migration-jobs/cassandra-artifact-export.yaml

159
services/veles/migration-jobs/cassandra-artifact-export.yaml Normal file
View File

@ -0,0 +1,159 @@
# services/veles/migration-jobs/cassandra-artifact-export.yaml
# Scale to one only during the Veles -> Cassandra artifact copy, then return to
# zero after the Cassandra import job verifies the copied content.
apiVersion: v1
kind: Service
metadata:
name: veles-artifact-export-for-cassandra
namespace: veles
spec:
selector:
app: veles-artifact-export-for-cassandra
ports:
- name: http
port: 8765
targetPort: http
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: veles-artifact-export-for-cassandra
namespace: veles
spec:
replicas: 0
selector:
matchLabels:
app: veles-artifact-export-for-cassandra
template:
metadata:
labels:
app: veles-artifact-export-for-cassandra
spec:
automountServiceAccountToken: false
nodeSelector:
cassandra.bstein.dev/node-pool: oceanus
tolerations:
- key: veles.bstein.dev/simulation
operator: Equal
value: "true"
effect: NoSchedule
containers:
- name: exporter
image: python:3.12-alpine
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8765
env:
- name: SOURCE_ROOT
value: /source
command: ["python", "-c"]
args:
- |
from __future__ import annotations
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import hashlib
import json
import os
import tarfile
ROOT = Path(os.environ.get("SOURCE_ROOT", "/source")).resolve()
EXCLUDE_TOP = {"lost+found", ".cassandra-artifact-migration"}
def iter_files():
for path in sorted(ROOT.rglob("*")):
if not path.is_file():
continue
rel = path.relative_to(ROOT).as_posix()
if rel.split("/", 1)[0] in EXCLUDE_TOP:
continue
yield path, rel
def digest_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def build_manifest() -> dict[str, object]:
files = []
total_bytes = 0
for path, rel in iter_files():
size = path.stat().st_size
total_bytes += size
files.append({"path": rel, "size": size, "sha256": digest_file(path)})
manifest_text = "\n".join(
f"{item['sha256']} {item['size']} {item['path']}" for item in files
)
return {
"schema": "cassandra.artifact_manifest.v1",
"source": "veles-artifacts",
"file_count": len(files),
"total_bytes": total_bytes,
"manifest_sha256": hashlib.sha256(manifest_text.encode()).hexdigest(),
"files": files,
}
MANIFEST = build_manifest()
MANIFEST_PATHS = [str(item["path"]) for item in MANIFEST["files"]]
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
if self.path == "/healthz":
self.send_response(200)
self.end_headers()
self.wfile.write(b"ok\n")
return
if self.path == "/manifest.json":
payload = json.dumps(MANIFEST, sort_keys=True).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
if self.path == "/archive.tar":
self.send_response(200)
self.send_header("Content-Type", "application/x-tar")
self.end_headers()
with tarfile.open(fileobj=self.wfile, mode="w|") as archive:
for rel in MANIFEST_PATHS:
path = ROOT / rel
if not path.is_file():
raise FileNotFoundError(rel)
with path.open("rb") as handle:
info = archive.gettarinfo(str(path), arcname=rel)
archive.addfile(info, handle)
return
self.send_response(404)
self.end_headers()
def log_message(self, fmt: str, *args: object) -> None:
print(fmt % args, flush=True)
print(
"ready file_count={file_count} total_bytes={total_bytes} manifest={manifest_sha256}".format(
**MANIFEST
),
flush=True,
)
ThreadingHTTPServer(("0.0.0.0", 8765), Handler).serve_forever()
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "2"
memory: 2Gi
volumeMounts:
- name: artifacts
mountPath: /source
readOnly: true
volumes:
- name: artifacts
persistentVolumeClaim:
claimName: veles-artifacts
readOnly: true