hermes(chat): expand isolated user capacity

services/hermes/chat-router.yaml

@@ -67,7 +67,7 @@ spec:
           ports:
             - {name: http, containerPort: 8080, protocol: TCP}
           env:
-            - {name: TENANT_SLOTS, value: "4"}
+            - {name: TENANT_SLOTS, value: "8"}
             - {name: TENANT_STATE_PATH, value: /state/tenants.json}
             - {name: TELEGRAM_CONFIG_PATH, value: /vault/secrets/telegram-config}
           readinessProbe:

services/hermes/chat-sandbox.yaml

@@ -447,3 +447,435 @@ spec:
         - name: tmp
           emptyDir:
             sizeLimit: 256Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-chat-sandbox-4
+  namespace: hermes
+  labels:
+    app: hermes-chat-sandbox
+    ai.bstein.dev/tenant-ordinal: "4"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "4"
+  template:
+    metadata:
+      labels:
+        app: hermes-chat-sandbox
+        ai.bstein.dev/tenant-ordinal: "4"
+      annotations:
+        ai.bstein.dev/role: isolated-user-computation
+        ai.bstein.dev/isolation: credential-free and default-deny network
+    spec:
+      hostname: hermes-chat-sandbox-4
+      subdomain: hermes-chat-sandbox
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: [arm64]
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values: ["true"]
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi5]
+            - weight: 40
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi4]
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: hermes-chat-sandbox
+                topologyKey: kubernetes.io/hostname
+      containers:
+        - name: sandbox
+          image: registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca
+          imagePullPolicy: IfNotPresent
+          ports:
+            - {name: http, containerPort: 9080, protocol: TCP}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: workspace, mountPath: /opt/data/workspace}
+            - {name: tmp, mountPath: /tmp}
+          readinessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 20000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: "1", memory: 1Gi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim:
+            claimName: workspace-hermes-chat-tenant-4
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-chat-sandbox-5
+  namespace: hermes
+  labels:
+    app: hermes-chat-sandbox
+    ai.bstein.dev/tenant-ordinal: "5"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "5"
+  template:
+    metadata:
+      labels:
+        app: hermes-chat-sandbox
+        ai.bstein.dev/tenant-ordinal: "5"
+      annotations:
+        ai.bstein.dev/role: isolated-user-computation
+        ai.bstein.dev/isolation: credential-free and default-deny network
+    spec:
+      hostname: hermes-chat-sandbox-5
+      subdomain: hermes-chat-sandbox
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: [arm64]
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values: ["true"]
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi5]
+            - weight: 40
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi4]
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: hermes-chat-sandbox
+                topologyKey: kubernetes.io/hostname
+      containers:
+        - name: sandbox
+          image: registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca
+          imagePullPolicy: IfNotPresent
+          ports:
+            - {name: http, containerPort: 9080, protocol: TCP}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: workspace, mountPath: /opt/data/workspace}
+            - {name: tmp, mountPath: /tmp}
+          readinessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 20000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: "1", memory: 1Gi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim:
+            claimName: workspace-hermes-chat-tenant-5
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-chat-sandbox-6
+  namespace: hermes
+  labels:
+    app: hermes-chat-sandbox
+    ai.bstein.dev/tenant-ordinal: "6"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "6"
+  template:
+    metadata:
+      labels:
+        app: hermes-chat-sandbox
+        ai.bstein.dev/tenant-ordinal: "6"
+      annotations:
+        ai.bstein.dev/role: isolated-user-computation
+        ai.bstein.dev/isolation: credential-free and default-deny network
+    spec:
+      hostname: hermes-chat-sandbox-6
+      subdomain: hermes-chat-sandbox
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: [arm64]
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values: ["true"]
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi5]
+            - weight: 40
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi4]
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: hermes-chat-sandbox
+                topologyKey: kubernetes.io/hostname
+      containers:
+        - name: sandbox
+          image: registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca
+          imagePullPolicy: IfNotPresent
+          ports:
+            - {name: http, containerPort: 9080, protocol: TCP}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: workspace, mountPath: /opt/data/workspace}
+            - {name: tmp, mountPath: /tmp}
+          readinessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 20000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: "1", memory: 1Gi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim:
+            claimName: workspace-hermes-chat-tenant-6
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hermes-chat-sandbox-7
+  namespace: hermes
+  labels:
+    app: hermes-chat-sandbox
+    ai.bstein.dev/tenant-ordinal: "7"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "7"
+  template:
+    metadata:
+      labels:
+        app: hermes-chat-sandbox
+        ai.bstein.dev/tenant-ordinal: "7"
+      annotations:
+        ai.bstein.dev/role: isolated-user-computation
+        ai.bstein.dev/isolation: credential-free and default-deny network
+    spec:
+      hostname: hermes-chat-sandbox-7
+      subdomain: hermes-chat-sandbox
+      automountServiceAccountToken: false
+      enableServiceLinks: false
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        seccompProfile:
+          type: RuntimeDefault
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: kubernetes.io/arch
+                    operator: In
+                    values: [arm64]
+                  - key: node-role.kubernetes.io/worker
+                    operator: In
+                    values: ["true"]
+                  - key: kubernetes.io/hostname
+                    operator: NotIn
+                    values: [titan-05, titan-08, titan-13, titan-14, titan-17, titan-18, titan-19]
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi5]
+            - weight: 40
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: [rpi4]
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 100
+              podAffinityTerm:
+                labelSelector:
+                  matchLabels:
+                    app: hermes-chat-sandbox
+                topologyKey: kubernetes.io/hostname
+      containers:
+        - name: sandbox
+          image: registry.bstein.dev/bstein/hermes-chat-sandbox@sha256:17ee62b8e61c08573a3a8cca903b38ec43800cb44ec29340e1bc095176544bca
+          imagePullPolicy: IfNotPresent
+          ports:
+            - {name: http, containerPort: 9080, protocol: TCP}
+          volumeMounts:
+            - {name: workspace, mountPath: /workspace}
+            - {name: workspace, mountPath: /opt/data/workspace}
+            - {name: tmp, mountPath: /tmp}
+          readinessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 3
+            periodSeconds: 10
+            timeoutSeconds: 3
+          livenessProbe:
+            httpGet: {path: /health, port: http}
+            initialDelaySeconds: 10
+            periodSeconds: 30
+            timeoutSeconds: 3
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop: [ALL]
+            readOnlyRootFilesystem: true
+            runAsNonRoot: true
+            runAsUser: 20000
+            runAsGroup: 10000
+            seccompProfile:
+              type: RuntimeDefault
+          resources:
+            requests: {cpu: 50m, memory: 128Mi}
+            limits: {cpu: "1", memory: 1Gi}
+      volumes:
+        - name: workspace
+          persistentVolumeClaim:
+            claimName: workspace-hermes-chat-tenant-7
+        - name: tmp
+          emptyDir:
+            sizeLimit: 256Mi

services/hermes/chat-statefulset.yaml

@@ -10,7 +10,7 @@ metadata:
     kustomize.toolkit.fluxcd.io/force: enabled
 spec:
   serviceName: hermes-chat-tenant
-  replicas: 4
+  replicas: 8
   podManagementPolicy: Parallel
   persistentVolumeClaimRetentionPolicy:
     whenDeleted: Retain

services/hermes/networkpolicy.yaml

@@ -207,6 +207,82 @@ spec:
 ---
 apiVersion: networking.k8s.io/v1
 kind: NetworkPolicy
+metadata:
+  name: hermes-chat-sandbox-tenant-4
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "4"
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              statefulset.kubernetes.io/pod-name: hermes-chat-tenant-4
+      ports:
+        - {protocol: TCP, port: 9080}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-chat-sandbox-tenant-5
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "5"
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              statefulset.kubernetes.io/pod-name: hermes-chat-tenant-5
+      ports:
+        - {protocol: TCP, port: 9080}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-chat-sandbox-tenant-6
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "6"
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              statefulset.kubernetes.io/pod-name: hermes-chat-tenant-6
+      ports:
+        - {protocol: TCP, port: 9080}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: hermes-chat-sandbox-tenant-7
+  namespace: hermes
+spec:
+  podSelector:
+    matchLabels:
+      app: hermes-chat-sandbox
+      ai.bstein.dev/tenant-ordinal: "7"
+  policyTypes: [Ingress]
+  ingress:
+    - from:
+        - podSelector:
+            matchLabels:
+              statefulset.kubernetes.io/pod-name: hermes-chat-tenant-7
+      ports:
+        - {protocol: TCP, port: 9080}
+---
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
 metadata:
   name: hermes-chat-tenant-isolation
   namespace: hermes