logging: add opensearch dashboards ui

infrastructure/sources/helm/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - jetstack.yaml
   - jenkins.yaml
   - mailu.yaml
+  - opensearch.yaml
   - harbor.yaml
   - prometheus.yaml
   - victoria-metrics.yaml

infrastructure/sources/helm/opensearch.yaml

@@ -0,0 +1,9 @@
+# infrastructure/sources/helm/opensearch.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: opensearch
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://opensearch-project.github.io/helm-charts

services/keycloak/kustomization.yaml

@@ -18,6 +18,7 @@ resources:
   - user-overrides-job.yaml
   - mas-secrets-ensure-job.yaml
   - synapse-oidc-secret-ensure-job.yaml
+  - logs-oidc-secret-ensure-job.yaml
   - service.yaml
   - ingress.yaml
 generatorOptions:

services/keycloak/logs-oidc-secret-ensure-job.yaml

@@ -0,0 +1,96 @@
+# services/keycloak/logs-oidc-secret-ensure-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: logs-oidc-secret-ensure-1
+  namespace: sso
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      serviceAccountName: mas-secrets-ensure
+      restartPolicy: Never
+      containers:
+        - name: apply
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              apk add --no-cache curl jq kubectl openssl >/dev/null
+
+              KC_URL="http://keycloak.sso.svc.cluster.local"
+              ACCESS_TOKEN=""
+              for attempt in 1 2 3 4 5; do
+                TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
+                  -H 'Content-Type: application/x-www-form-urlencoded' \
+                  -d "grant_type=password" \
+                  -d "client_id=admin-cli" \
+                  -d "username=${KEYCLOAK_ADMIN}" \
+                  -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
+                ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
+                if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
+                  break
+                fi
+                echo "Keycloak token request failed (attempt ${attempt})" >&2
+                sleep $((attempt * 2))
+              done
+              if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+                echo "Failed to fetch Keycloak admin token" >&2
+                exit 1
+              fi
+
+              CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                "$KC_URL/admin/realms/atlas/clients?clientId=logs" || true)"
+              CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+
+              if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+                create_payload='{"clientId":"logs","enabled":true,"protocol":"openid-connect","publicClient":false,"standardFlowEnabled":true,"implicitFlowEnabled":false,"directAccessGrantsEnabled":false,"serviceAccountsEnabled":false,"redirectUris":["https://logs.bstein.dev/oauth2/callback"],"webOrigins":["https://logs.bstein.dev"],"rootUrl":"https://logs.bstein.dev","baseUrl":"/"}'
+                status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
+                  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                  -H 'Content-Type: application/json' \
+                  -d "${create_payload}" \
+                  "$KC_URL/admin/realms/atlas/clients")"
+                if [ "$status" != "201" ] && [ "$status" != "204" ]; then
+                  echo "Keycloak client create failed (status ${status})" >&2
+                  exit 1
+                fi
+                CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                  "$KC_URL/admin/realms/atlas/clients?clientId=logs" || true)"
+                CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+              fi
+
+              if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+                echo "Keycloak client logs not found" >&2
+                exit 1
+              fi
+
+              CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
+              if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
+                echo "Keycloak client secret not found" >&2
+                exit 1
+              fi
+
+              if kubectl -n logging get secret oauth2-proxy-logs-oidc >/dev/null 2>&1; then
+                exit 0
+              fi
+
+              COOKIE_SECRET="$(openssl rand -base64 32 | tr -d '\n')"
+              kubectl -n logging create secret generic oauth2-proxy-logs-oidc \
+                --from-literal=client_id="logs" \
+                --from-literal=client_secret="${CLIENT_SECRET}" \
+                --from-literal=cookie_secret="${COOKIE_SECRET}" \
+                --dry-run=client -o yaml | kubectl -n logging apply -f - >/dev/null
+          env:
+            - name: KEYCLOAK_ADMIN
+              valueFrom:
+                secretKeyRef:
+                  name: keycloak-admin
+                  key: username
+            - name: KEYCLOAK_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: keycloak-admin
+                  key: password

services/logging/fluent-bit-helmrelease.yaml

@@ -84,17 +84,21 @@ spec:
             K8S-Logging.Exclude On
       outputs: |
         [OUTPUT]
-            Name loki
+            Name es
             Match kube.*
-            Host loki.logging.svc.cluster.local
+            Host opensearch-master.logging.svc.cluster.local
-            Port 3100
+            Port 9200
-            labels job=fluent-bit,namespace=$kubernetes['namespace_name'],pod=$kubernetes['pod_name'],container=$kubernetes['container_name']
+            Logstash_Format On
-            line_format json
+            Logstash_Prefix kube
+            Replace_Dots On
+            Suppress_Type_Name On
 
         [OUTPUT]
-            Name loki
+            Name es
             Match journald.*
-            Host loki.logging.svc.cluster.local
+            Host opensearch-master.logging.svc.cluster.local
-            Port 3100
+            Port 9200
-            labels job=systemd
+            Logstash_Format On
-            line_format json
+            Logstash_Prefix journald
+            Replace_Dots On
+            Suppress_Type_Name On

services/logging/ingress.yaml

@@ -2,7 +2,7 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: loki
+  name: logs
   namespace: logging
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt
@@ -20,6 +20,6 @@ spec:
             pathType: Prefix
             backend:
               service:
-                name: oauth2-proxy-loki
+                name: oauth2-proxy-logs
                 port:
                   name: http

services/logging/kustomization.yaml

@@ -3,7 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
-  - loki-helmrelease.yaml
+  - opensearch-helmrelease.yaml
+  - opensearch-dashboards-helmrelease.yaml
+  - opensearch-ism-job.yaml
   - fluent-bit-helmrelease.yaml
+  - loki-helmrelease.yaml
   - oauth2-proxy.yaml
   - ingress.yaml

services/logging/oauth2-proxy.yaml

@@ -2,36 +2,36 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: oauth2-proxy-loki
+  name: oauth2-proxy-logs
   namespace: logging
   labels:
-    app: oauth2-proxy-loki
+    app: oauth2-proxy-logs
 spec:
   ports:
     - name: http
       port: 80
       targetPort: 4180
   selector:
-    app: oauth2-proxy-loki
+    app: oauth2-proxy-logs
 
 ---
 
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: oauth2-proxy-loki
+  name: oauth2-proxy-logs
   namespace: logging
   labels:
-    app: oauth2-proxy-loki
+    app: oauth2-proxy-logs
 spec:
   replicas: 2
   selector:
     matchLabels:
-      app: oauth2-proxy-loki
+      app: oauth2-proxy-logs
   template:
     metadata:
       labels:
-        app: oauth2-proxy-loki
+        app: oauth2-proxy-logs
     spec:
       nodeSelector:
         node-role.kubernetes.io/worker: "true"
@@ -63,7 +63,7 @@ spec:
             - --cookie-refresh=20m
             - --cookie-expire=168h
             - --insecure-oidc-allow-unverified-email=true
-            - --upstream=http://loki-gateway.logging.svc.cluster.local
+            - --upstream=http://opensearch-dashboards.logging.svc.cluster.local:5601
             - --http-address=0.0.0.0:4180
             - --skip-provider-button=true
             - --skip-jwt-bearer-tokens=true
@@ -72,17 +72,17 @@ spec:
             - name: OAUTH2_PROXY_CLIENT_ID
               valueFrom:
                 secretKeyRef:
-                  name: oauth2-proxy-loki-oidc
+                  name: oauth2-proxy-logs-oidc
                   key: client_id
             - name: OAUTH2_PROXY_CLIENT_SECRET
               valueFrom:
                 secretKeyRef:
-                  name: oauth2-proxy-loki-oidc
+                  name: oauth2-proxy-logs-oidc
                   key: client_secret
             - name: OAUTH2_PROXY_COOKIE_SECRET
               valueFrom:
                 secretKeyRef:
-                  name: oauth2-proxy-loki-oidc
+                  name: oauth2-proxy-logs-oidc
                   key: cookie_secret
           ports:
             - containerPort: 4180

services/logging/opensearch-dashboards-helmrelease.yaml

@@ -0,0 +1,46 @@
+# services/logging/opensearch-dashboards-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: opensearch-dashboards
+  namespace: logging
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: opensearch-dashboards
+      version: "~2.32.0"
+      sourceRef:
+        kind: HelmRepository
+        name: opensearch
+        namespace: flux-system
+  values:
+    fullnameOverride: opensearch-dashboards
+    opensearchHosts: "http://opensearch-master.logging.svc.cluster.local:9200"
+    replicaCount: 1
+    config:
+      opensearch_dashboards.yml: |
+        server.host: 0.0.0.0
+        opensearch.hosts: ["http://opensearch-master.logging.svc.cluster.local:9200"]
+        opensearch_security.enabled: false
+    extraEnvs:
+      - name: NODE_OPTIONS
+        value: "--max-old-space-size=512"
+    resources:
+      requests:
+        cpu: "200m"
+        memory: "512Mi"
+      limits:
+        memory: "512Mi"
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+                    - rpi4

services/logging/opensearch-helmrelease.yaml

@@ -0,0 +1,56 @@
+# services/logging/opensearch-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: opensearch
+  namespace: logging
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: opensearch
+      version: "~2.36.0"
+      sourceRef:
+        kind: HelmRepository
+        name: opensearch
+        namespace: flux-system
+  values:
+    fullnameOverride: opensearch
+    clusterName: opensearch
+    nodeGroup: master
+    masterService: opensearch-master
+    singleNode: true
+    replicas: 1
+    minimumMasterNodes: 1
+    opensearchJavaOpts: "-Xms1g -Xmx1g"
+    resources:
+      requests:
+        cpu: "500m"
+        memory: "2Gi"
+      limits:
+        memory: "2Gi"
+    persistence:
+      enabled: true
+      storageClass: asteria
+      size: 500Gi
+    config:
+      opensearch.yml: |
+        cluster.name: opensearch
+        network.host: 0.0.0.0
+        discovery.type: single-node
+        plugins.security.disabled: true
+        node.store.allow_mmap: false
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+                    - rpi4
+    sysctlInit:
+      enabled: true

services/logging/opensearch-ism-job.yaml

@@ -0,0 +1,47 @@
+# services/logging/opensearch-ism-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: opensearch-ism-setup-1
+  namespace: logging
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: apply
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              apk add --no-cache curl >/dev/null
+
+              OS_URL="http://opensearch-master.logging.svc.cluster.local:9200"
+              for attempt in $(seq 1 60); do
+                if curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then
+                  break
+                fi
+                sleep 5
+              done
+              if ! curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then
+                echo "OpenSearch did not become ready in time" >&2
+                exit 1
+              fi
+
+              policy='{"policy":{"description":"Delete logs after 180 days","schema_version":1,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"180d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}'
+              curl -sS -X PUT "${OS_URL}/_plugins/_ism/policies/logging-180d" \
+                -H 'Content-Type: application/json' \
+                -d "${policy}" >/dev/null
+
+              kube_template='{"index_patterns":["kube-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
+              curl -sS -X PUT "${OS_URL}/_index_template/kube-logs" \
+                -H 'Content-Type: application/json' \
+                -d "${kube_template}" >/dev/null
+
+              journal_template='{"index_patterns":["journald-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
+              curl -sS -X PUT "${OS_URL}/_index_template/journald-logs" \
+                -H 'Content-Type: application/json' \
+                -d "${journal_template}" >/dev/null