logging: add opensearch dashboards ui

2026-01-09 08:54:07 -03:00 · 2026-01-09 08:54:07 -03:00 · cac71e4a41
commit cac71e4a41
parent 719f16c4e3
11 changed files with 287 additions and 24 deletions
--- a/infrastructure/sources/helm/kustomization.yaml
+++ b/infrastructure/sources/helm/kustomization.yaml
@ -8,6 +8,7 @@ resources:
  - jetstack.yaml
  - jenkins.yaml
  - mailu.yaml
+  - opensearch.yaml
  - harbor.yaml
  - prometheus.yaml
  - victoria-metrics.yaml
--- a/infrastructure/sources/helm/opensearch.yaml
+++ b/infrastructure/sources/helm/opensearch.yaml
@ -0,0 +1,9 @@
+# infrastructure/sources/helm/opensearch.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: opensearch
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://opensearch-project.github.io/helm-charts
--- a/services/keycloak/kustomization.yaml
+++ b/services/keycloak/kustomization.yaml
@ -18,6 +18,7 @@ resources:
  - user-overrides-job.yaml
  - mas-secrets-ensure-job.yaml
  - synapse-oidc-secret-ensure-job.yaml
+  - logs-oidc-secret-ensure-job.yaml
  - service.yaml
  - ingress.yaml
 generatorOptions:
--- a/services/keycloak/logs-oidc-secret-ensure-job.yaml
+++ b/services/keycloak/logs-oidc-secret-ensure-job.yaml
@ -0,0 +1,96 @@
+# services/keycloak/logs-oidc-secret-ensure-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: logs-oidc-secret-ensure-1
+  namespace: sso
+spec:
+  backoffLimit: 0
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      serviceAccountName: mas-secrets-ensure
+      restartPolicy: Never
+      containers:
+        - name: apply
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              apk add --no-cache curl jq kubectl openssl >/dev/null
+
+              KC_URL="http://keycloak.sso.svc.cluster.local"
+              ACCESS_TOKEN=""
+              for attempt in 1 2 3 4 5; do
+                TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \
+                  -H 'Content-Type: application/x-www-form-urlencoded' \
+                  -d "grant_type=password" \
+                  -d "client_id=admin-cli" \
+                  -d "username=${KEYCLOAK_ADMIN}" \
+                  -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)"
+                ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)"
+                if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then
+                  break
+                fi
+                echo "Keycloak token request failed (attempt ${attempt})" >&2
+                sleep $((attempt * 2))
+              done
+              if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+                echo "Failed to fetch Keycloak admin token" >&2
+                exit 1
+              fi
+
+              CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                "$KC_URL/admin/realms/atlas/clients?clientId=logs" || true)"
+              CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+
+              if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+                create_payload='{"clientId":"logs","enabled":true,"protocol":"openid-connect","publicClient":false,"standardFlowEnabled":true,"implicitFlowEnabled":false,"directAccessGrantsEnabled":false,"serviceAccountsEnabled":false,"redirectUris":["https://logs.bstein.dev/oauth2/callback"],"webOrigins":["https://logs.bstein.dev"],"rootUrl":"https://logs.bstein.dev","baseUrl":"/"}'
+                status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \
+                  -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                  -H 'Content-Type: application/json' \
+                  -d "${create_payload}" \
+                  "$KC_URL/admin/realms/atlas/clients")"
+                if [ "$status" != "201" ] && [ "$status" != "204" ]; then
+                  echo "Keycloak client create failed (status ${status})" >&2
+                  exit 1
+                fi
+                CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                  "$KC_URL/admin/realms/atlas/clients?clientId=logs" || true)"
+                CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)"
+              fi
+
+              if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then
+                echo "Keycloak client logs not found" >&2
+                exit 1
+              fi
+
+              CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \
+                "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/client-secret" | jq -r '.value' 2>/dev/null || true)"
+              if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then
+                echo "Keycloak client secret not found" >&2
+                exit 1
+              fi
+
+              if kubectl -n logging get secret oauth2-proxy-logs-oidc >/dev/null 2>&1; then
+                exit 0
+              fi
+
+              COOKIE_SECRET="$(openssl rand -base64 32 | tr -d '\n')"
+              kubectl -n logging create secret generic oauth2-proxy-logs-oidc \
+                --from-literal=client_id="logs" \
+                --from-literal=client_secret="${CLIENT_SECRET}" \
+                --from-literal=cookie_secret="${COOKIE_SECRET}" \
+                --dry-run=client -o yaml | kubectl -n logging apply -f - >/dev/null
+          env:
+            - name: KEYCLOAK_ADMIN
+              valueFrom:
+                secretKeyRef:
+                  name: keycloak-admin
+                  key: username
+            - name: KEYCLOAK_ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: keycloak-admin
+                  key: password
--- a/services/logging/fluent-bit-helmrelease.yaml
+++ b/services/logging/fluent-bit-helmrelease.yaml
@ -84,17 +84,21 @@ spec:
            K8S-Logging.Exclude On
      outputs: |
        [OUTPUT]
-            Name loki
+            Name es
            Match kube.*
-            Host loki.logging.svc.cluster.local
-            Port 3100
-            labels job=fluent-bit,namespace=$kubernetes['namespace_name'],pod=$kubernetes['pod_name'],container=$kubernetes['container_name']
-            line_format json
+            Host opensearch-master.logging.svc.cluster.local
+            Port 9200
+            Logstash_Format On
+            Logstash_Prefix kube
+            Replace_Dots On
+            Suppress_Type_Name On

        [OUTPUT]
-            Name loki
+            Name es
            Match journald.*
-            Host loki.logging.svc.cluster.local
-            Port 3100
-            labels job=systemd
-            line_format json
+            Host opensearch-master.logging.svc.cluster.local
+            Port 9200
+            Logstash_Format On
+            Logstash_Prefix journald
+            Replace_Dots On
+            Suppress_Type_Name On
--- a/services/logging/ingress.yaml
+++ b/services/logging/ingress.yaml
@ -2,7 +2,7 @@
 apiVersion: networking.k8s.io/v1
 kind: Ingress
 metadata:
-  name: loki
+  name: logs
  namespace: logging
  annotations:
    cert-manager.io/cluster-issuer: letsencrypt
@ -20,6 +20,6 @@ spec:
            pathType: Prefix
            backend:
              service:
-                name: oauth2-proxy-loki
+                name: oauth2-proxy-logs
                port:
                  name: http
--- a/services/logging/kustomization.yaml
+++ b/services/logging/kustomization.yaml
@ -3,7 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
  - namespace.yaml
-  - loki-helmrelease.yaml
+  - opensearch-helmrelease.yaml
+  - opensearch-dashboards-helmrelease.yaml
+  - opensearch-ism-job.yaml
  - fluent-bit-helmrelease.yaml
+  - loki-helmrelease.yaml
  - oauth2-proxy.yaml
  - ingress.yaml
--- a/services/logging/oauth2-proxy.yaml
+++ b/services/logging/oauth2-proxy.yaml
@ -2,36 +2,36 @@
 apiVersion: v1
 kind: Service
 metadata:
-  name: oauth2-proxy-loki
+  name: oauth2-proxy-logs
  namespace: logging
  labels:
-    app: oauth2-proxy-loki
+    app: oauth2-proxy-logs
 spec:
  ports:
    - name: http
      port: 80
      targetPort: 4180
  selector:
-    app: oauth2-proxy-loki
+    app: oauth2-proxy-logs

 ---

 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: oauth2-proxy-loki
+  name: oauth2-proxy-logs
  namespace: logging
  labels:
-    app: oauth2-proxy-loki
+    app: oauth2-proxy-logs
 spec:
  replicas: 2
  selector:
    matchLabels:
-      app: oauth2-proxy-loki
+      app: oauth2-proxy-logs
  template:
    metadata:
      labels:
-        app: oauth2-proxy-loki
+        app: oauth2-proxy-logs
    spec:
      nodeSelector:
        node-role.kubernetes.io/worker: "true"
@ -63,7 +63,7 @@ spec:
            - --cookie-refresh=20m
            - --cookie-expire=168h
            - --insecure-oidc-allow-unverified-email=true
-            - --upstream=http://loki-gateway.logging.svc.cluster.local
+            - --upstream=http://opensearch-dashboards.logging.svc.cluster.local:5601
            - --http-address=0.0.0.0:4180
            - --skip-provider-button=true
            - --skip-jwt-bearer-tokens=true
@ -72,17 +72,17 @@ spec:
            - name: OAUTH2_PROXY_CLIENT_ID
              valueFrom:
                secretKeyRef:
-                  name: oauth2-proxy-loki-oidc
+                  name: oauth2-proxy-logs-oidc
                  key: client_id
            - name: OAUTH2_PROXY_CLIENT_SECRET
              valueFrom:
                secretKeyRef:
-                  name: oauth2-proxy-loki-oidc
+                  name: oauth2-proxy-logs-oidc
                  key: client_secret
            - name: OAUTH2_PROXY_COOKIE_SECRET
              valueFrom:
                secretKeyRef:
-                  name: oauth2-proxy-loki-oidc
+                  name: oauth2-proxy-logs-oidc
                  key: cookie_secret
          ports:
            - containerPort: 4180
--- a/services/logging/opensearch-dashboards-helmrelease.yaml
+++ b/services/logging/opensearch-dashboards-helmrelease.yaml
@ -0,0 +1,46 @@
+# services/logging/opensearch-dashboards-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: opensearch-dashboards
+  namespace: logging
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: opensearch-dashboards
+      version: "~2.32.0"
+      sourceRef:
+        kind: HelmRepository
+        name: opensearch
+        namespace: flux-system
+  values:
+    fullnameOverride: opensearch-dashboards
+    opensearchHosts: "http://opensearch-master.logging.svc.cluster.local:9200"
+    replicaCount: 1
+    config:
+      opensearch_dashboards.yml: |
+        server.host: 0.0.0.0
+        opensearch.hosts: ["http://opensearch-master.logging.svc.cluster.local:9200"]
+        opensearch_security.enabled: false
+    extraEnvs:
+      - name: NODE_OPTIONS
+        value: "--max-old-space-size=512"
+    resources:
+      requests:
+        cpu: "200m"
+        memory: "512Mi"
+      limits:
+        memory: "512Mi"
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+                    - rpi4
--- a/services/logging/opensearch-helmrelease.yaml
+++ b/services/logging/opensearch-helmrelease.yaml
@ -0,0 +1,56 @@
+# services/logging/opensearch-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: opensearch
+  namespace: logging
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: opensearch
+      version: "~2.36.0"
+      sourceRef:
+        kind: HelmRepository
+        name: opensearch
+        namespace: flux-system
+  values:
+    fullnameOverride: opensearch
+    clusterName: opensearch
+    nodeGroup: master
+    masterService: opensearch-master
+    singleNode: true
+    replicas: 1
+    minimumMasterNodes: 1
+    opensearchJavaOpts: "-Xms1g -Xmx1g"
+    resources:
+      requests:
+        cpu: "500m"
+        memory: "2Gi"
+      limits:
+        memory: "2Gi"
+    persistence:
+      enabled: true
+      storageClass: asteria
+      size: 500Gi
+    config:
+      opensearch.yml: |
+        cluster.name: opensearch
+        network.host: 0.0.0.0
+        discovery.type: single-node
+        plugins.security.disabled: true
+        node.store.allow_mmap: false
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5
+                    - rpi4
+    sysctlInit:
+      enabled: true
--- a/services/logging/opensearch-ism-job.yaml
+++ b/services/logging/opensearch-ism-job.yaml
@ -0,0 +1,47 @@
+# services/logging/opensearch-ism-job.yaml
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: opensearch-ism-setup-1
+  namespace: logging
+spec:
+  backoffLimit: 3
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+        - name: apply
+          image: alpine:3.20
+          command: ["/bin/sh", "-c"]
+          args:
+            - |
+              set -euo pipefail
+              apk add --no-cache curl >/dev/null
+
+              OS_URL="http://opensearch-master.logging.svc.cluster.local:9200"
+              for attempt in $(seq 1 60); do
+                if curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then
+                  break
+                fi
+                sleep 5
+              done
+              if ! curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then
+                echo "OpenSearch did not become ready in time" >&2
+                exit 1
+              fi
+
+              policy='{"policy":{"description":"Delete logs after 180 days","schema_version":1,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"180d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}'
+              curl -sS -X PUT "${OS_URL}/_plugins/_ism/policies/logging-180d" \
+                -H 'Content-Type: application/json' \
+                -d "${policy}" >/dev/null
+
+              kube_template='{"index_patterns":["kube-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
+              curl -sS -X PUT "${OS_URL}/_index_template/kube-logs" \
+                -H 'Content-Type: application/json' \
+                -d "${kube_template}" >/dev/null
+
+              journal_template='{"index_patterns":["journald-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
+              curl -sS -X PUT "${OS_URL}/_index_template/journald-logs" \
+                -H 'Content-Type: application/json' \
+                -d "${journal_template}" >/dev/null