From cac71e4a41437430b4d42131173fa5bde45e1dce Mon Sep 17 00:00:00 2001 From: Brad Stein Date: Fri, 9 Jan 2026 08:54:07 -0300 Subject: [PATCH] logging: add opensearch dashboards ui --- .../sources/helm/kustomization.yaml | 1 + infrastructure/sources/helm/opensearch.yaml | 9 ++ services/keycloak/kustomization.yaml | 1 + .../keycloak/logs-oidc-secret-ensure-job.yaml | 96 +++++++++++++++++++ services/logging/fluent-bit-helmrelease.yaml | 24 +++-- services/logging/ingress.yaml | 4 +- services/logging/kustomization.yaml | 5 +- services/logging/oauth2-proxy.yaml | 22 ++--- .../opensearch-dashboards-helmrelease.yaml | 46 +++++++++ services/logging/opensearch-helmrelease.yaml | 56 +++++++++++ services/logging/opensearch-ism-job.yaml | 47 +++++++++ 11 files changed, 287 insertions(+), 24 deletions(-) create mode 100644 infrastructure/sources/helm/opensearch.yaml create mode 100644 services/keycloak/logs-oidc-secret-ensure-job.yaml create mode 100644 services/logging/opensearch-dashboards-helmrelease.yaml create mode 100644 services/logging/opensearch-helmrelease.yaml create mode 100644 services/logging/opensearch-ism-job.yaml diff --git a/infrastructure/sources/helm/kustomization.yaml b/infrastructure/sources/helm/kustomization.yaml index 1cbf20e..97fd70e 100644 --- a/infrastructure/sources/helm/kustomization.yaml +++ b/infrastructure/sources/helm/kustomization.yaml @@ -8,6 +8,7 @@ resources: - jetstack.yaml - jenkins.yaml - mailu.yaml + - opensearch.yaml - harbor.yaml - prometheus.yaml - victoria-metrics.yaml diff --git a/infrastructure/sources/helm/opensearch.yaml b/infrastructure/sources/helm/opensearch.yaml new file mode 100644 index 0000000..e5b60c3 --- /dev/null +++ b/infrastructure/sources/helm/opensearch.yaml @@ -0,0 +1,9 @@ +# infrastructure/sources/helm/opensearch.yaml +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: opensearch + namespace: flux-system +spec: + interval: 1h + url: https://opensearch-project.github.io/helm-charts diff --git a/services/keycloak/kustomization.yaml b/services/keycloak/kustomization.yaml index e3d6513..05639e5 100644 --- a/services/keycloak/kustomization.yaml +++ b/services/keycloak/kustomization.yaml @@ -18,6 +18,7 @@ resources: - user-overrides-job.yaml - mas-secrets-ensure-job.yaml - synapse-oidc-secret-ensure-job.yaml + - logs-oidc-secret-ensure-job.yaml - service.yaml - ingress.yaml generatorOptions: diff --git a/services/keycloak/logs-oidc-secret-ensure-job.yaml b/services/keycloak/logs-oidc-secret-ensure-job.yaml new file mode 100644 index 0000000..9550d8b --- /dev/null +++ b/services/keycloak/logs-oidc-secret-ensure-job.yaml @@ -0,0 +1,96 @@ +# services/keycloak/logs-oidc-secret-ensure-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: logs-oidc-secret-ensure-1 + namespace: sso +spec: + backoffLimit: 0 + ttlSecondsAfterFinished: 3600 + template: + spec: + serviceAccountName: mas-secrets-ensure + restartPolicy: Never + containers: + - name: apply + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + apk add --no-cache curl jq kubectl openssl >/dev/null + + KC_URL="http://keycloak.sso.svc.cluster.local" + ACCESS_TOKEN="" + for attempt in 1 2 3 4 5; do + TOKEN_JSON="$(curl -sS -X POST "$KC_URL/realms/master/protocol/openid-connect/token" \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -d "grant_type=password" \ + -d "client_id=admin-cli" \ + -d "username=${KEYCLOAK_ADMIN}" \ + -d "password=${KEYCLOAK_ADMIN_PASSWORD}" || true)" + ACCESS_TOKEN="$(echo "$TOKEN_JSON" | jq -r '.access_token' 2>/dev/null || true)" + if [ -n "$ACCESS_TOKEN" ] && [ "$ACCESS_TOKEN" != "null" ]; then + break + fi + echo "Keycloak token request failed (attempt ${attempt})" >&2 + sleep $((attempt * 2)) + done + if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then + echo "Failed to fetch Keycloak admin token" >&2 + exit 1 + fi + + CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "$KC_URL/admin/realms/atlas/clients?clientId=logs" || true)" + CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" + + if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then + create_payload='{"clientId":"logs","enabled":true,"protocol":"openid-connect","publicClient":false,"standardFlowEnabled":true,"implicitFlowEnabled":false,"directAccessGrantsEnabled":false,"serviceAccountsEnabled":false,"redirectUris":["https://logs.bstein.dev/oauth2/callback"],"webOrigins":["https://logs.bstein.dev"],"rootUrl":"https://logs.bstein.dev","baseUrl":"/"}' + status="$(curl -sS -o /dev/null -w "%{http_code}" -X POST \ + -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + -H 'Content-Type: application/json' \ + -d "${create_payload}" \ + "$KC_URL/admin/realms/atlas/clients")" + if [ "$status" != "201" ] && [ "$status" != "204" ]; then + echo "Keycloak client create failed (status ${status})" >&2 + exit 1 + fi + CLIENT_QUERY="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "$KC_URL/admin/realms/atlas/clients?clientId=logs" || true)" + CLIENT_ID="$(echo "$CLIENT_QUERY" | jq -r '.[0].id' 2>/dev/null || true)" + fi + + if [ -z "$CLIENT_ID" ] || [ "$CLIENT_ID" = "null" ]; then + echo "Keycloak client logs not found" >&2 + exit 1 + fi + + CLIENT_SECRET="$(curl -sS -H "Authorization: Bearer ${ACCESS_TOKEN}" \ + "$KC_URL/admin/realms/atlas/clients/${CLIENT_ID}/client-secret" | jq -r '.value' 2>/dev/null || true)" + if [ -z "$CLIENT_SECRET" ] || [ "$CLIENT_SECRET" = "null" ]; then + echo "Keycloak client secret not found" >&2 + exit 1 + fi + + if kubectl -n logging get secret oauth2-proxy-logs-oidc >/dev/null 2>&1; then + exit 0 + fi + + COOKIE_SECRET="$(openssl rand -base64 32 | tr -d '\n')" + kubectl -n logging create secret generic oauth2-proxy-logs-oidc \ + --from-literal=client_id="logs" \ + --from-literal=client_secret="${CLIENT_SECRET}" \ + --from-literal=cookie_secret="${COOKIE_SECRET}" \ + --dry-run=client -o yaml | kubectl -n logging apply -f - >/dev/null + env: + - name: KEYCLOAK_ADMIN + valueFrom: + secretKeyRef: + name: keycloak-admin + key: username + - name: KEYCLOAK_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: keycloak-admin + key: password diff --git a/services/logging/fluent-bit-helmrelease.yaml b/services/logging/fluent-bit-helmrelease.yaml index ca156e4..a3f1c26 100644 --- a/services/logging/fluent-bit-helmrelease.yaml +++ b/services/logging/fluent-bit-helmrelease.yaml @@ -84,17 +84,21 @@ spec: K8S-Logging.Exclude On outputs: | [OUTPUT] - Name loki + Name es Match kube.* - Host loki.logging.svc.cluster.local - Port 3100 - labels job=fluent-bit,namespace=$kubernetes['namespace_name'],pod=$kubernetes['pod_name'],container=$kubernetes['container_name'] - line_format json + Host opensearch-master.logging.svc.cluster.local + Port 9200 + Logstash_Format On + Logstash_Prefix kube + Replace_Dots On + Suppress_Type_Name On [OUTPUT] - Name loki + Name es Match journald.* - Host loki.logging.svc.cluster.local - Port 3100 - labels job=systemd - line_format json + Host opensearch-master.logging.svc.cluster.local + Port 9200 + Logstash_Format On + Logstash_Prefix journald + Replace_Dots On + Suppress_Type_Name On diff --git a/services/logging/ingress.yaml b/services/logging/ingress.yaml index f3211b2..7beeb9a 100644 --- a/services/logging/ingress.yaml +++ b/services/logging/ingress.yaml @@ -2,7 +2,7 @@ apiVersion: networking.k8s.io/v1 kind: Ingress metadata: - name: loki + name: logs namespace: logging annotations: cert-manager.io/cluster-issuer: letsencrypt @@ -20,6 +20,6 @@ spec: pathType: Prefix backend: service: - name: oauth2-proxy-loki + name: oauth2-proxy-logs port: name: http diff --git a/services/logging/kustomization.yaml b/services/logging/kustomization.yaml index 476d88b..9132b8e 100644 --- a/services/logging/kustomization.yaml +++ b/services/logging/kustomization.yaml @@ -3,7 +3,10 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - namespace.yaml - - loki-helmrelease.yaml + - opensearch-helmrelease.yaml + - opensearch-dashboards-helmrelease.yaml + - opensearch-ism-job.yaml - fluent-bit-helmrelease.yaml + - loki-helmrelease.yaml - oauth2-proxy.yaml - ingress.yaml diff --git a/services/logging/oauth2-proxy.yaml b/services/logging/oauth2-proxy.yaml index f9b9616..ef3621f 100644 --- a/services/logging/oauth2-proxy.yaml +++ b/services/logging/oauth2-proxy.yaml @@ -2,36 +2,36 @@ apiVersion: v1 kind: Service metadata: - name: oauth2-proxy-loki + name: oauth2-proxy-logs namespace: logging labels: - app: oauth2-proxy-loki + app: oauth2-proxy-logs spec: ports: - name: http port: 80 targetPort: 4180 selector: - app: oauth2-proxy-loki + app: oauth2-proxy-logs --- apiVersion: apps/v1 kind: Deployment metadata: - name: oauth2-proxy-loki + name: oauth2-proxy-logs namespace: logging labels: - app: oauth2-proxy-loki + app: oauth2-proxy-logs spec: replicas: 2 selector: matchLabels: - app: oauth2-proxy-loki + app: oauth2-proxy-logs template: metadata: labels: - app: oauth2-proxy-loki + app: oauth2-proxy-logs spec: nodeSelector: node-role.kubernetes.io/worker: "true" @@ -63,7 +63,7 @@ spec: - --cookie-refresh=20m - --cookie-expire=168h - --insecure-oidc-allow-unverified-email=true - - --upstream=http://loki-gateway.logging.svc.cluster.local + - --upstream=http://opensearch-dashboards.logging.svc.cluster.local:5601 - --http-address=0.0.0.0:4180 - --skip-provider-button=true - --skip-jwt-bearer-tokens=true @@ -72,17 +72,17 @@ spec: - name: OAUTH2_PROXY_CLIENT_ID valueFrom: secretKeyRef: - name: oauth2-proxy-loki-oidc + name: oauth2-proxy-logs-oidc key: client_id - name: OAUTH2_PROXY_CLIENT_SECRET valueFrom: secretKeyRef: - name: oauth2-proxy-loki-oidc + name: oauth2-proxy-logs-oidc key: client_secret - name: OAUTH2_PROXY_COOKIE_SECRET valueFrom: secretKeyRef: - name: oauth2-proxy-loki-oidc + name: oauth2-proxy-logs-oidc key: cookie_secret ports: - containerPort: 4180 diff --git a/services/logging/opensearch-dashboards-helmrelease.yaml b/services/logging/opensearch-dashboards-helmrelease.yaml new file mode 100644 index 0000000..4943200 --- /dev/null +++ b/services/logging/opensearch-dashboards-helmrelease.yaml @@ -0,0 +1,46 @@ +# services/logging/opensearch-dashboards-helmrelease.yaml +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: opensearch-dashboards + namespace: logging +spec: + interval: 15m + chart: + spec: + chart: opensearch-dashboards + version: "~2.32.0" + sourceRef: + kind: HelmRepository + name: opensearch + namespace: flux-system + values: + fullnameOverride: opensearch-dashboards + opensearchHosts: "http://opensearch-master.logging.svc.cluster.local:9200" + replicaCount: 1 + config: + opensearch_dashboards.yml: | + server.host: 0.0.0.0 + opensearch.hosts: ["http://opensearch-master.logging.svc.cluster.local:9200"] + opensearch_security.enabled: false + extraEnvs: + - name: NODE_OPTIONS + value: "--max-old-space-size=512" + resources: + requests: + cpu: "200m" + memory: "512Mi" + limits: + memory: "512Mi" + nodeSelector: + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - rpi4 diff --git a/services/logging/opensearch-helmrelease.yaml b/services/logging/opensearch-helmrelease.yaml new file mode 100644 index 0000000..9ccbad0 --- /dev/null +++ b/services/logging/opensearch-helmrelease.yaml @@ -0,0 +1,56 @@ +# services/logging/opensearch-helmrelease.yaml +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: opensearch + namespace: logging +spec: + interval: 15m + chart: + spec: + chart: opensearch + version: "~2.36.0" + sourceRef: + kind: HelmRepository + name: opensearch + namespace: flux-system + values: + fullnameOverride: opensearch + clusterName: opensearch + nodeGroup: master + masterService: opensearch-master + singleNode: true + replicas: 1 + minimumMasterNodes: 1 + opensearchJavaOpts: "-Xms1g -Xmx1g" + resources: + requests: + cpu: "500m" + memory: "2Gi" + limits: + memory: "2Gi" + persistence: + enabled: true + storageClass: asteria + size: 500Gi + config: + opensearch.yml: | + cluster.name: opensearch + network.host: 0.0.0.0 + discovery.type: single-node + plugins.security.disabled: true + node.store.allow_mmap: false + nodeSelector: + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - rpi4 + sysctlInit: + enabled: true diff --git a/services/logging/opensearch-ism-job.yaml b/services/logging/opensearch-ism-job.yaml new file mode 100644 index 0000000..c33a700 --- /dev/null +++ b/services/logging/opensearch-ism-job.yaml @@ -0,0 +1,47 @@ +# services/logging/opensearch-ism-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: opensearch-ism-setup-1 + namespace: logging +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 3600 + template: + spec: + restartPolicy: OnFailure + containers: + - name: apply + image: alpine:3.20 + command: ["/bin/sh", "-c"] + args: + - | + set -euo pipefail + apk add --no-cache curl >/dev/null + + OS_URL="http://opensearch-master.logging.svc.cluster.local:9200" + for attempt in $(seq 1 60); do + if curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then + break + fi + sleep 5 + done + if ! curl -s -o /dev/null -w "%{http_code}" "${OS_URL}" | grep -q "200"; then + echo "OpenSearch did not become ready in time" >&2 + exit 1 + fi + + policy='{"policy":{"description":"Delete logs after 180 days","schema_version":1,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"180d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}' + curl -sS -X PUT "${OS_URL}/_plugins/_ism/policies/logging-180d" \ + -H 'Content-Type: application/json' \ + -d "${policy}" >/dev/null + + kube_template='{"index_patterns":["kube-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}' + curl -sS -X PUT "${OS_URL}/_index_template/kube-logs" \ + -H 'Content-Type: application/json' \ + -d "${kube_template}" >/dev/null + + journal_template='{"index_patterns":["journald-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}' + curl -sS -X PUT "${OS_URL}/_index_template/journald-logs" \ + -H 'Content-Type: application/json' \ + -d "${journal_template}" >/dev/null