logging: add trace analytics ingestion

infrastructure/sources/helm/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
   - jetstack.yaml
   - jenkins.yaml
   - mailu.yaml
+  - opentelemetry.yaml
   - opensearch.yaml
   - harbor.yaml
   - prometheus.yaml

infrastructure/sources/helm/opentelemetry.yaml

@@ -0,0 +1,9 @@
+# infrastructure/sources/helm/opentelemetry.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: opentelemetry
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://open-telemetry.github.io/opentelemetry-helm-charts

scripts/logging_render_observability.py

@@ -180,7 +180,7 @@ def build_objects() -> tuple[list[dict], list[dict], list[dict]]:
             "description": app.description,
             "baseQuery": app.base_query,
             "servicesEntities": [],
-            "traceGroups": [],
+            "traceGroups": [app.name],
         }
         for app in apps
     ]

services/logging/data-prepper-helmrelease.yaml

@@ -0,0 +1,75 @@
+# services/logging/data-prepper-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: data-prepper
+  namespace: logging
+spec:
+  interval: 15m
+  timeout: 10m
+  chart:
+    spec:
+      chart: data-prepper
+      version: "~0.3.1"
+      sourceRef:
+        kind: HelmRepository
+        name: opensearch
+        namespace: flux-system
+  values:
+    fullnameOverride: data-prepper
+    replicaCount: 1
+    config:
+      data-prepper-config.yaml: |
+        ssl: false
+    pipelineConfig:
+      enabled: true
+      config:
+        entry-pipeline:
+          delay: "100"
+          source:
+            otel_trace_source:
+              ssl: false
+          sink:
+            - pipeline:
+                name: "raw-pipeline"
+            - pipeline:
+                name: "service-map-pipeline"
+        raw-pipeline:
+          source:
+            pipeline:
+              name: "entry-pipeline"
+          processor:
+            - otel_traces:
+          sink:
+            - opensearch:
+                hosts: ["http://opensearch-master.logging.svc.cluster.local:9200"]
+                index_type: trace-analytics-raw
+        service-map-pipeline:
+          delay: "100"
+          source:
+            pipeline:
+              name: "entry-pipeline"
+          processor:
+            - service_map:
+          sink:
+            - opensearch:
+                hosts: ["http://opensearch-master.logging.svc.cluster.local:9200"]
+                index_type: trace-analytics-service-map
+    resources:
+      requests:
+        cpu: "200m"
+        memory: "512Mi"
+      limits:
+        memory: "1Gi"
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+      hardware: rpi5
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5

services/logging/kustomization.yaml

@@ -3,11 +3,13 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - namespace.yaml
+  - opensearch-dashboards-objects.yaml
+  - opensearch-observability-objects.yaml
   - opensearch-pvc.yaml
   - opensearch-helmrelease.yaml
   - opensearch-dashboards-helmrelease.yaml
-  - opensearch-dashboards-objects.yaml
+  - data-prepper-helmrelease.yaml
-  - opensearch-observability-objects.yaml
+  - otel-collector-helmrelease.yaml
   - opensearch-ism-job.yaml
   - opensearch-dashboards-setup-job.yaml
   - opensearch-observability-setup-job.yaml

services/logging/observability/applications.json

@@ -4,132 +4,170 @@
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'bstein-dev-home'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "bstein-dev-home"
+    ]
   },
   {
     "name": "pegasus",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'jellyfin' and kubernetes.labels.app = 'pegasus'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "pegasus"
+    ]
   },
   {
     "name": "jellyfin",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'jellyfin' and kubernetes.labels.app = 'jellyfin'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "jellyfin"
+    ]
   },
   {
     "name": "vaultwarden",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'vaultwarden'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "vaultwarden"
+    ]
   },
   {
     "name": "mailu",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'mailu-mailserver'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "mailu"
+    ]
   },
   {
     "name": "nextcloud",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'nextcloud'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "nextcloud"
+    ]
   },
   {
     "name": "gitea",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'gitea'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "gitea"
+    ]
   },
   {
     "name": "jenkins",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'jenkins'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "jenkins"
+    ]
   },
   {
     "name": "harbor",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'harbor'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "harbor"
+    ]
   },
   {
     "name": "vault",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'vault'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "vault"
+    ]
   },
   {
     "name": "keycloak",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'sso'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "keycloak"
+    ]
   },
   {
     "name": "flux-system",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'flux-system'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "flux-system"
+    ]
   },
   {
     "name": "comms",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "comms"
+    ]
   },
   {
     "name": "element-web",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.container_name = 'element-web'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "element-web"
+    ]
   },
   {
     "name": "element-call",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.labels.app = 'element-call'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "element-call"
+    ]
   },
   {
     "name": "matrix-synapse",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.container_name = 'synapse'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "matrix-synapse"
+    ]
   },
   {
     "name": "livekit",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.labels.app = 'livekit'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "livekit"
+    ]
   },
   {
     "name": "coturn",
     "description": "",
     "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.labels.app = 'coturn'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "coturn"
+    ]
   },
   {
     "name": "lesavka",
     "description": "",
     "baseQuery": "source = journald-* | where _HOSTNAME = 'titan-jh'",
     "servicesEntities": [],
-    "traceGroups": []
+    "traceGroups": [
+      "lesavka"
+    ]
   }
 ]

services/logging/opensearch-ism-job.yaml

@@ -2,7 +2,7 @@
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: opensearch-ism-setup-4
+  name: opensearch-ism-setup-5
   namespace: logging
 spec:
   backoffLimit: 3
@@ -48,6 +48,11 @@ spec:
                 -H 'Content-Type: application/json' \
                 -d "${policy}" >/dev/null
 
+              trace_policy='{"policy":{"description":"Delete trace analytics after 30 days","schema_version":1,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"30d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}'
+              curl -sS -X PUT "${OS_URL}/_plugins/_ism/policies/trace-analytics-30d" \
+                -H 'Content-Type: application/json' \
+                -d "${trace_policy}" >/dev/null
+
               kube_template='{"index_patterns":["kube-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"logging-180d"},"mappings":{"properties":{"@timestamp":{"type":"date"}}}}}'
               curl -sS -X PUT "${OS_URL}/_index_template/kube-logs" \
                 -H 'Content-Type: application/json' \
@@ -58,6 +63,11 @@ spec:
                 -H 'Content-Type: application/json' \
                 -d "${journal_template}" >/dev/null
 
+              trace_template='{"index_patterns":["trace-analytics-*"],"priority":200,"template":{"settings":{"index.number_of_shards":1,"index.number_of_replicas":0,"index.refresh_interval":"30s","plugins.index_state_management.policy_id":"trace-analytics-30d"}}}'
+              curl -sS -X PUT "${OS_URL}/_index_template/trace-analytics" \
+                -H 'Content-Type: application/json' \
+                -d "${trace_template}" >/dev/null
+
               curl -sS -X PUT "${OS_URL}/_all/_settings" \
                 -H 'Content-Type: application/json' \
                 -d '{"index":{"number_of_replicas":0}}' >/dev/null

services/logging/opensearch-observability-objects.yaml

@@ -13,133 +13,171 @@ data:
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'bstein-dev-home'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "bstein-dev-home"
+        ]
       },
       {
         "name": "pegasus",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'jellyfin' and kubernetes.labels.app = 'pegasus'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "pegasus"
+        ]
       },
       {
         "name": "jellyfin",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'jellyfin' and kubernetes.labels.app = 'jellyfin'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "jellyfin"
+        ]
       },
       {
         "name": "vaultwarden",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'vaultwarden'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "vaultwarden"
+        ]
       },
       {
         "name": "mailu",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'mailu-mailserver'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "mailu"
+        ]
       },
       {
         "name": "nextcloud",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'nextcloud'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "nextcloud"
+        ]
       },
       {
         "name": "gitea",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'gitea'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "gitea"
+        ]
       },
       {
         "name": "jenkins",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'jenkins'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "jenkins"
+        ]
       },
       {
         "name": "harbor",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'harbor'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "harbor"
+        ]
       },
       {
         "name": "vault",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'vault'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "vault"
+        ]
       },
       {
         "name": "keycloak",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'sso'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "keycloak"
+        ]
       },
       {
         "name": "flux-system",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'flux-system'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "flux-system"
+        ]
       },
       {
         "name": "comms",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "comms"
+        ]
       },
       {
         "name": "element-web",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.container_name = 'element-web'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "element-web"
+        ]
       },
       {
         "name": "element-call",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.labels.app = 'element-call'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "element-call"
+        ]
       },
       {
         "name": "matrix-synapse",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.container_name = 'synapse'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "matrix-synapse"
+        ]
       },
       {
         "name": "livekit",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.labels.app = 'livekit'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "livekit"
+        ]
       },
       {
         "name": "coturn",
         "description": "",
         "baseQuery": "source = kube-* | where kubernetes.namespace_name = 'comms' and kubernetes.labels.app = 'coturn'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "coturn"
+        ]
       },
       {
         "name": "lesavka",
         "description": "",
         "baseQuery": "source = journald-* | where _HOSTNAME = 'titan-jh'",
         "servicesEntities": [],
-        "traceGroups": []
+        "traceGroups": [
+          "lesavka"
+        ]
       }
     ]
   saved_queries.json: |

services/logging/opensearch-observability-setup-job.yaml

@@ -150,7 +150,7 @@ data:
 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: opensearch-observability-setup-1
+  name: opensearch-observability-setup-2
   namespace: logging
 spec:
   backoffLimit: 3

services/logging/opensearch-prune-cronjob.yaml

@@ -122,7 +122,7 @@ spec:
                 - name: LOG_LIMIT_BYTES
                   value: "1099511627776"
                 - name: LOG_INDEX_PATTERNS
-                  value: "kube-*,journald-*"
+                  value: "kube-*,journald-*,trace-analytics-*"
               volumeMounts:
                 - name: scripts
                   mountPath: /scripts

services/logging/otel-collector-helmrelease.yaml

@@ -0,0 +1,87 @@
+# services/logging/otel-collector-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: otel-collector
+  namespace: logging
+spec:
+  interval: 15m
+  timeout: 10m
+  chart:
+    spec:
+      chart: opentelemetry-collector
+      version: "~0.143.0"
+      sourceRef:
+        kind: HelmRepository
+        name: opentelemetry
+        namespace: flux-system
+  values:
+    fullnameOverride: otel-collector
+    mode: deployment
+    replicaCount: 1
+    ports:
+      otlp:
+        enabled: true
+        containerPort: 4317
+        servicePort: 4317
+        protocol: TCP
+      otlp-http:
+        enabled: true
+        containerPort: 4318
+        servicePort: 4318
+        protocol: TCP
+      jaeger-compact:
+        enabled: false
+      jaeger-thrift:
+        enabled: false
+      jaeger-grpc:
+        enabled: false
+      zipkin:
+        enabled: false
+      metrics:
+        enabled: false
+    config:
+      receivers:
+        otlp:
+          protocols:
+            grpc:
+              endpoint: ${env:MY_POD_IP}:4317
+            http:
+              endpoint: ${env:MY_POD_IP}:4318
+      processors:
+        memory_limiter:
+          check_interval: 5s
+          limit_percentage: 80
+          spike_limit_percentage: 25
+        batch: {}
+      exporters:
+        otlp/data-prepper:
+          endpoint: data-prepper.logging.svc.cluster.local:21890
+          tls:
+            insecure: true
+      service:
+        extensions:
+          - health_check
+        pipelines:
+          traces:
+            receivers: [otlp]
+            processors: [memory_limiter, batch]
+            exporters: [otlp/data-prepper]
+    resources:
+      requests:
+        cpu: "100m"
+        memory: "256Mi"
+      limits:
+        memory: "512Mi"
+    nodeSelector:
+      node-role.kubernetes.io/worker: "true"
+      hardware: rpi5
+    affinity:
+      nodeAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          nodeSelectorTerms:
+            - matchExpressions:
+                - key: hardware
+                  operator: In
+                  values:
+                    - rpi5