logging: add loki and fluent-bit

clusters/atlas/applications/kustomization.yaml

@@ -7,6 +7,7 @@ resources:
   - ../../services/jellyfin
   - ../../services/comms
   - ../../services/monitoring
+  - ../../services/logging
   - ../../services/pegasus
   - ../../services/vault
   - ../../services/bstein-dev-home

clusters/atlas/flux-system/platform/kustomization.yaml

@@ -8,5 +8,6 @@ resources:
   - traefik/kustomization.yaml
   - gitops-ui/kustomization.yaml
   - monitoring/kustomization.yaml
+  - logging/kustomization.yaml
   - longhorn-ui/kustomization.yaml
   - ../platform/vault-csi/kustomization.yaml

clusters/atlas/flux-system/platform/logging/kustomization.yaml

@@ -0,0 +1,14 @@
+# clusters/atlas/flux-system/platform/logging/kustomization.yaml
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: logging
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: ./services/logging
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: flux-system
+  wait: false

infrastructure/sources/helm/fluent-bit.yaml

@@ -0,0 +1,9 @@
+# infrastructure/sources/helm/fluent-bit.yaml
+apiVersion: source.toolkit.fluxcd.io/v1
+kind: HelmRepository
+metadata:
+  name: fluent
+  namespace: flux-system
+spec:
+  interval: 1h
+  url: https://fluent.github.io/helm-charts

infrastructure/sources/helm/kustomization.yaml

@@ -2,6 +2,7 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
+  - fluent-bit.yaml
   - grafana.yaml
   - hashicorp.yaml
   - jetstack.yaml

services/logging/fluent-bit-helmrelease.yaml

@@ -0,0 +1,99 @@
+# services/logging/fluent-bit-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: fluent-bit
+  namespace: logging
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: fluent-bit
+      version: "~0.46.0"
+      sourceRef:
+        kind: HelmRepository
+        name: fluent
+        namespace: flux-system
+  values:
+    serviceAccount:
+      create: true
+    rbac:
+      create: true
+    tolerations:
+      - key: node-role.kubernetes.io/control-plane
+        operator: Exists
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        operator: Exists
+        effect: NoSchedule
+    extraVolumes:
+      - name: runlogjournal
+        hostPath:
+          path: /run/log/journal
+      - name: varlogjournal
+        hostPath:
+          path: /var/log/journal
+    extraVolumeMounts:
+      - name: runlogjournal
+        mountPath: /run/log/journal
+        readOnly: true
+      - name: varlogjournal
+        mountPath: /var/log/journal
+        readOnly: true
+    config:
+      service: |
+        [SERVICE]
+            Flush 1
+            Log_Level info
+            Daemon Off
+            Parsers_File parsers.conf
+            Parsers_File custom_parsers.conf
+            HTTP_Server On
+            HTTP_Listen 0.0.0.0
+            HTTP_Port 2020
+      inputs: |
+        [INPUT]
+            Name tail
+            Tag kube.*
+            Path /var/log/containers/*.log
+            Parser cri
+            Mem_Buf_Limit 50MB
+            Skip_Long_Lines On
+            Refresh_Interval 10
+            Rotate_Wait 30
+            storage.type memory
+
+        [INPUT]
+            Name systemd
+            Tag journald.*
+            Read_From_Tail On
+            storage.type memory
+      filters: |
+        [FILTER]
+            Name kubernetes
+            Match kube.*
+            Kube_URL https://kubernetes.default.svc:443
+            Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+            Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token
+            Merge_Log On
+            Keep_Log Off
+            K8S-Logging.Parser On
+            K8S-Logging.Exclude On
+      outputs: |
+        [OUTPUT]
+            Name loki
+            Match kube.*
+            Host loki.logging.svc.cluster.local
+            Port 3100
+            Labels {job="fluent-bit", namespace="$kubernetes['namespace_name']", pod="$kubernetes['pod_name']", container="$kubernetes['container_name']"}
+            LabelKeys stream
+            Line_Format json
+
+        [OUTPUT]
+            Name loki
+            Match journald.*
+            Host loki.logging.svc.cluster.local
+            Port 3100
+            Labels {job="systemd"}
+            LabelKeys _SYSTEMD_UNIT,_HOSTNAME,SYSLOG_IDENTIFIER
+            Line_Format json

services/logging/ingress.yaml

@@ -0,0 +1,25 @@
+# services/logging/ingress.yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: loki
+  namespace: logging
+  annotations:
+    cert-manager.io/cluster-issuer: letsencrypt
+spec:
+  ingressClassName: traefik
+  tls:
+    - hosts:
+        - logs.bstein.dev
+      secretName: logs-tls
+  rules:
+    - host: logs.bstein.dev
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: oauth2-proxy-loki
+                port:
+                  name: http

services/logging/kustomization.yaml

@@ -0,0 +1,9 @@
+# services/logging/kustomization.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - namespace.yaml
+  - loki-helmrelease.yaml
+  - fluent-bit-helmrelease.yaml
+  - oauth2-proxy.yaml
+  - ingress.yaml

services/logging/loki-helmrelease.yaml

@@ -0,0 +1,59 @@
+# services/logging/loki-helmrelease.yaml
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: loki
+  namespace: logging
+spec:
+  interval: 15m
+  chart:
+    spec:
+      chart: loki
+      version: "~6.6.0"
+      sourceRef:
+        kind: HelmRepository
+        name: grafana
+        namespace: flux-system
+  values:
+    fullnameOverride: loki
+    deploymentMode: SingleBinary
+    loki:
+      auth_enabled: false
+      commonConfig:
+        replication_factor: 1
+      storage:
+        type: filesystem
+      storageConfig:
+        filesystem:
+          directory: /var/loki/chunks
+        tsdb_shipper:
+          active_index_directory: /var/loki/index
+          cache_location: /var/loki/index_cache
+      schemaConfig:
+        configs:
+          - from: 2024-01-01
+            store: tsdb
+            object_store: filesystem
+            schema: v13
+            index:
+              prefix: loki_index_
+              period: 24h
+      compactor:
+        working_directory: /var/loki/compactor
+        shared_store: filesystem
+        retention_enabled: true
+        delete_request_store: filesystem
+      limits_config:
+        retention_period: 4320h
+        reject_old_samples: true
+        reject_old_samples_max_age: 168h
+    singleBinary:
+      replicas: 1
+      persistence:
+        enabled: true
+        size: 200Gi
+        storageClass: asteria
+    service:
+      type: ClusterIP
+    ingress:
+      enabled: false

services/logging/namespace.yaml

@@ -0,0 +1,5 @@
+# services/logging/namespace.yaml
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logging

services/logging/oauth2-proxy.yaml

@@ -0,0 +1,101 @@
+# services/logging/oauth2-proxy.yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: oauth2-proxy-loki
+  namespace: logging
+  labels:
+    app: oauth2-proxy-loki
+spec:
+  ports:
+    - name: http
+      port: 80
+      targetPort: 4180
+  selector:
+    app: oauth2-proxy-loki
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: oauth2-proxy-loki
+  namespace: logging
+  labels:
+    app: oauth2-proxy-loki
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: oauth2-proxy-loki
+  template:
+    metadata:
+      labels:
+        app: oauth2-proxy-loki
+    spec:
+      nodeSelector:
+        node-role.kubernetes.io/worker: "true"
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            - weight: 90
+              preference:
+                matchExpressions:
+                  - key: hardware
+                    operator: In
+                    values: ["rpi5","rpi4"]
+      containers:
+        - name: oauth2-proxy
+          image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --provider=oidc
+            - --redirect-url=https://logs.bstein.dev/oauth2/callback
+            - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas
+            - --scope=openid profile email groups
+            - --email-domain=*
+            - --set-xauthrequest=true
+            - --pass-access-token=true
+            - --set-authorization-header=true
+            - --cookie-secure=true
+            - --cookie-samesite=lax
+            - --cookie-refresh=20m
+            - --cookie-expire=168h
+            - --insecure-oidc-allow-unverified-email=true
+            - --upstream=http://loki.logging.svc.cluster.local:3100
+            - --http-address=0.0.0.0:4180
+            - --skip-provider-button=true
+            - --skip-jwt-bearer-tokens=true
+            - --oidc-groups-claim=groups
+            - --cookie-domain=logs.bstein.dev
+          env:
+            - name: OAUTH2_PROXY_CLIENT_ID
+              valueFrom:
+                secretKeyRef:
+                  name: oauth2-proxy-loki-oidc
+                  key: client_id
+            - name: OAUTH2_PROXY_CLIENT_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: oauth2-proxy-loki-oidc
+                  key: client_secret
+            - name: OAUTH2_PROXY_COOKIE_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: oauth2-proxy-loki-oidc
+                  key: cookie_secret
+          ports:
+            - containerPort: 4180
+              name: http
+          readinessProbe:
+            httpGet:
+              path: /ping
+              port: 4180
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet:
+              path: /ping
+              port: 4180
+            initialDelaySeconds: 20
+            periodSeconds: 20

services/monitoring/helmrelease.yaml

@@ -320,6 +320,13 @@ spec:
               timeInterval: "15s"
             uid: atlas-vm
             orgId: 2
+          - name: Loki
+            type: loki
+            access: proxy
+            url: http://loki.logging.svc.cluster.local:3100
+            isDefault: false
+            uid: atlas-loki
+            orgId: 1
     dashboardProviders:
       dashboardproviders.yaml:
         apiVersion: 1