diff --git a/clusters/atlas/applications/kustomization.yaml b/clusters/atlas/applications/kustomization.yaml index f5c64e8..ed6d795 100644 --- a/clusters/atlas/applications/kustomization.yaml +++ b/clusters/atlas/applications/kustomization.yaml @@ -7,6 +7,7 @@ resources: - ../../services/jellyfin - ../../services/comms - ../../services/monitoring + - ../../services/logging - ../../services/pegasus - ../../services/vault - ../../services/bstein-dev-home diff --git a/clusters/atlas/flux-system/platform/kustomization.yaml b/clusters/atlas/flux-system/platform/kustomization.yaml index e1c5d23..df226e2 100644 --- a/clusters/atlas/flux-system/platform/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/kustomization.yaml @@ -8,5 +8,6 @@ resources: - traefik/kustomization.yaml - gitops-ui/kustomization.yaml - monitoring/kustomization.yaml + - logging/kustomization.yaml - longhorn-ui/kustomization.yaml - ../platform/vault-csi/kustomization.yaml diff --git a/clusters/atlas/flux-system/platform/logging/kustomization.yaml b/clusters/atlas/flux-system/platform/logging/kustomization.yaml new file mode 100644 index 0000000..c51eb5e --- /dev/null +++ b/clusters/atlas/flux-system/platform/logging/kustomization.yaml @@ -0,0 +1,14 @@ +# clusters/atlas/flux-system/platform/logging/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: logging + namespace: flux-system +spec: + interval: 10m + path: ./services/logging + prune: true + sourceRef: + kind: GitRepository + name: flux-system + wait: false diff --git a/infrastructure/sources/helm/fluent-bit.yaml b/infrastructure/sources/helm/fluent-bit.yaml new file mode 100644 index 0000000..b4cb214 --- /dev/null +++ b/infrastructure/sources/helm/fluent-bit.yaml @@ -0,0 +1,9 @@ +# infrastructure/sources/helm/fluent-bit.yaml +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: fluent + namespace: flux-system +spec: + interval: 1h + url: https://fluent.github.io/helm-charts diff --git a/infrastructure/sources/helm/kustomization.yaml b/infrastructure/sources/helm/kustomization.yaml index 7b2163b..1cbf20e 100644 --- a/infrastructure/sources/helm/kustomization.yaml +++ b/infrastructure/sources/helm/kustomization.yaml @@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: + - fluent-bit.yaml - grafana.yaml - hashicorp.yaml - jetstack.yaml diff --git a/services/logging/fluent-bit-helmrelease.yaml b/services/logging/fluent-bit-helmrelease.yaml new file mode 100644 index 0000000..7e0614d --- /dev/null +++ b/services/logging/fluent-bit-helmrelease.yaml @@ -0,0 +1,99 @@ +# services/logging/fluent-bit-helmrelease.yaml +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: fluent-bit + namespace: logging +spec: + interval: 15m + chart: + spec: + chart: fluent-bit + version: "~0.46.0" + sourceRef: + kind: HelmRepository + name: fluent + namespace: flux-system + values: + serviceAccount: + create: true + rbac: + create: true + tolerations: + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + extraVolumes: + - name: runlogjournal + hostPath: + path: /run/log/journal + - name: varlogjournal + hostPath: + path: /var/log/journal + extraVolumeMounts: + - name: runlogjournal + mountPath: /run/log/journal + readOnly: true + - name: varlogjournal + mountPath: /var/log/journal + readOnly: true + config: + service: | + [SERVICE] + Flush 1 + Log_Level info + Daemon Off + Parsers_File parsers.conf + Parsers_File custom_parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + inputs: | + [INPUT] + Name tail + Tag kube.* + Path /var/log/containers/*.log + Parser cri + Mem_Buf_Limit 50MB + Skip_Long_Lines On + Refresh_Interval 10 + Rotate_Wait 30 + storage.type memory + + [INPUT] + Name systemd + Tag journald.* + Read_From_Tail On + storage.type memory + filters: | + [FILTER] + Name kubernetes + Match kube.* + Kube_URL https://kubernetes.default.svc:443 + Kube_CA_File /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + Kube_Token_File /var/run/secrets/kubernetes.io/serviceaccount/token + Merge_Log On + Keep_Log Off + K8S-Logging.Parser On + K8S-Logging.Exclude On + outputs: | + [OUTPUT] + Name loki + Match kube.* + Host loki.logging.svc.cluster.local + Port 3100 + Labels {job="fluent-bit", namespace="$kubernetes['namespace_name']", pod="$kubernetes['pod_name']", container="$kubernetes['container_name']"} + LabelKeys stream + Line_Format json + + [OUTPUT] + Name loki + Match journald.* + Host loki.logging.svc.cluster.local + Port 3100 + Labels {job="systemd"} + LabelKeys _SYSTEMD_UNIT,_HOSTNAME,SYSLOG_IDENTIFIER + Line_Format json diff --git a/services/logging/ingress.yaml b/services/logging/ingress.yaml new file mode 100644 index 0000000..f3211b2 --- /dev/null +++ b/services/logging/ingress.yaml @@ -0,0 +1,25 @@ +# services/logging/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: loki + namespace: logging + annotations: + cert-manager.io/cluster-issuer: letsencrypt +spec: + ingressClassName: traefik + tls: + - hosts: + - logs.bstein.dev + secretName: logs-tls + rules: + - host: logs.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: oauth2-proxy-loki + port: + name: http diff --git a/services/logging/kustomization.yaml b/services/logging/kustomization.yaml new file mode 100644 index 0000000..476d88b --- /dev/null +++ b/services/logging/kustomization.yaml @@ -0,0 +1,9 @@ +# services/logging/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - loki-helmrelease.yaml + - fluent-bit-helmrelease.yaml + - oauth2-proxy.yaml + - ingress.yaml diff --git a/services/logging/loki-helmrelease.yaml b/services/logging/loki-helmrelease.yaml new file mode 100644 index 0000000..a1ce519 --- /dev/null +++ b/services/logging/loki-helmrelease.yaml @@ -0,0 +1,59 @@ +# services/logging/loki-helmrelease.yaml +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: loki + namespace: logging +spec: + interval: 15m + chart: + spec: + chart: loki + version: "~6.6.0" + sourceRef: + kind: HelmRepository + name: grafana + namespace: flux-system + values: + fullnameOverride: loki + deploymentMode: SingleBinary + loki: + auth_enabled: false + commonConfig: + replication_factor: 1 + storage: + type: filesystem + storageConfig: + filesystem: + directory: /var/loki/chunks + tsdb_shipper: + active_index_directory: /var/loki/index + cache_location: /var/loki/index_cache + schemaConfig: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: loki_index_ + period: 24h + compactor: + working_directory: /var/loki/compactor + shared_store: filesystem + retention_enabled: true + delete_request_store: filesystem + limits_config: + retention_period: 4320h + reject_old_samples: true + reject_old_samples_max_age: 168h + singleBinary: + replicas: 1 + persistence: + enabled: true + size: 200Gi + storageClass: asteria + service: + type: ClusterIP + ingress: + enabled: false diff --git a/services/logging/namespace.yaml b/services/logging/namespace.yaml new file mode 100644 index 0000000..ac585d9 --- /dev/null +++ b/services/logging/namespace.yaml @@ -0,0 +1,5 @@ +# services/logging/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: logging diff --git a/services/logging/oauth2-proxy.yaml b/services/logging/oauth2-proxy.yaml new file mode 100644 index 0000000..ac6accc --- /dev/null +++ b/services/logging/oauth2-proxy.yaml @@ -0,0 +1,101 @@ +# services/logging/oauth2-proxy.yaml +apiVersion: v1 +kind: Service +metadata: + name: oauth2-proxy-loki + namespace: logging + labels: + app: oauth2-proxy-loki +spec: + ports: + - name: http + port: 80 + targetPort: 4180 + selector: + app: oauth2-proxy-loki + +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: oauth2-proxy-loki + namespace: logging + labels: + app: oauth2-proxy-loki +spec: + replicas: 2 + selector: + matchLabels: + app: oauth2-proxy-loki + template: + metadata: + labels: + app: oauth2-proxy-loki + spec: + nodeSelector: + node-role.kubernetes.io/worker: "true" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 90 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] + containers: + - name: oauth2-proxy + image: quay.io/oauth2-proxy/oauth2-proxy:v7.6.0 + imagePullPolicy: IfNotPresent + args: + - --provider=oidc + - --redirect-url=https://logs.bstein.dev/oauth2/callback + - --oidc-issuer-url=https://sso.bstein.dev/realms/atlas + - --scope=openid profile email groups + - --email-domain=* + - --set-xauthrequest=true + - --pass-access-token=true + - --set-authorization-header=true + - --cookie-secure=true + - --cookie-samesite=lax + - --cookie-refresh=20m + - --cookie-expire=168h + - --insecure-oidc-allow-unverified-email=true + - --upstream=http://loki.logging.svc.cluster.local:3100 + - --http-address=0.0.0.0:4180 + - --skip-provider-button=true + - --skip-jwt-bearer-tokens=true + - --oidc-groups-claim=groups + - --cookie-domain=logs.bstein.dev + env: + - name: OAUTH2_PROXY_CLIENT_ID + valueFrom: + secretKeyRef: + name: oauth2-proxy-loki-oidc + key: client_id + - name: OAUTH2_PROXY_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: oauth2-proxy-loki-oidc + key: client_secret + - name: OAUTH2_PROXY_COOKIE_SECRET + valueFrom: + secretKeyRef: + name: oauth2-proxy-loki-oidc + key: cookie_secret + ports: + - containerPort: 4180 + name: http + readinessProbe: + httpGet: + path: /ping + port: 4180 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /ping + port: 4180 + initialDelaySeconds: 20 + periodSeconds: 20 diff --git a/services/monitoring/helmrelease.yaml b/services/monitoring/helmrelease.yaml index ddd24e5..873a323 100644 --- a/services/monitoring/helmrelease.yaml +++ b/services/monitoring/helmrelease.yaml @@ -320,6 +320,13 @@ spec: timeInterval: "15s" uid: atlas-vm orgId: 2 + - name: Loki + type: loki + access: proxy + url: http://loki.logging.svc.cluster.local:3100 + isDefault: false + uid: atlas-loki + orgId: 1 dashboardProviders: dashboardproviders.yaml: apiVersion: 1