diff --git a/clusters/atlas/applications/kustomization.yaml b/clusters/atlas/applications/kustomization.yaml index a32ec81..c25257b 100644 --- a/clusters/atlas/applications/kustomization.yaml +++ b/clusters/atlas/applications/kustomization.yaml @@ -5,7 +5,7 @@ resources: - ../../services/crypto - ../../services/gitea - ../../services/jellyfin - - ../../services/jitsi + - ../../services/communication - ../../services/monitoring - ../../services/pegasus - ../../services/vault diff --git a/clusters/atlas/flux-system/applications/communication/kustomization.yaml b/clusters/atlas/flux-system/applications/communication/kustomization.yaml new file mode 100644 index 0000000..0d3b07a --- /dev/null +++ b/clusters/atlas/flux-system/applications/communication/kustomization.yaml @@ -0,0 +1,17 @@ +# clusters/atlas/flux-system/applications/communication/kustomization.yaml +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: communication + namespace: flux-system +spec: + interval: 10m + prune: true + sourceRef: + kind: GitRepository + name: flux-system + path: ./services/communication + targetNamespace: communication + timeout: 2m + dependsOn: + - name: traefik diff --git a/clusters/atlas/flux-system/applications/kustomization.yaml b/clusters/atlas/flux-system/applications/kustomization.yaml index b5a5e62..e1d1feb 100644 --- a/clusters/atlas/flux-system/applications/kustomization.yaml +++ b/clusters/atlas/flux-system/applications/kustomization.yaml @@ -4,7 +4,7 @@ kind: Kustomization resources: - gitea/kustomization.yaml - vault/kustomization.yaml - - jitsi/kustomization.yaml + - communication/kustomization.yaml - crypto/kustomization.yaml - monerod/kustomization.yaml - pegasus/kustomization.yaml diff --git a/clusters/atlas/flux-system/platform/kustomization.yaml b/clusters/atlas/flux-system/platform/kustomization.yaml index fbca36e..e1c5d23 100644 --- a/clusters/atlas/flux-system/platform/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/kustomization.yaml @@ -4,6 +4,7 @@ kind: Kustomization resources: - core/kustomization.yaml - helm/kustomization.yaml + - metallb/kustomization.yaml - traefik/kustomization.yaml - gitops-ui/kustomization.yaml - monitoring/kustomization.yaml diff --git a/clusters/atlas/flux-system/applications/jitsi/kustomization.yaml b/clusters/atlas/flux-system/platform/metallb/kustomization.yaml similarity index 53% rename from clusters/atlas/flux-system/applications/jitsi/kustomization.yaml rename to clusters/atlas/flux-system/platform/metallb/kustomization.yaml index 8e96feb..98baaff 100644 --- a/clusters/atlas/flux-system/applications/jitsi/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/metallb/kustomization.yaml @@ -1,19 +1,16 @@ -# clusters/atlas/flux-system/applications/jitsi/kustomization.yaml +# clusters/atlas/flux-system/platform/metallb/kustomization.yaml apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: jitsi + name: metallb namespace: flux-system spec: - interval: 10m - path: ./services/jitsi - targetNamespace: jitsi - prune: true + interval: 30m sourceRef: kind: GitRepository name: flux-system namespace: flux-system - dependsOn: - - name: core + path: ./infrastructure/metallb + prune: true wait: true - timeout: 5m + targetNamespace: metallb-system diff --git a/clusters/atlas/flux-system/platform/traefik/kustomization.yaml b/clusters/atlas/flux-system/platform/traefik/kustomization.yaml index 0f53de7..336eb89 100644 --- a/clusters/atlas/flux-system/platform/traefik/kustomization.yaml +++ b/clusters/atlas/flux-system/platform/traefik/kustomization.yaml @@ -15,4 +15,5 @@ spec: namespace: flux-system dependsOn: - name: core + - name: metallb wait: true diff --git a/clusters/atlas/platform/kustomization.yaml b/clusters/atlas/platform/kustomization.yaml index c7b144a..43fa993 100644 --- a/clusters/atlas/platform/kustomization.yaml +++ b/clusters/atlas/platform/kustomization.yaml @@ -5,3 +5,4 @@ resources: - ../../../infrastructure/modules/base - ../../../infrastructure/modules/profiles/atlas-ha - ../../../infrastructure/sources/cert-manager/letsencrypt.yaml + - ../../../infrastructure/metallb diff --git a/infrastructure/metallb/ippool.yaml b/infrastructure/metallb/ippool.yaml new file mode 100644 index 0000000..e792280 --- /dev/null +++ b/infrastructure/metallb/ippool.yaml @@ -0,0 +1,20 @@ +# infrastructure/metallb/ippool.yaml +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: communication-pool + namespace: metallb-system +spec: + addresses: + - 192.168.22.4-192.168.22.6 + - 192.168.22.9-192.168.22.9 + autoAssign: true +--- +apiVersion: metallb.io/v1beta1 +kind: L2Advertisement +metadata: + name: communication-adv + namespace: metallb-system +spec: + ipAddressPools: + - communication-pool diff --git a/infrastructure/metallb/kustomization.yaml b/infrastructure/metallb/kustomization.yaml new file mode 100644 index 0000000..f6df7e6 --- /dev/null +++ b/infrastructure/metallb/kustomization.yaml @@ -0,0 +1,9 @@ +# infrastructure/metallb/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - namespace.yaml + - metallb-rendered.yaml + - ippool.yaml +patchesStrategicMerge: + - patches/node-placement.yaml diff --git a/infrastructure/metallb/metallb-rendered.yaml b/infrastructure/metallb/metallb-rendered.yaml new file mode 100644 index 0000000..0f8ad10 --- /dev/null +++ b/infrastructure/metallb/metallb-rendered.yaml @@ -0,0 +1,2411 @@ +--- +# Source: metallb/templates/service-accounts.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metallb-controller + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller +--- +# Source: metallb/templates/service-accounts.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: metallb-speaker + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: speaker +--- +# Source: metallb/templates/webhooks.yaml +apiVersion: v1 +kind: Secret +metadata: + name: metallb-webhook-cert + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +--- +# Source: metallb/templates/exclude-l2-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: metallb-excludel2 + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +data: + excludel2.yaml: | + announcedInterfacesToExclude: + - ^docker.* + - ^cbr.* + - ^dummy.* + - ^virbr.* + - ^lxcbr.* + - ^veth.* + - ^lo$ + - ^cali.* + - ^tunl.* + - ^flannel.* + - ^kube-ipvs.* + - ^cni.* + - ^nodelocaldns.* + - ^lxc.* +--- +# Source: metallb/templates/speaker.yaml +# FRR expects to have these files owned by frr:frr on startup. +# Having them in a ConfigMap allows us to modify behaviors: for example enabling more daemons on startup. +apiVersion: v1 +kind: ConfigMap +metadata: + name: metallb-frr-startup + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: speaker +data: + daemons: | + # This file tells the frr package which daemons to start. + # + # Sample configurations for these daemons can be found in + # /usr/share/doc/frr/examples/. + # + # ATTENTION: + # + # When activating a daemon for the first time, a config file, even if it is + # empty, has to be present *and* be owned by the user and group "frr", else + # the daemon will not be started by /etc/init.d/frr. The permissions should + # be u=rw,g=r,o=. + # When using "vtysh" such a config file is also needed. It should be owned by + # group "frrvty" and set to ug=rw,o= though. Check /etc/pam.d/frr, too. + # + # The watchfrr and zebra daemons are always started. + # + bgpd=yes + ospfd=no + ospf6d=no + ripd=no + ripngd=no + isisd=no + pimd=no + ldpd=no + nhrpd=no + eigrpd=no + babeld=no + sharpd=no + pbrd=no + bfdd=yes + fabricd=no + vrrpd=no + + # + # If this option is set the /etc/init.d/frr script automatically loads + # the config via "vtysh -b" when the servers are started. + # Check /etc/pam.d/frr if you intend to use "vtysh"! + # + vtysh_enable=yes + zebra_options=" -A 127.0.0.1 -s 90000000 --limit-fds 100000" + bgpd_options=" -A 127.0.0.1 -p 0 --limit-fds 100000" + ospfd_options=" -A 127.0.0.1" + ospf6d_options=" -A ::1" + ripd_options=" -A 127.0.0.1" + ripngd_options=" -A ::1" + isisd_options=" -A 127.0.0.1" + pimd_options=" -A 127.0.0.1" + ldpd_options=" -A 127.0.0.1" + nhrpd_options=" -A 127.0.0.1" + eigrpd_options=" -A 127.0.0.1" + babeld_options=" -A 127.0.0.1" + sharpd_options=" -A 127.0.0.1" + pbrd_options=" -A 127.0.0.1" + staticd_options="-A 127.0.0.1 --limit-fds 100000" + bfdd_options=" -A 127.0.0.1 --limit-fds 100000" + fabricd_options="-A 127.0.0.1" + vrrpd_options=" -A 127.0.0.1" + + # configuration profile + # + #frr_profile="traditional" + #frr_profile="datacenter" + + # + # This is the maximum number of FD's that will be available. + # Upon startup this is read by the control files and ulimit + # is called. Uncomment and use a reasonable value for your + # setup if you are expecting a large number of peers in + # say BGP. + #MAX_FDS=1024 + + # The list of daemons to watch is automatically generated by the init script. + #watchfrr_options="" + + # for debugging purposes, you can specify a "wrap" command to start instead + # of starting the daemon directly, e.g. to use valgrind on ospfd: + # ospfd_wrap="/usr/bin/valgrind" + # or you can use "all_wrap" for all daemons, e.g. to use perf record: + # all_wrap="/usr/bin/perf record --call-graph -" + # the normal daemon command is added to this at the end. + vtysh.conf: |+ + service integrated-vtysh-config + frr.conf: |+ + ! This file gets overriden the first time the speaker renders a config. + ! So anything configured here is only temporary. + frr version 8.0 + frr defaults traditional + hostname Router + line vty + log file /etc/frr/frr.log informational +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: bfdprofiles.metallb.io +spec: + group: metallb.io + names: + kind: BFDProfile + listKind: BFDProfileList + plural: bfdprofiles + singular: bfdprofile + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.passiveMode + name: Passive Mode + type: boolean + - jsonPath: .spec.transmitInterval + name: Transmit Interval + type: integer + - jsonPath: .spec.receiveInterval + name: Receive Interval + type: integer + - jsonPath: .spec.detectMultiplier + name: Multiplier + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + BFDProfile represents the settings of the bfd session that can be + optionally associated with a BGP session. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BFDProfileSpec defines the desired state of BFDProfile. + properties: + detectMultiplier: + description: |- + Configures the detection multiplier to determine + packet loss. The remote transmission interval will be multiplied + by this value to determine the connection loss detection timer. + format: int32 + maximum: 255 + minimum: 2 + type: integer + echoInterval: + description: |- + Configures the minimal echo receive transmission + interval that this system is capable of handling in milliseconds. + Defaults to 50ms + format: int32 + maximum: 60000 + minimum: 10 + type: integer + echoMode: + description: |- + Enables or disables the echo transmission mode. + This mode is disabled by default, and not supported on multi + hops setups. + type: boolean + minimumTtl: + description: |- + For multi hop sessions only: configure the minimum + expected TTL for an incoming BFD control packet. + format: int32 + maximum: 254 + minimum: 1 + type: integer + passiveMode: + description: |- + Mark session as passive: a passive session will not + attempt to start the connection and will wait for control packets + from peer before it begins replying. + type: boolean + receiveInterval: + description: |- + The minimum interval that this system is capable of + receiving control packets in milliseconds. + Defaults to 300ms. + format: int32 + maximum: 60000 + minimum: 10 + type: integer + transmitInterval: + description: |- + The minimum transmission interval (less jitter) + that this system wants to use to send BFD control packets in + milliseconds. Defaults to 300ms + format: int32 + maximum: 60000 + minimum: 10 + type: integer + type: object + status: + description: BFDProfileStatus defines the observed state of BFDProfile. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: bgpadvertisements.metallb.io +spec: + group: metallb.io + names: + kind: BGPAdvertisement + listKind: BGPAdvertisementList + plural: bgpadvertisements + singular: bgpadvertisement + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.ipAddressPools + name: IPAddressPools + type: string + - jsonPath: .spec.ipAddressPoolSelectors + name: IPAddressPool Selectors + type: string + - jsonPath: .spec.peers + name: Peers + type: string + - jsonPath: .spec.nodeSelectors + name: Node Selectors + priority: 10 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + BGPAdvertisement allows to advertise the IPs coming + from the selected IPAddressPools via BGP, setting the parameters of the + BGP Advertisement. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BGPAdvertisementSpec defines the desired state of BGPAdvertisement. + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets you “roll up” the /32s into a larger prefix. Defaults to 32. Works for IPv4 addresses. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: The aggregation-length advertisement option lets you “roll up” the /128s into a larger prefix. Defaults to 128. Works for IPv6 addresses. + format: int32 + type: integer + communities: + description: |- + The BGP communities to be associated with the announcement. Each item can be a standard community of the + form 1234:1234, a large community of the form large:1234:1234:1234 or the name of an alias defined in the + Community CRD. + items: + type: string + type: array + ipAddressPoolSelectors: + description: |- + A selector for the IPAddressPools which would get advertised via this advertisement. + If no IPAddressPool is selected by this or by the list, the advertisement is applied to all the IPAddressPools. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + ipAddressPools: + description: The list of IPAddressPools to advertise via this advertisement, selected by name. + items: + type: string + type: array + localPref: + description: |- + The BGP LOCAL_PREF attribute which is used by BGP best path algorithm, + Path with higher localpref is preferred over one with lower localpref. + format: int32 + type: integer + nodeSelectors: + description: NodeSelectors allows to limit the nodes to announce as next hops for the LoadBalancer IP. When empty, all the nodes having are announced as next hops. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + peers: + description: |- + Peers limits the bgppeer to advertise the ips of the selected pools to. + When empty, the loadbalancer IP is announced to all the BGPPeers configured. + items: + type: string + type: array + type: object + status: + description: BGPAdvertisementStatus defines the observed state of BGPAdvertisement. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: bgppeers.metallb.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /convert + conversionReviewVersions: + - v1beta1 + - v1beta2 + group: metallb.io + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.peerAddress + name: Address + type: string + - jsonPath: .spec.peerASN + name: ASN + type: string + - jsonPath: .spec.bfdProfile + name: BFD Profile + type: string + - jsonPath: .spec.ebgpMultiHop + name: Multi Hops + type: string + deprecated: true + deprecationWarning: v1beta1 is deprecated, please use v1beta2 + name: v1beta1 + schema: + openAPIV3Schema: + description: BGPPeer is the Schema for the peers API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BGPPeerSpec defines the desired state of Peer. + properties: + bfdProfile: + type: string + ebgpMultiHop: + description: EBGP peer is multi-hops away + type: boolean + holdTime: + description: Requested BGP hold time, per RFC4271. + type: string + keepaliveTime: + description: Requested BGP keepalive time, per RFC4271. + type: string + myASN: + description: AS number to use for the local end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + nodeSelectors: + description: |- + Only connect to this peer on nodes that match one of these + selectors. + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + minItems: 1 + type: array + required: + - key + - operator + - values + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + password: + description: Authentication password for routers enforcing TCP MD5 authenticated sessions + type: string + peerASN: + description: AS number to expect from the remote end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + peerAddress: + description: Address to dial when establishing the session. + type: string + peerPort: + description: Port to dial when establishing the session. + maximum: 16384 + minimum: 0 + type: integer + routerID: + description: BGP router ID to advertise to the peer + type: string + sourceAddress: + description: Source address to use when establishing the session. + type: string + required: + - myASN + - peerASN + - peerAddress + type: object + status: + description: BGPPeerStatus defines the observed state of Peer. + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.peerAddress + name: Address + type: string + - jsonPath: .spec.peerASN + name: ASN + type: string + - jsonPath: .spec.bfdProfile + name: BFD Profile + type: string + - jsonPath: .spec.ebgpMultiHop + name: Multi Hops + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BGPPeer is the Schema for the peers API. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: BGPPeerSpec defines the desired state of Peer. + properties: + bfdProfile: + description: The name of the BFD Profile to be used for the BFD session associated to the BGP session. If not set, the BFD session won't be set up. + type: string + connectTime: + description: Requested BGP connect time, controls how long BGP waits between connection attempts to a neighbor. + type: string + x-kubernetes-validations: + - message: connect time should be between 1 seconds to 65535 + rule: duration(self).getSeconds() >= 1 && duration(self).getSeconds() <= 65535 + - message: connect time should contain a whole number of seconds + rule: duration(self).getMilliseconds() % 1000 == 0 + disableMP: + default: false + description: |- + To set if we want to disable MP BGP that will separate IPv4 and IPv6 route exchanges into distinct BGP sessions. + Deprecated: DisableMP is deprecated in favor of dualStackAddressFamily. + type: boolean + dualStackAddressFamily: + default: false + description: |- + To set if we want to enable the neighbor not only for the ipfamily related to its session, + but also the other one. This allows to advertise/receive IPv4 prefixes over IPv6 sessions and vice versa. + type: boolean + dynamicASN: + description: |- + DynamicASN detects the AS number to use for the remote end of the session + without explicitly setting it via the ASN field. Limited to: + internal - if the neighbor's ASN is different than MyASN connection is denied. + external - if the neighbor's ASN is the same as MyASN the connection is denied. + ASN and DynamicASN are mutually exclusive and one of them must be specified. + enum: + - internal + - external + type: string + ebgpMultiHop: + description: To set if the BGPPeer is multi-hops away. Needed for FRR mode only. + type: boolean + enableGracefulRestart: + description: |- + EnableGracefulRestart allows BGP peer to continue to forward data packets + along known routes while the routing protocol information is being + restored. This field is immutable because it requires restart of the BGP + session. Supported for FRR mode only. + type: boolean + x-kubernetes-validations: + - message: EnableGracefulRestart cannot be changed after creation + rule: self == oldSelf + holdTime: + description: Requested BGP hold time, per RFC4271. + type: string + interface: + description: |- + Interface is the node interface over which the unnumbered BGP peering will + be established. No API validation takes place as that string value + represents an interface name on the host and if user provides an invalid + value, only the actual BGP session will not be established. + Address and Interface are mutually exclusive and one of them must be specified. + type: string + keepaliveTime: + description: Requested BGP keepalive time, per RFC4271. + type: string + myASN: + description: AS number to use for the local end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + nodeSelectors: + description: |- + Only connect to this peer on nodes that match one of these + selectors. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + password: + description: Authentication password for routers enforcing TCP MD5 authenticated sessions + type: string + passwordSecret: + description: |- + passwordSecret is name of the authentication secret for BGP Peer. + the secret must be of type "kubernetes.io/basic-auth", and created in the + same namespace as the MetalLB deployment. The password is stored in the + secret as the key "password". + properties: + name: + description: name is unique within a namespace to reference a secret resource. + type: string + namespace: + description: namespace defines the space within which the secret name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + peerASN: + description: |- + AS number to expect from the remote end of the session. + ASN and DynamicASN are mutually exclusive and one of them must be specified. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + peerAddress: + description: Address to dial when establishing the session. + type: string + peerPort: + default: 179 + description: Port to dial when establishing the session. + maximum: 16384 + minimum: 1 + type: integer + routerID: + description: BGP router ID to advertise to the peer + type: string + sourceAddress: + description: Source address to use when establishing the session. + type: string + vrf: + description: |- + To set if we want to peer with the BGPPeer using an interface belonging to + a host vrf + type: string + required: + - myASN + type: object + status: + description: BGPPeerStatus defines the observed state of Peer. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: communities.metallb.io +spec: + group: metallb.io + names: + kind: Community + listKind: CommunityList + plural: communities + singular: community + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: |- + Community is a collection of aliases for communities. + Users can define named aliases to be used in the BGPPeer CRD. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: CommunitySpec defines the desired state of Community. + properties: + communities: + items: + properties: + name: + description: The name of the alias for the community. + type: string + value: + description: |- + The BGP community value corresponding to the given name. Can be a standard community of the form 1234:1234 + or a large community of the form large:1234:1234:1234. + type: string + type: object + type: array + type: object + status: + description: CommunityStatus defines the observed state of Community. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: configurationstates.metallb.io +spec: + group: metallb.io + names: + kind: ConfigurationState + listKind: ConfigurationStateList + plural: configurationstates + singular: configurationstate + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.result + name: Result + type: string + - jsonPath: .status.errorSummary + name: ErrorSummary + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + ConfigurationState is a status-only CRD that reports configuration validation results from MetalLB components. + Labels: + - metallb.io/component-type: "controller" or "speaker" + - metallb.io/node-name: node name (only for speaker) + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + description: ConfigurationStateStatus defines the observed state of ConfigurationState. + properties: + conditions: + description: Conditions contains the status conditions from the reconcilers running in this component. + items: + description: Condition contains details for one aspect of the current state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + errorSummary: + description: |- + ErrorSummary contains the aggregated error messages from reconciliation failures. + This field is empty when Result is "Valid". + type: string + result: + description: Result indicates the configuration validation result. + enum: + - Valid + - Invalid + - Unknown + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: ipaddresspools.metallb.io +spec: + group: metallb.io + names: + kind: IPAddressPool + listKind: IPAddressPoolList + plural: ipaddresspools + singular: ipaddresspool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.autoAssign + name: Auto Assign + type: boolean + - jsonPath: .spec.avoidBuggyIPs + name: Avoid Buggy IPs + type: boolean + - jsonPath: .spec.addresses + name: Addresses + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + IPAddressPool represents a pool of IP addresses that can be allocated + to LoadBalancer services. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: IPAddressPoolSpec defines the desired state of IPAddressPool. + properties: + addresses: + description: |- + A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share the + same settings. Each range can be either a CIDR prefix, or an explicit + start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: |- + AutoAssign flag used to prevent MetallB from automatic allocation + for a pool. + type: boolean + avoidBuggyIPs: + default: false + description: |- + AvoidBuggyIPs prevents addresses ending with .0 and .255 + to be used by a pool. + type: boolean + serviceAllocation: + description: |- + AllocateTo makes ip pool allocation to specific namespace and/or service. + The controller will use the pool with lowest value of priority in case of + multiple matches. A pool with no priority set will be used only if the + pools with priority can't be used. If multiple matching IPAddressPools are + available it will check for the availability of IPs sorting the matching + IPAddressPools by priority, starting from the highest to the lowest. If + multiple IPAddressPools have the same priority, choice will be random. + properties: + namespaceSelectors: + description: |- + NamespaceSelectors list of label selectors to select namespace(s) for ip pool, + an alternative to using namespace list. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + namespaces: + description: Namespaces list of namespace(s) on which ip pool can be attached. + items: + type: string + type: array + priority: + description: Priority priority given for ip pool while ip allocation on a service. + type: integer + serviceSelectors: + description: |- + ServiceSelectors list of label selector to select service(s) for which ip pool + can be used for ip allocation. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + type: object + required: + - addresses + type: object + status: + description: IPAddressPoolStatus defines the observed state of IPAddressPool. + properties: + assignedIPv4: + description: AssignedIPv4 is the number of assigned IPv4 addresses. + format: int64 + type: integer + assignedIPv6: + description: AssignedIPv6 is the number of assigned IPv6 addresses. + format: int64 + type: integer + availableIPv4: + description: AvailableIPv4 is the number of available IPv4 addresses. + format: int64 + type: integer + availableIPv6: + description: AvailableIPv6 is the number of available IPv6 addresses. + format: int64 + type: integer + required: + - assignedIPv4 + - assignedIPv6 + - availableIPv4 + - availableIPv6 + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: l2advertisements.metallb.io +spec: + group: metallb.io + names: + kind: L2Advertisement + listKind: L2AdvertisementList + plural: l2advertisements + singular: l2advertisement + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.ipAddressPools + name: IPAddressPools + type: string + - jsonPath: .spec.ipAddressPoolSelectors + name: IPAddressPool Selectors + type: string + - jsonPath: .spec.interfaces + name: Interfaces + type: string + - jsonPath: .spec.nodeSelectors + name: Node Selectors + priority: 10 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: |- + L2Advertisement allows to advertise the LoadBalancer IPs provided + by the selected pools via L2. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: L2AdvertisementSpec defines the desired state of L2Advertisement. + properties: + interfaces: + description: |- + A list of interfaces to announce from. The LB IP will be announced only from these interfaces. + If the field is not set, we advertise from all the interfaces on the host. + items: + type: string + type: array + ipAddressPoolSelectors: + description: |- + A selector for the IPAddressPools which would get advertised via this advertisement. + If no IPAddressPool is selected by this or by the list, the advertisement is applied to all the IPAddressPools. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + ipAddressPools: + description: The list of IPAddressPools to advertise via this advertisement, selected by name. + items: + type: string + type: array + nodeSelectors: + description: NodeSelectors allows to limit the nodes to announce as next hops for the LoadBalancer IP. When empty, all the nodes having are announced as next hops. + items: + description: |- + A label selector is a label query over a set of resources. The result of matchLabels and + matchExpressions are ANDed. An empty label selector matches all objects. A null + label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + type: object + status: + description: L2AdvertisementStatus defines the observed state of L2Advertisement. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: servicebgpstatuses.metallb.io +spec: + group: metallb.io + names: + kind: ServiceBGPStatus + listKind: ServiceBGPStatusList + plural: servicebgpstatuses + singular: servicebgpstatus + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.node + name: Node + type: string + - jsonPath: .status.serviceName + name: Service Name + type: string + - jsonPath: .status.serviceNamespace + name: Service Namespace + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ServiceBGPStatus exposes the BGP peers a service is configured to be advertised to, per relevant node. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ServiceBGPStatusSpec defines the desired state of ServiceBGPStatus. + type: object + status: + description: MetalLBServiceBGPStatus defines the observed state of ServiceBGPStatus. + properties: + node: + description: Node indicates the node announcing the service. + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + peers: + description: |- + Peers indicate the BGP peers for which the service is configured to be advertised to. + The service being actually advertised to a given peer depends on the session state and is not indicated here. + items: + type: string + type: array + serviceName: + description: ServiceName indicates the service this status represents. + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + serviceNamespace: + description: ServiceNamespace indicates the namespace of the service. + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/charts/crds/templates/crds.yaml +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + name: servicel2statuses.metallb.io +spec: + group: metallb.io + names: + kind: ServiceL2Status + listKind: ServiceL2StatusList + plural: servicel2statuses + singular: servicel2status + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.node + name: Allocated Node + type: string + - jsonPath: .status.serviceName + name: Service Name + type: string + - jsonPath: .status.serviceNamespace + name: Service Namespace + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: ServiceL2Status reveals the actual traffic status of loadbalancer services in layer2 mode. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ServiceL2StatusSpec defines the desired state of ServiceL2Status. + type: object + status: + description: MetalLBServiceL2Status defines the observed state of ServiceL2Status. + properties: + interfaces: + description: Interfaces indicates the interfaces that receive the directed traffic + items: + description: InterfaceInfo defines interface info of layer2 announcement. + properties: + name: + description: Name the name of network interface card + type: string + type: object + type: array + node: + description: Node indicates the node that receives the directed traffic + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + serviceName: + description: ServiceName indicates the service this status represents + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + serviceNamespace: + description: ServiceNamespace indicates the namespace of the service + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metallb:controller + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: [""] + resources: ["services", "namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] +- apiGroups: [""] + resources: ["services/status"] + verbs: ["update"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + resourceNames: ["metallb-webhook-configuration"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["admissionregistration.k8s.io"] + resources: ["validatingwebhookconfigurations"] + verbs: ["list", "watch"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["bfdprofiles.metallb.io","bgpadvertisements.metallb.io", + "bgppeers.metallb.io","ipaddresspools.metallb.io","l2advertisements.metallb.io","communities.metallb.io","configurationstates.metallb.io"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["configurationstates"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["metallb.io"] + resources: ["configurationstates/status"] + verbs: ["get", "patch", "update"] +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: metallb:speaker + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: [""] + resources: ["services", "endpoints", "nodes", "namespaces"] + verbs: ["get", "list", "watch"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create", "patch"] +- apiGroups: ["metallb.io"] + resources: ["servicel2statuses","servicel2statuses/status","configurationstates","configurationstates/status"] + verbs: ["*"] +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metallb:controller + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +subjects: +- kind: ServiceAccount + name: metallb-controller + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb:controller +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: metallb:speaker + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +subjects: +- kind: ServiceAccount + name: metallb-speaker + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb:speaker +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metallb-pod-lister + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "get"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["bfdprofiles"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["bgppeers"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["l2advertisements"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["bgpadvertisements"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["ipaddresspools"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["communities"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["servicebgpstatuses","servicebgpstatuses/status"] + verbs: ["*"] +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: metallb-controller + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "list", "watch"] +- apiGroups: [""] + resources: ["secrets"] + resourceNames: ["metallb-memberlist"] + verbs: ["list"] +- apiGroups: ["apps"] + resources: ["deployments"] + resourceNames: ["metallb-controller"] + verbs: ["get"] +- apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "delete", "get", "list", "patch", "update", "watch"] +- apiGroups: ["metallb.io"] + resources: ["ipaddresspools"] + verbs: ["get", "list", "watch"] +- apiGroups: ["metallb.io"] + resources: ["ipaddresspools/status"] + verbs: ["update"] +- apiGroups: ["metallb.io"] + resources: ["bgppeers"] + verbs: ["get", "list"] +- apiGroups: ["metallb.io"] + resources: ["bgpadvertisements"] + verbs: ["get", "list"] +- apiGroups: ["metallb.io"] + resources: ["l2advertisements"] + verbs: ["get", "list"] +- apiGroups: ["metallb.io"] + resources: ["communities"] + verbs: ["get", "list","watch"] +- apiGroups: ["metallb.io"] + resources: ["bfdprofiles"] + verbs: ["get", "list","watch"] +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metallb-pod-lister + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: metallb-pod-lister +subjects: +- kind: ServiceAccount + name: metallb-speaker +--- +# Source: metallb/templates/rbac.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: metallb-controller + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: metallb-controller +subjects: +- kind: ServiceAccount + name: metallb-controller +--- +# Source: metallb/templates/webhooks.yaml +apiVersion: v1 +kind: Service +metadata: + name: metallb-webhook-service + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/component: controller +--- +# Source: metallb/templates/speaker.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metallb-speaker + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: speaker +spec: + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/component: speaker + template: + metadata: + labels: + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/component: speaker + spec: + serviceAccountName: metallb-speaker + terminationGracePeriodSeconds: 0 + hostNetwork: true + volumes: + - name: memberlist + secret: + secretName: metallb-memberlist + defaultMode: 420 + - name: metallb-excludel2 + configMap: + defaultMode: 256 + name: metallb-excludel2 + - name: frr-sockets + emptyDir: {} + - name: frr-startup + configMap: + name: metallb-frr-startup + - name: frr-conf + emptyDir: {} + - name: reloader + emptyDir: {} + - name: metrics + emptyDir: {} + - name: frr-tmp + emptyDir: {} + - name: frr-lib + emptyDir: {} + - name: frr-log + emptyDir: {} + initContainers: + # Copies the initial config files with the right permissions to the shared volume. + - name: cp-frr-files + image: quay.io/frrouting/frr:10.4.1 + securityContext: + runAsUser: 100 + runAsGroup: 101 + command: ["/bin/sh", "-c", "cp -rLf /tmp/frr/* /etc/frr/"] + volumeMounts: + - name: frr-startup + mountPath: /tmp/frr + - name: frr-conf + mountPath: /etc/frr + # Copies the reloader to the shared volume between the speaker and reloader. + - name: cp-reloader + image: quay.io/metallb/speaker:v0.15.3 + command: ["/cp-tool","/frr-reloader.sh","/etc/frr_reloader/frr-reloader.sh"] + volumeMounts: + - name: reloader + mountPath: /etc/frr_reloader + # Copies the metrics exporter + - name: cp-metrics + image: quay.io/metallb/speaker:v0.15.3 + command: ["/cp-tool","/frr-metrics","/etc/frr_metrics/frr-metrics"] + volumeMounts: + - name: metrics + mountPath: /etc/frr_metrics + shareProcessNamespace: true + containers: + - name: speaker + image: quay.io/metallb/speaker:v0.15.3 + args: + - --port=7472 + - --log-level=info + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: METALLB_ML_BIND_ADDR + valueFrom: + fieldRef: + fieldPath: status.podIP + + - name: METALLB_ML_LABELS + value: "app.kubernetes.io/name=metallb,app.kubernetes.io/component=speaker" + - name: METALLB_ML_BIND_PORT + value: "7946" + - name: METALLB_ML_SECRET_KEY_PATH + value: "/etc/ml_secret_key" + - name: FRR_CONFIG_FILE + value: /etc/frr_reloader/frr.conf + - name: FRR_RELOADER_PID_FILE + value: /etc/frr_reloader/reloader.pid + - name: METALLB_BGP_TYPE + value: frr + - name: METALLB_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + ports: + - name: monitoring + containerPort: 7472 + - name: memberlist-tcp + containerPort: 7946 + protocol: TCP + - name: memberlist-udp + containerPort: 7946 + protocol: UDP + livenessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + add: + - NET_RAW + volumeMounts: + - name: memberlist + mountPath: /etc/ml_secret_key + - name: reloader + mountPath: /etc/frr_reloader + - name: metallb-excludel2 + mountPath: /etc/metallb + - name: frr + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + add: + - NET_ADMIN + - NET_RAW + - SYS_ADMIN + - NET_BIND_SERVICE + image: quay.io/frrouting/frr:10.4.1 + env: + - name: TINI_SUBREAPER + value: "true" + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: frr-tmp + mountPath: /var/tmp/frr + - name: frr-lib + mountPath: /var/lib/frr + # The command is FRR's default entrypoint & waiting for the log file to appear and tailing it. + # If the log file isn't created in 60 seconds the tail fails and the container is restarted. + # This workaround is needed to have the frr logs as part of kubectl logs -c frr < speaker_pod_name >. + command: + - /bin/sh + - -c + - | + /sbin/tini -- /usr/lib/frr/docker-start & + attempts=0 + until [[ -f /etc/frr/frr.log || $attempts -eq 60 ]]; do + sleep 1 + attempts=$(( $attempts + 1 )) + done + tail -f /etc/frr/frr.log + livenessProbe: + httpGet: + path: livez + port: 7473 + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + startupProbe: + httpGet: + path: /livez + port: 7473 + failureThreshold: 30 + periodSeconds: 5 + - name: reloader + image: quay.io/frrouting/frr:10.4.1 + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + command: ["/etc/frr_reloader/frr-reloader.sh"] + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: reloader + mountPath: /etc/frr_reloader + - name: frr-log + mountPath: /var/log/frr + - name: frr-metrics + image: quay.io/frrouting/frr:10.4.1 + securityContext: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + command: ["/etc/frr_metrics/frr-metrics"] + args: + - --metrics-port=7473 + env: + - name: VTYSH_HISTFILE + value: /dev/null + ports: + - containerPort: 7473 + name: frrmetrics + volumeMounts: + - name: frr-sockets + mountPath: /var/run/frr + - name: frr-conf + mountPath: /etc/frr + - name: metrics + mountPath: /etc/frr_metrics + nodeSelector: + "kubernetes.io/os": linux + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + operator: Exists + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + operator: Exists +--- +# Source: metallb/templates/controller.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metallb-controller + namespace: "metallb-system" + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: controller +spec: + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/component: controller + template: + metadata: + labels: + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/component: controller + spec: + serviceAccountName: metallb-controller + terminationGracePeriodSeconds: 0 + securityContext: + fsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + containers: + - name: controller + image: quay.io/metallb/controller:v0.15.3 + args: + - --port=7472 + - --log-level=info + - --webhook-mode=enabled + - --tls-min-version=VersionTLS12 + env: + - name: METALLB_ML_SECRET_NAME + value: metallb-memberlist + - name: METALLB_DEPLOYMENT + value: metallb-controller + - name: METALLB_BGP_TYPE + value: frr + ports: + - name: monitoring + containerPort: 7472 + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + livenessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + nodeSelector: + "kubernetes.io/os": linux + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: metallb-webhook-cert +--- +# Source: metallb/templates/webhooks.yaml +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: metallb-webhook-configuration + labels: + helm.sh/chart: metallb-0.15.3 + app.kubernetes.io/name: metallb + app.kubernetes.io/instance: metallb + app.kubernetes.io/version: "v0.15.3" + app.kubernetes.io/managed-by: Helm +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta2-bgppeer + failurePolicy: Fail + name: bgppeervalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - bgppeers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-ipaddresspool + failurePolicy: Fail + name: ipaddresspoolvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ipaddresspools + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-bgpadvertisement + failurePolicy: Fail + name: bgpadvertisementvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - bgpadvertisements + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-community + failurePolicy: Fail + name: communityvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - communities + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-bfdprofile + failurePolicy: Fail + name: bfdprofilevalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - DELETE + resources: + - bfdprofiles + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: metallb-webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-l2advertisement + failurePolicy: Fail + name: l2advertisementvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - l2advertisements + sideEffects: None diff --git a/infrastructure/metallb/namespace.yaml b/infrastructure/metallb/namespace.yaml new file mode 100644 index 0000000..02b2add --- /dev/null +++ b/infrastructure/metallb/namespace.yaml @@ -0,0 +1,5 @@ +# infrastructure/metallb/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: metallb-system diff --git a/infrastructure/metallb/patches/node-placement.yaml b/infrastructure/metallb/patches/node-placement.yaml new file mode 100644 index 0000000..e32337e --- /dev/null +++ b/infrastructure/metallb/patches/node-placement.yaml @@ -0,0 +1,30 @@ +# infrastructure/metallb/patches/node-placement.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metallb-controller + namespace: metallb-system +spec: + template: + spec: + containers: + - name: controller + args: + - --port=7472 + - --log-level=info + - --webhook-mode=enabled + - --tls-min-version=VersionTLS12 + - --lb-class=metallb + nodeSelector: + hardware: rpi5 +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metallb-speaker + namespace: metallb-system +spec: + template: + spec: + nodeSelector: + hardware: rpi5 diff --git a/infrastructure/traefik/clusterrole.yaml b/infrastructure/traefik/clusterrole.yaml index 52ed126..353eaff 100644 --- a/infrastructure/traefik/clusterrole.yaml +++ b/infrastructure/traefik/clusterrole.yaml @@ -71,9 +71,10 @@ rules: - tlsoptions - tlsstores - serverstransports + - serverstransporttcps - traefikservices + - middlewaretcps verbs: - get - list - watch - diff --git a/infrastructure/traefik/kustomization.yaml b/infrastructure/traefik/kustomization.yaml index 1dce445..4e36574 100644 --- a/infrastructure/traefik/kustomization.yaml +++ b/infrastructure/traefik/kustomization.yaml @@ -10,3 +10,4 @@ resources: - clusterrole.yaml - clusterrolebinding.yaml - service.yaml + - traefik-service-lb.yaml diff --git a/infrastructure/traefik/traefik-service-lb.yaml b/infrastructure/traefik/traefik-service-lb.yaml new file mode 100644 index 0000000..e4929f1 --- /dev/null +++ b/infrastructure/traefik/traefik-service-lb.yaml @@ -0,0 +1,24 @@ +# infrastructure/traefik/traefik-service-lb.yaml +apiVersion: v1 +kind: Service +metadata: + name: traefik + namespace: kube-system + annotations: + metallb.universe.tf/address-pool: communication-pool +spec: + type: LoadBalancer + loadBalancerClass: metallb + loadBalancerIP: 192.168.22.9 + ports: + - name: web + port: 80 + targetPort: web + protocol: TCP + - name: websecure + port: 443 + targetPort: websecure + protocol: TCP + selector: + app.kubernetes.io/instance: traefik-kube-system + app.kubernetes.io/name: traefik diff --git a/services/communication/atlasbot-configmap.yaml b/services/communication/atlasbot-configmap.yaml new file mode 100644 index 0000000..92f77ba --- /dev/null +++ b/services/communication/atlasbot-configmap.yaml @@ -0,0 +1,131 @@ +# services/communication/atlasbot-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: atlasbot +data: + bot.py: | + import json, os, time, collections + from urllib import request, parse, error + + BASE = os.environ.get("MATRIX_BASE", "http://othrys-synapse-matrix-synapse:8008") + USER = os.environ["BOT_USER"] + PASSWORD = os.environ["BOT_PASS"] + ROOM_ALIAS = "#othrys:live.bstein.dev" + OLLAMA_URL = os.environ.get("OLLAMA_URL", "https://chat.ai.bstein.dev/") + MODEL = os.environ.get("OLLAMA_MODEL", "qwen2.5-coder:7b-instruct-q4_0") + API_KEY = os.environ.get("CHAT_API_KEY", "") + + def req(method: str, path: str, token: str | None = None, body=None, timeout=60): + url = BASE + path + data = None + headers = {} + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if token: + headers["Authorization"] = f"Bearer {token}" + r = request.Request(url, data=data, headers=headers, method=method) + with request.urlopen(r, timeout=timeout) as resp: + raw = resp.read() + return json.loads(raw.decode()) if raw else {} + + def login() -> str: + payload = { + "type": "m.login.password", + "identifier": {"type": "m.id.user", "user": USER}, + "password": PASSWORD, + } + res = req("POST", "/_matrix/client/v3/login", body=payload) + return res["access_token"] + + def resolve_alias(token: str, alias: str) -> str: + enc = parse.quote(alias) + res = req("GET", f"/_matrix/client/v3/directory/room/{enc}", token) + return res["room_id"] + + def join_room(token: str, room: str): + req("POST", f"/_matrix/client/v3/rooms/{parse.quote(room)}/join", token, body={}) + + def send_msg(token: str, room: str, text: str): + path = f"/_matrix/client/v3/rooms/{parse.quote(room)}/send/m.room.message" + req("POST", path, token, body={"msgtype": "m.text", "body": text}) + + history = collections.defaultdict(list) # room_id -> list of str (short transcript) + greeted = set() + + def ollama_reply(room_id: str, prompt: str) -> str: + try: + # Keep short context as plain text transcript + transcript = "\n".join(history[room_id][-12:] + [f"User: {prompt}"]) + payload = {"model": MODEL, "message": transcript} + headers = {"Content-Type": "application/json"} + if API_KEY: + headers["x-api-key"] = API_KEY + r = request.Request(OLLAMA_URL, data=json.dumps(payload).encode(), headers=headers) + with request.urlopen(r, timeout=15) as resp: + data = json.loads(resp.read().decode()) + reply = data.get("message") or data.get("response") or data.get("reply") or "I'm here to help." + history[room_id].append(f"Atlas: {reply}") + return reply + except Exception: + return "Hi! I'm Atlas." + + def sync_loop(token: str, room_id: str): + since = None + while True: + params = {"timeout": 30000} + if since: + params["since"] = since + query = parse.urlencode(params) + try: + res = req("GET", f"/_matrix/client/v3/sync?{query}", token, timeout=35) + except Exception: + time.sleep(5) + continue + since = res.get("next_batch", since) + + # invites + for rid, data in res.get("rooms", {}).get("invite", {}).items(): + try: + join_room(token, rid) + send_msg(token, rid, "Atlas online.") + except Exception: + pass + + # messages + for rid, data in res.get("rooms", {}).get("join", {}).items(): + if rid not in greeted and room_id and rid == room_id: + greeted.add(rid) + send_msg(token, rid, "Atlas online.") + timeline = data.get("timeline", {}).get("events", []) + for ev in timeline: + if ev.get("type") != "m.room.message": + continue + content = ev.get("content", {}) + body = content.get("body", "") + if not body.strip(): + continue + sender = ev.get("sender", "") + if sender == f"@{USER}:live.bstein.dev": + continue + # Only respond if bot is mentioned or in a DM + joined_count = data.get("summary", {}).get("m.joined_member_count") + is_dm = joined_count is not None and joined_count <= 2 + mentioned = f"@{USER}" in body or "atlas" in body.lower() + history[rid].append(f"{sender}: {body}") + if is_dm or mentioned: + reply = ollama_reply(rid, body) + send_msg(token, rid, reply) + + def main(): + token = login() + try: + room_id = resolve_alias(token, ROOM_ALIAS) + join_room(token, room_id) + except Exception: + room_id = None + sync_loop(token, room_id) + + if __name__ == "__main__": + main() diff --git a/services/communication/atlasbot-credentials.yaml b/services/communication/atlasbot-credentials.yaml new file mode 100644 index 0000000..6676d4e --- /dev/null +++ b/services/communication/atlasbot-credentials.yaml @@ -0,0 +1,9 @@ +# services/communication/atlasbot-credentials.yaml +apiVersion: v1 +kind: Secret +metadata: + name: atlasbot-credentials +type: Opaque +stringData: + bot-password: "x8eU9xwsjJ2S7Xv1G4mQ" + seeder-password: "Qv5sjyH8nD6pPz7Lk3R0" diff --git a/services/communication/atlasbot-deployment.yaml b/services/communication/atlasbot-deployment.yaml new file mode 100644 index 0000000..bd39ae7 --- /dev/null +++ b/services/communication/atlasbot-deployment.yaml @@ -0,0 +1,61 @@ +# services/communication/atlasbot-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: atlasbot + namespace: communication + labels: + app: atlasbot +spec: + replicas: 1 + selector: + matchLabels: + app: atlasbot + template: + metadata: + labels: + app: atlasbot + spec: + nodeSelector: + hardware: rpi5 + containers: + - name: atlasbot + image: python:3.11-slim + command: ["/bin/sh","-c"] + args: + - | + python /app/bot.py + env: + - name: MATRIX_BASE + value: http://othrys-synapse-matrix-synapse:8008 + - name: BOT_USER + value: atlasbot + - name: BOT_PASS + valueFrom: + secretKeyRef: + name: atlasbot-credentials + key: bot-password + - name: CHAT_API_KEY + valueFrom: + secretKeyRef: + name: chat-ai-keys + key: matrix + - name: OLLAMA_URL + value: https://chat.ai.bstein.dev/ + - name: OLLAMA_MODEL + value: qwen2.5-coder:7b-instruct-q4_0 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumeMounts: + - name: code + mountPath: /app/bot.py + subPath: bot.py + volumes: + - name: code + configMap: + name: atlasbot diff --git a/services/communication/chat-ai-keys.yaml b/services/communication/chat-ai-keys.yaml new file mode 100644 index 0000000..ac6c4e8 --- /dev/null +++ b/services/communication/chat-ai-keys.yaml @@ -0,0 +1,9 @@ +# services/communication/chat-ai-keys.yaml +apiVersion: v1 +kind: Secret +metadata: + name: chat-ai-keys + namespace: communication +type: Opaque +stringData: + matrix: "3d9b1e5e80f146f2b3f6a9fbe01b7b77" diff --git a/services/communication/coturn.yaml b/services/communication/coturn.yaml new file mode 100644 index 0000000..73a4d14 --- /dev/null +++ b/services/communication/coturn.yaml @@ -0,0 +1,323 @@ +# services/communication/coturn.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coturn + labels: + app: coturn +spec: + replicas: 1 + selector: + matchLabels: + app: coturn + template: + metadata: + labels: + app: coturn + spec: + nodeSelector: + hardware: rpi5 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] + containers: + - name: coturn + image: ghcr.io/coturn/coturn:4.6.2 + command: + - /bin/sh + - -c + - | + exec /usr/bin/turnserver \ + --no-cli \ + --fingerprint \ + --lt-cred-mech \ + --user=livekit:"${TURN_STATIC_AUTH_SECRET}" \ + --realm=live.bstein.dev \ + --listening-port=3478 \ + --tls-listening-port=5349 \ + --min-port=50000 \ + --max-port=50050 \ + --cert=/etc/coturn/tls/tls.crt \ + --pkey=/etc/coturn/tls/tls.key \ + --log-file=stdout \ + --no-software-attribute + env: + - name: TURN_STATIC_AUTH_SECRET + valueFrom: + secretKeyRef: + name: turn-shared-secret + key: TURN_STATIC_AUTH_SECRET + ports: + - name: turn-udp + containerPort: 3478 + protocol: UDP + - name: turn-tcp + containerPort: 3478 + protocol: TCP + - name: turn-tls + containerPort: 5349 + protocol: TCP + volumeMounts: + - name: tls + mountPath: /etc/coturn/tls + readOnly: true + resources: + requests: + cpu: 200m + memory: 256Mi + limits: + cpu: "2" + memory: 512Mi + volumes: + - name: tls + secret: + secretName: turn-live-tls +--- +apiVersion: v1 +kind: Service +metadata: + name: coturn + annotations: + metallb.universe.tf/address-pool: communication-pool +spec: + type: LoadBalancer + loadBalancerClass: metallb + loadBalancerIP: 192.168.22.5 + selector: + app: coturn + ports: + - name: turn-udp + port: 3478 + targetPort: 3478 + protocol: UDP + - name: turn-tcp + port: 3478 + targetPort: 3478 + protocol: TCP + - name: turn-tls + port: 5349 + targetPort: 5349 + protocol: TCP + # Expose relay range for UDP media + - name: relay-50000 + port: 50000 + targetPort: 50000 + protocol: UDP + - name: relay-50001 + port: 50001 + targetPort: 50001 + protocol: UDP + - name: relay-50002 + port: 50002 + targetPort: 50002 + protocol: UDP + - name: relay-50003 + port: 50003 + targetPort: 50003 + protocol: UDP + - name: relay-50004 + port: 50004 + targetPort: 50004 + protocol: UDP + - name: relay-50005 + port: 50005 + targetPort: 50005 + protocol: UDP + - name: relay-50006 + port: 50006 + targetPort: 50006 + protocol: UDP + - name: relay-50007 + port: 50007 + targetPort: 50007 + protocol: UDP + - name: relay-50008 + port: 50008 + targetPort: 50008 + protocol: UDP + - name: relay-50009 + port: 50009 + targetPort: 50009 + protocol: UDP + - name: relay-50010 + port: 50010 + targetPort: 50010 + protocol: UDP + - name: relay-50011 + port: 50011 + targetPort: 50011 + protocol: UDP + - name: relay-50012 + port: 50012 + targetPort: 50012 + protocol: UDP + - name: relay-50013 + port: 50013 + targetPort: 50013 + protocol: UDP + - name: relay-50014 + port: 50014 + targetPort: 50014 + protocol: UDP + - name: relay-50015 + port: 50015 + targetPort: 50015 + protocol: UDP + - name: relay-50016 + port: 50016 + targetPort: 50016 + protocol: UDP + - name: relay-50017 + port: 50017 + targetPort: 50017 + protocol: UDP + - name: relay-50018 + port: 50018 + targetPort: 50018 + protocol: UDP + - name: relay-50019 + port: 50019 + targetPort: 50019 + protocol: UDP + - name: relay-50020 + port: 50020 + targetPort: 50020 + protocol: UDP + - name: relay-50021 + port: 50021 + targetPort: 50021 + protocol: UDP + - name: relay-50022 + port: 50022 + targetPort: 50022 + protocol: UDP + - name: relay-50023 + port: 50023 + targetPort: 50023 + protocol: UDP + - name: relay-50024 + port: 50024 + targetPort: 50024 + protocol: UDP + - name: relay-50025 + port: 50025 + targetPort: 50025 + protocol: UDP + - name: relay-50026 + port: 50026 + targetPort: 50026 + protocol: UDP + - name: relay-50027 + port: 50027 + targetPort: 50027 + protocol: UDP + - name: relay-50028 + port: 50028 + targetPort: 50028 + protocol: UDP + - name: relay-50029 + port: 50029 + targetPort: 50029 + protocol: UDP + - name: relay-50030 + port: 50030 + targetPort: 50030 + protocol: UDP + - name: relay-50031 + port: 50031 + targetPort: 50031 + protocol: UDP + - name: relay-50032 + port: 50032 + targetPort: 50032 + protocol: UDP + - name: relay-50033 + port: 50033 + targetPort: 50033 + protocol: UDP + - name: relay-50034 + port: 50034 + targetPort: 50034 + protocol: UDP + - name: relay-50035 + port: 50035 + targetPort: 50035 + protocol: UDP + - name: relay-50036 + port: 50036 + targetPort: 50036 + protocol: UDP + - name: relay-50037 + port: 50037 + targetPort: 50037 + protocol: UDP + - name: relay-50038 + port: 50038 + targetPort: 50038 + protocol: UDP + - name: relay-50039 + port: 50039 + targetPort: 50039 + protocol: UDP + - name: relay-50040 + port: 50040 + targetPort: 50040 + protocol: UDP + - name: relay-50041 + port: 50041 + targetPort: 50041 + protocol: UDP + - name: relay-50042 + port: 50042 + targetPort: 50042 + protocol: UDP + - name: relay-50043 + port: 50043 + targetPort: 50043 + protocol: UDP + - name: relay-50044 + port: 50044 + targetPort: 50044 + protocol: UDP + - name: relay-50045 + port: 50045 + targetPort: 50045 + protocol: UDP + - name: relay-50046 + port: 50046 + targetPort: 50046 + protocol: UDP + - name: relay-50047 + port: 50047 + targetPort: 50047 + protocol: UDP + - name: relay-50048 + port: 50048 + targetPort: 50048 + protocol: UDP + - name: relay-50049 + port: 50049 + targetPort: 50049 + protocol: UDP + - name: relay-50050 + port: 50050 + targetPort: 50050 + protocol: UDP +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: turn-live-cert +spec: + secretName: turn-live-tls + issuerRef: + name: letsencrypt + kind: ClusterIssuer + dnsNames: + - turn.live.bstein.dev diff --git a/services/communication/element-call-config.yaml b/services/communication/element-call-config.yaml new file mode 100644 index 0000000..c86bbb6 --- /dev/null +++ b/services/communication/element-call-config.yaml @@ -0,0 +1,25 @@ +# services/communication/element-call-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: element-call-config + namespace: communication +data: + config.json: | + { + "default_server_config": { + "m.homeserver": { + "base_url": "https://matrix.live.bstein.dev", + "server_name": "live.bstein.dev" + }, + "m.identity_server": { + "base_url": "https://vector.im" + } + }, + "livekit": { + "livekit_service_url": "https://kit.live.bstein.dev/livekit/jwt" + }, + "branding": { + "app_name": "Othrys Call" + } + } diff --git a/services/communication/element-call-deployment.yaml b/services/communication/element-call-deployment.yaml new file mode 100644 index 0000000..f5752ac --- /dev/null +++ b/services/communication/element-call-deployment.yaml @@ -0,0 +1,78 @@ +# services/communication/element-call-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: element-call + namespace: communication + labels: + app: element-call +spec: + replicas: 1 + selector: + matchLabels: + app: element-call + template: + metadata: + labels: + app: element-call + spec: + nodeSelector: + hardware: rpi5 + containers: + - name: element-call + image: ghcr.io/element-hq/element-call:latest + ports: + - containerPort: 8080 + name: http + volumeMounts: + - name: config + mountPath: /app/config.json + subPath: config.json + volumes: + - name: config + configMap: + name: element-call-config + items: + - key: config.json + path: config.json + optional: false +--- +apiVersion: v1 +kind: Service +metadata: + name: element-call + namespace: communication +spec: + selector: + app: element-call + ports: + - name: http + port: 80 + targetPort: 8080 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: element-call + namespace: communication + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + cert-manager.io/cluster-issuer: letsencrypt +spec: + tls: + - hosts: + - call.live.bstein.dev + secretName: call-live-tls + rules: + - host: call.live.bstein.dev + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: element-call + port: + number: 80 diff --git a/services/communication/element-rendered.yaml b/services/communication/element-rendered.yaml new file mode 100644 index 0000000..c0b03c1 --- /dev/null +++ b/services/communication/element-rendered.yaml @@ -0,0 +1,223 @@ +--- +# Source: element-web/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: othrys-element-element-web + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm +--- +# Source: element-web/templates/configuration-nginx.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-element-element-web-nginx + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm +data: + default.conf: | + server { + listen 8080; + listen [::]:8080; + server_name localhost; + + root /usr/share/nginx/html; + index index.html; + + add_header X-Frame-Options SAMEORIGIN; + add_header X-Content-Type-Options nosniff; + add_header X-XSS-Protection "1; mode=block"; + add_header Content-Security-Policy "frame-ancestors 'self'"; + + # Set no-cache for the index.html only so that browsers always check for a new copy of Element Web. + location = /index.html { + add_header Cache-Control "no-cache"; + } + + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + } +--- +# Source: element-web/templates/configuration.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-element-element-web + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm +data: + config.json: | + {"brand":"Othrys","default_server_config":{"m.homeserver":{"base_url":"https://matrix.live.bstein.dev","server_name":"live.bstein.dev"},"m.identity_server":{"base_url":"https://vector.im"}},"default_theme":"dark","disable_custom_urls":true,"disable_login_language_selector":true,"disable_guests":false,"show_labs_settings":true,"features":{"feature_group_calls":true,"feature_video_rooms":true,"feature_element_call_video_rooms":true},"room_directory":{"servers":["live.bstein.dev"]},"jitsi":{},"element_call":{"url":"https://call.live.bstein.dev","participant_limit":16,"brand":"Othrys Call"}} +--- +# Source: element-web/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: othrys-element-element-web + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element +--- +# Source: element-web/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: othrys-element-element-web + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + template: + metadata: + annotations: + checksum/config: manual-rtc-enable-1 + checksum/config-nginx: 085061d0925f4840c3770233509dc0b00fe8fa1a5fef8bf282a514fd101c76fa + labels: + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + spec: + serviceAccountName: othrys-element-element-web + securityContext: + {} + containers: + - name: element-web + securityContext: + {} + image: "ghcr.io/element-hq/element-web:v1.12.6" + imagePullPolicy: IfNotPresent + env: + - name: ELEMENT_WEB_PORT + value: '8080' + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - mountPath: /app/config.json + name: config + subPath: config.json + - mountPath: /etc/nginx/conf.d/config.json + name: config-nginx + subPath: config.json + volumes: + - name: config + configMap: + name: othrys-element-element-web + - name: config-nginx + configMap: + name: othrys-element-element-web-nginx + nodeSelector: + hardware: rpi5 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - rpi4 + weight: 50 +--- +# Source: element-web/templates/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: othrys-element-element-web + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure +spec: + ingressClassName: traefik + tls: + - hosts: + - "live.bstein.dev" + secretName: live-othrys-tls + rules: + - host: "live.bstein.dev" + http: + paths: + - path: / + backend: + service: + name: othrys-element-element-web + port: + number: 80 + pathType: Prefix +--- +# Source: element-web/templates/tests/test-connection.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "othrys-element-element-web-test-connection" + labels: + helm.sh/chart: element-web-1.4.26 + app.kubernetes.io/name: element-web + app.kubernetes.io/instance: othrys-element + app.kubernetes.io/version: "1.12.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['othrys-element-element-web:80'] + restartPolicy: Never diff --git a/services/communication/guest-name-job.yaml b/services/communication/guest-name-job.yaml new file mode 100644 index 0000000..fcd44e7 --- /dev/null +++ b/services/communication/guest-name-job.yaml @@ -0,0 +1,89 @@ +# services/communication/guest-name-job.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: guest-name-randomizer + namespace: communication +spec: + schedule: "*/1 * * * *" + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: rename + image: python:3.11-slim + env: + - name: SYNAPSE_BASE + value: http://othrys-synapse-matrix-synapse:8008 + - name: SEEDER_USER + value: othrys-seeder + - name: SEEDER_PASS + valueFrom: + secretKeyRef: + name: atlasbot-credentials + key: seeder-password + command: + - /bin/sh + - -c + - | + set -euo pipefail + pip install --no-cache-dir requests >/dev/null + python - <<'PY' + import os, random, requests, urllib.parse + + ADJ = ["brisk","calm","eager","gentle","merry","nifty","rapid","sunny","witty","zesty"] + NOUN = ["otter","falcon","comet","ember","grove","harbor","meadow","raven","river","summit"] + + BASE = os.environ["SYNAPSE_BASE"] + OTHRYS = "!orejZnVfvbAmwQDYba:live.bstein.dev" + + def login(user, password): + r = requests.post(f"{BASE}/_matrix/client/v3/login", json={ + "type": "m.login.password", + "identifier": {"type": "m.id.user", "user": user}, + "password": password, + }) + r.raise_for_status() + return r.json()["access_token"] + + def list_guests(token): + headers = {"Authorization": f"Bearer {token}"} + users = [] + from_token = None + while True: + url = f"{BASE}/_synapse/admin/v2/users?local=true&deactivated=false&limit=100" + if from_token: + url += f"&from={from_token}" + res = requests.get(url, headers=headers) + res.raise_for_status() + data = res.json() + for u in data.get("users", []): + disp = u.get("displayname", "") + if u.get("is_guest") and (not disp or disp.isdigit()): + users.append(u["name"]) + from_token = data.get("next_token") + if not from_token: + break + return users + + def set_displayname(token, user_id, name): + headers = {"Authorization": f"Bearer {token}"} + payload = {"displayname": name} + # Update global profile + r = requests.put(f"{BASE}/_matrix/client/v3/profile/{urllib.parse.quote(user_id)}/displayname", headers=headers, json=payload) + r.raise_for_status() + # Update Othrys member event so clients see the change quickly + state_url = f"{BASE}/_matrix/client/v3/rooms/{urllib.parse.quote(OTHRYS)}/state/m.room.member/{urllib.parse.quote(user_id)}" + r2 = requests.get(state_url, headers=headers) + content = r2.json() if r2.status_code == 200 else {"membership": "join"} + content["displayname"] = name + requests.put(state_url, headers=headers, json=content) + + token = login(os.environ["SEEDER_USER"], os.environ["SEEDER_PASS"]) + guests = list_guests(token) + for g in guests: + new = f"{random.choice(ADJ)}-{random.choice(NOUN)}" + set_displayname(token, g, new) + PY diff --git a/services/communication/kustomization.yaml b/services/communication/kustomization.yaml new file mode 100644 index 0000000..9cd7f38 --- /dev/null +++ b/services/communication/kustomization.yaml @@ -0,0 +1,24 @@ +# services/communication/kustomization.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: communication +resources: + - namespace.yaml + - synapse-rendered.yaml + - element-rendered.yaml + - livekit-config.yaml + - livekit.yaml + - coturn.yaml + - livekit-token-deployment.yaml + - livekit-ingress.yaml + - livekit-middlewares.yaml + - element-call-config.yaml + - element-call-deployment.yaml + - pin-othrys-job.yaml + - guest-name-job.yaml + - chat-ai-keys.yaml + - atlasbot-credentials.yaml + - atlasbot-configmap.yaml + - atlasbot-deployment.yaml + - seed-othrys-room.yaml + - wellknown.yaml diff --git a/services/communication/livekit-config.yaml b/services/communication/livekit-config.yaml new file mode 100644 index 0000000..e9c13bb --- /dev/null +++ b/services/communication/livekit-config.yaml @@ -0,0 +1,21 @@ +# services/communication/livekit-config.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: livekit-config +data: + livekit.yaml: | + port: 7880 + rtc: + udp_port: 7882 + tcp_port: 7881 + use_external_ip: true + turn_servers: + - host: turn.live.bstein.dev + port: 5349 + protocol: tls + - host: turn.live.bstein.dev + port: 3478 + protocol: udp + room: + auto_create: true diff --git a/services/communication/livekit-ingress.yaml b/services/communication/livekit-ingress.yaml new file mode 100644 index 0000000..796eb3d --- /dev/null +++ b/services/communication/livekit-ingress.yaml @@ -0,0 +1,28 @@ +# services/communication/livekit-ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: livekit-ingress + namespace: communication + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: communication-livekit-sfu-strip@kubernetescrd + cert-manager.io/cluster-issuer: letsencrypt +spec: + tls: + - hosts: + - kit.live.bstein.dev + secretName: kit-live-tls + rules: + - host: kit.live.bstein.dev + http: + paths: + - path: /livekit/sfu + pathType: Prefix + backend: + service: + name: livekit + port: + number: 7880 diff --git a/services/communication/livekit-middlewares.yaml b/services/communication/livekit-middlewares.yaml new file mode 100644 index 0000000..49a3e8f --- /dev/null +++ b/services/communication/livekit-middlewares.yaml @@ -0,0 +1,48 @@ +# services/communication/livekit-middlewares.yaml +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: livekit-sfu-strip + namespace: communication +spec: + stripPrefix: + prefixes: + - /livekit/sfu +--- +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: livekit-jwt-strip + namespace: communication +spec: + stripPrefix: + prefixes: + - /livekit/jwt +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: livekit-jwt-ingress + namespace: communication + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + traefik.ingress.kubernetes.io/router.middlewares: communication-livekit-jwt-strip@kubernetescrd + cert-manager.io/cluster-issuer: letsencrypt +spec: + tls: + - hosts: + - kit.live.bstein.dev + secretName: kit-live-tls + rules: + - host: kit.live.bstein.dev + http: + paths: + - path: /livekit/jwt + pathType: Prefix + backend: + service: + name: livekit-token-service + port: + number: 8080 diff --git a/services/communication/livekit-token-deployment.yaml b/services/communication/livekit-token-deployment.yaml new file mode 100644 index 0000000..f9d1a87 --- /dev/null +++ b/services/communication/livekit-token-deployment.yaml @@ -0,0 +1,69 @@ +# services/communication/livekit-token-deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: livekit-token-service + labels: + app: livekit-token-service +spec: + replicas: 1 + selector: + matchLabels: + app: livekit-token-service + template: + metadata: + labels: + app: livekit-token-service + spec: + nodeSelector: + hardware: rpi5 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] + hostAliases: + - ip: 10.43.60.6 + hostnames: + - live.bstein.dev + containers: + - name: token-service + image: ghcr.io/element-hq/lk-jwt-service:0.3.0 + env: + - name: LIVEKIT_URL + value: wss://kit.live.bstein.dev/livekit/sfu + - name: LIVEKIT_KEY + value: primary + - name: LIVEKIT_SECRET + valueFrom: + secretKeyRef: + name: livekit-api + key: primary + - name: LIVEKIT_FULL_ACCESS_HOMESERVERS + value: live.bstein.dev + ports: + - containerPort: 8080 + name: http + resources: + requests: + cpu: 50m + memory: 128Mi + limits: + cpu: 300m + memory: 256Mi +--- +apiVersion: v1 +kind: Service +metadata: + name: livekit-token-service +spec: + selector: + app: livekit-token-service + ports: + - name: http + port: 8080 + targetPort: 8080 diff --git a/services/communication/livekit.yaml b/services/communication/livekit.yaml new file mode 100644 index 0000000..1f3d7e9 --- /dev/null +++ b/services/communication/livekit.yaml @@ -0,0 +1,120 @@ +# services/communication/livekit.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: livekit + labels: + app: livekit +spec: + replicas: 1 + selector: + matchLabels: + app: livekit + template: + metadata: + labels: + app: livekit + spec: + enableServiceLinks: false + nodeSelector: + hardware: rpi5 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] + containers: + - name: livekit + image: livekit/livekit-server:v1.9.0 + command: + - /bin/sh + - -c + - | + set -euo pipefail + umask 077 + printf "%s: %s\n" "${LIVEKIT_API_KEY_ID}" "${LIVEKIT_API_SECRET}" > /var/run/livekit/keys + chmod 600 /var/run/livekit/keys + exec /livekit-server --config /etc/livekit/livekit.yaml --key-file /var/run/livekit/keys + env: + - name: LIVEKIT_API_KEY_ID + value: primary + - name: LIVEKIT_API_SECRET + valueFrom: + secretKeyRef: + name: livekit-api + key: primary + - name: LIVEKIT_RTC__TURN_SERVERS_0__USERNAME + value: livekit + - name: LIVEKIT_RTC__TURN_SERVERS_0__CREDENTIAL + valueFrom: + secretKeyRef: + name: turn-shared-secret + key: TURN_STATIC_AUTH_SECRET + - name: LIVEKIT_RTC__TURN_SERVERS_1__USERNAME + value: livekit + - name: LIVEKIT_RTC__TURN_SERVERS_1__CREDENTIAL + valueFrom: + secretKeyRef: + name: turn-shared-secret + key: TURN_STATIC_AUTH_SECRET + ports: + - containerPort: 7880 + name: http + protocol: TCP + - containerPort: 7881 + name: tcp-media + protocol: TCP + - containerPort: 7882 + name: udp-media + protocol: UDP + volumeMounts: + - name: config + mountPath: /etc/livekit + - name: runtime-keys + mountPath: /var/run/livekit + resources: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: "2" + memory: 1Gi + volumes: + - name: config + configMap: + name: livekit-config + items: + - key: livekit.yaml + path: livekit.yaml + - name: runtime-keys + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: livekit + annotations: + metallb.universe.tf/address-pool: communication-pool +spec: + type: LoadBalancer + loadBalancerClass: metallb + loadBalancerIP: 192.168.22.6 + selector: + app: livekit + ports: + - name: http + port: 7880 + targetPort: 7880 + protocol: TCP + - name: tcp-media + port: 7881 + targetPort: 7881 + protocol: TCP + - name: udp-media + port: 7882 + targetPort: 7882 + protocol: UDP diff --git a/services/communication/namespace.yaml b/services/communication/namespace.yaml new file mode 100644 index 0000000..d566429 --- /dev/null +++ b/services/communication/namespace.yaml @@ -0,0 +1,5 @@ +# services/communication/namespace.yaml +apiVersion: v1 +kind: Namespace +metadata: + name: communication diff --git a/services/communication/pin-othrys-job.yaml b/services/communication/pin-othrys-job.yaml new file mode 100644 index 0000000..a3178e2 --- /dev/null +++ b/services/communication/pin-othrys-job.yaml @@ -0,0 +1,68 @@ +# services/communication/pin-othrys-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: pin-othrys-invite + namespace: communication +spec: + ttlSecondsAfterFinished: 3600 + template: + spec: + restartPolicy: OnFailure + containers: + - name: pin + image: python:3.11-slim + env: + - name: SYNAPSE_BASE + value: http://othrys-synapse-matrix-synapse:8008 + - name: SEEDER_USER + value: othrys-seeder + - name: SEEDER_PASS + valueFrom: + secretKeyRef: + name: atlasbot-credentials + key: seeder-password + command: + - /bin/sh + - -c + - | + set -euo pipefail + pip install --no-cache-dir requests >/dev/null + python - <<'PY' + import requests, urllib.parse, os + + BASE = os.environ["SYNAPSE_BASE"] + def login(user, password): + r = requests.post(f"{BASE}/_matrix/client/v3/login", json={ + "type": "m.login.password", + "identifier": {"type": "m.id.user", "user": user}, + "password": password, + }) + r.raise_for_status() + return r.json()["access_token"] + + def resolve(alias, token): + enc = urllib.parse.quote(alias) + r = requests.get(f"{BASE}/_matrix/client/v3/directory/room/{enc}", headers={"Authorization": f"Bearer {token}"}) + r.raise_for_status() + return r.json()["room_id"] + + def send(room_id, token, body): + r = requests.post(f"{BASE}/_matrix/client/v3/rooms/{urllib.parse.quote(room_id)}/send/m.room.message", + headers={"Authorization": f"Bearer {token}"}, + json={"msgtype": "m.text", "body": body}) + r.raise_for_status() + return r.json()["event_id"] + + def pin(room_id, token, event_id): + r = requests.put(f"{BASE}/_matrix/client/v3/rooms/{urllib.parse.quote(room_id)}/state/m.room.pinned_events", + headers={"Authorization": f"Bearer {token}"}, + json={"pinned": [event_id]}) + r.raise_for_status() + + token = login(os.environ["SEEDER_USER"], os.environ["SEEDER_PASS"]) + room_id = resolve("#othrys:live.bstein.dev", token) + msg = "Invite guests: share https://live.bstein.dev/#/room/#othrys:live.bstein.dev?action=join and choose 'Continue' -> 'Join as guest'." + eid = send(room_id, token, msg) + pin(room_id, token, eid) + PY diff --git a/services/communication/seed-othrys-room.yaml b/services/communication/seed-othrys-room.yaml new file mode 100644 index 0000000..a3d4a1d --- /dev/null +++ b/services/communication/seed-othrys-room.yaml @@ -0,0 +1,135 @@ +# services/communication/seed-othrys-room.yaml +apiVersion: batch/v1 +kind: CronJob +metadata: + name: seed-othrys-room + namespace: communication +spec: + schedule: "*/10 * * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: seed + image: python:3.11-slim + env: + - name: SYNAPSE_BASE + value: http://othrys-synapse-matrix-synapse:8008 + - name: REG_SECRET + valueFrom: + secretKeyRef: + name: othrys-synapse-matrix-synapse + key: config.yaml + - name: SEEDER_USER + value: othrys-seeder + - name: SEEDER_PASS + valueFrom: + secretKeyRef: + name: atlasbot-credentials + key: seeder-password + - name: BOT_USER + value: atlasbot + - name: BOT_PASS + valueFrom: + secretKeyRef: + name: atlasbot-credentials + key: bot-password + command: + - /bin/sh + - -c + - | + set -euo pipefail + pip install --no-cache-dir requests pyyaml matrix-synapse >/dev/null + python - <<'PY' + import os, subprocess, requests, yaml + + BASE = os.environ["SYNAPSE_BASE"] + CONFIG = "/config/config.yaml" + + def register(user, password, admin=False): + args = ["register_new_matrix_user", "-c", CONFIG, "-u", user, "-p", password] + if admin: + args.append("-a") + args.append(BASE) + res = subprocess.run(args, capture_output=True, text=True) + if res.returncode not in (0, 1): # 1 = already exists + raise SystemExit(f"register {user} failed: {res.returncode} {res.stderr}") + + def login(user, password): + r = requests.post(f"{BASE}/_matrix/client/v3/login", json={ + "type": "m.login.password", + "identifier": {"type": "m.id.user", "user": user}, + "password": password, + }) + if r.status_code != 200: + raise SystemExit(f"login failed: {r.status_code} {r.text}") + return r.json()["access_token"] + + def ensure_room(token): + headers = {"Authorization": f"Bearer {token}"} + alias = "#othrys:live.bstein.dev" + alias_enc = "%23othrys%3Alive.bstein.dev" + exists = requests.get(f"{BASE}/_matrix/client/v3/directory/room/{alias_enc}", headers=headers) + if exists.status_code == 200: + room_id = exists.json()["room_id"] + else: + create = requests.post(f"{BASE}/_matrix/client/v3/createRoom", headers=headers, json={ + "preset": "public_chat", + "name": "Othrys", + "room_alias_name": "othrys", + "initial_state": [], + "power_level_content_override": {"events_default": 0, "users_default": 0, "state_default": 50}, + }) + if create.status_code not in (200, 409): + raise SystemExit(f"create room failed: {create.status_code} {create.text}") + exists = requests.get(f"{BASE}/_matrix/client/v3/directory/room/{alias_enc}", headers=headers) + room_id = exists.json()["room_id"] + state_events = [ + ("m.room.join_rules", {"join_rule": "public"}), + ("m.room.guest_access", {"guest_access": "can_join"}), + ("m.room.history_visibility", {"history_visibility": "shared"}), + ("m.room.canonical_alias", {"alias": alias}), + ] + for ev_type, content in state_events: + requests.put(f"{BASE}/_matrix/client/v3/rooms/{room_id}/state/{ev_type}", headers=headers, json=content) + requests.put(f"{BASE}/_matrix/client/v3/directory/list/room/{room_id}", headers=headers, json={"visibility": "public"}) + return room_id + + def join_user(token, room_id, user_id): + headers = {"Authorization": f"Bearer {token}"} + requests.post(f"{BASE}/_synapse/admin/v1/join/{room_id}", headers=headers, json={"user_id": user_id}) + + def join_all_locals(token, room_id): + headers = {"Authorization": f"Bearer {token}"} + users = [] + from_token = None + while True: + url = f"{BASE}/_synapse/admin/v2/users?local=true&deactivated=false&limit=100" + if from_token: + url += f"&from={from_token}" + res = requests.get(url, headers=headers).json() + users.extend([u["name"] for u in res.get("users", [])]) + from_token = res.get("next_token") + if not from_token: + break + for uid in users: + join_user(token, room_id, uid) + + register(os.environ["SEEDER_USER"], os.environ["SEEDER_PASS"], admin=True) + register(os.environ["BOT_USER"], os.environ["BOT_PASS"], admin=False) + token = login(os.environ["SEEDER_USER"], os.environ["SEEDER_PASS"]) + room_id = ensure_room(token) + join_user(token, room_id, f"@{os.environ['BOT_USER']}:live.bstein.dev") + join_all_locals(token, room_id) + PY + volumeMounts: + - name: synapse-config + mountPath: /config + readOnly: true + volumes: + - name: synapse-config + secret: + secretName: othrys-synapse-matrix-synapse diff --git a/services/communication/synapse-rendered.yaml b/services/communication/synapse-rendered.yaml new file mode 100644 index 0000000..db24fd4 --- /dev/null +++ b/services/communication/synapse-rendered.yaml @@ -0,0 +1,1155 @@ +--- +# Source: matrix-synapse/charts/redis/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + name: othrys-synapse-redis + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 +--- +# Source: matrix-synapse/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: othrys-synapse-matrix-synapse + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm +stringData: + config.yaml: | + ## Registration ## + + registration_shared_secret: "PlxXRFAiRfLDp8RbAS6aHN7b" + + ## API Configuration ## + + ## Database configuration ## + + database: + name: "psycopg2" + args: + user: "synapse" + password: "@@POSTGRES_PASSWORD@@" + database: "synapse" + host: "postgres-service.postgres.svc.cluster.local" + port: 5432 + sslmode: "prefer" + cp_min: 5 + cp_max: 10 + + + ## Redis configuration ## + + redis: + enabled: true + host: "othrys-synapse-redis-master" + port: 6379 + password: "@@REDIS_PASSWORD@@" +--- +# Source: matrix-synapse/charts/redis/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-synapse-redis-configuration + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 +data: + redis.conf: |- + # User-supplied common configuration: + # Enable AOF https://redis.io/topics/persistence#append-only-file + appendonly yes + # Disable RDB persistence, AOF persistence already enabled. + save "" + # End of common configuration + master.conf: |- + dir /data + # User-supplied master configuration: + rename-command FLUSHDB "" + rename-command FLUSHALL "" + # End of master configuration + replica.conf: |- + dir /data + # User-supplied replica configuration: + rename-command FLUSHDB "" + rename-command FLUSHALL "" + # End of replica configuration +--- +# Source: matrix-synapse/charts/redis/templates/health-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-synapse-redis-health + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 +data: + ping_readiness_local.sh: |- + #!/bin/bash + + [[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")" + [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD" + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h localhost \ + -p $REDIS_PORT \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + ping_liveness_local.sh: |- + #!/bin/bash + + [[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")" + [[ -n "$REDIS_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_PASSWORD" + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h localhost \ + -p $REDIS_PORT \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + responseFirstWord=$(echo $response | head -n1 | awk '{print $1;}') + if [ "$response" != "PONG" ] && [ "$responseFirstWord" != "LOADING" ] && [ "$responseFirstWord" != "MASTERDOWN" ]; then + echo "$response" + exit 1 + fi + ping_readiness_master.sh: |- + #!/bin/bash + + [[ -f $REDIS_MASTER_PASSWORD_FILE ]] && export REDIS_MASTER_PASSWORD="$(< "${REDIS_MASTER_PASSWORD_FILE}")" + [[ -n "$REDIS_MASTER_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_MASTER_PASSWORD" + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h $REDIS_MASTER_HOST \ + -p $REDIS_MASTER_PORT_NUMBER \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + ping_liveness_master.sh: |- + #!/bin/bash + + [[ -f $REDIS_MASTER_PASSWORD_FILE ]] && export REDIS_MASTER_PASSWORD="$(< "${REDIS_MASTER_PASSWORD_FILE}")" + [[ -n "$REDIS_MASTER_PASSWORD" ]] && export REDISCLI_AUTH="$REDIS_MASTER_PASSWORD" + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h $REDIS_MASTER_HOST \ + -p $REDIS_MASTER_PORT_NUMBER \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + responseFirstWord=$(echo $response | head -n1 | awk '{print $1;}') + if [ "$response" != "PONG" ] && [ "$responseFirstWord" != "LOADING" ]; then + echo "$response" + exit 1 + fi + ping_readiness_local_and_master.sh: |- + script_dir="$(dirname "$0")" + exit_status=0 + "$script_dir/ping_readiness_local.sh" $1 || exit_status=$? + "$script_dir/ping_readiness_master.sh" $1 || exit_status=$? + exit $exit_status + ping_liveness_local_and_master.sh: |- + script_dir="$(dirname "$0")" + exit_status=0 + "$script_dir/ping_liveness_local.sh" $1 || exit_status=$? + "$script_dir/ping_liveness_master.sh" $1 || exit_status=$? + exit $exit_status +--- +# Source: matrix-synapse/charts/redis/templates/scripts-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-synapse-redis-scripts + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 +data: + start-master.sh: | + #!/bin/bash + + [[ -f $REDIS_PASSWORD_FILE ]] && export REDIS_PASSWORD="$(< "${REDIS_PASSWORD_FILE}")" + if [[ -f /opt/bitnami/redis/mounted-etc/master.conf ]];then + cp /opt/bitnami/redis/mounted-etc/master.conf /opt/bitnami/redis/etc/master.conf + fi + if [[ -f /opt/bitnami/redis/mounted-etc/redis.conf ]];then + cp /opt/bitnami/redis/mounted-etc/redis.conf /opt/bitnami/redis/etc/redis.conf + fi + ARGS=("--port" "${REDIS_PORT}") + ARGS+=("--requirepass" "${REDIS_PASSWORD}") + ARGS+=("--masterauth" "${REDIS_PASSWORD}") + ARGS+=("--include" "/opt/bitnami/redis/etc/redis.conf") + ARGS+=("--include" "/opt/bitnami/redis/etc/master.conf") + exec redis-server "${ARGS[@]}" +--- +# Source: matrix-synapse/templates/configuration.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-synapse-matrix-synapse + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm +data: + log.yaml: | + version: 1 + formatters: + precise: + format: '%(asctime)s - %(name)s - %(lineno)d - %(levelname)s - %(request)s- %(message)s' + filters: + context: + (): synapse.util.logcontext.LoggingContextFilter + request: "" + handlers: + console: + class: logging.StreamHandler + formatter: precise + filters: [context] + level: INFO + loggers: + synapse: + level: INFO + root: + level: INFO + handlers: [console] + homeserver.yaml: | + # NOTE: + # Secrets are stored in separate configs to better fit K8s concepts + + ## Server ## + + server_name: "live.bstein.dev" + public_baseurl: "https://matrix.live.bstein.dev" + pid_file: /homeserver.pid + web_client: False + soft_file_limit: 0 + log_config: "/synapse/config/log.yaml" + report_stats: false + + instance_map: + main: + host: othrys-synapse-replication + port: 9093 + + ## Ports ## + + listeners: + - port: 8008 + tls: false + bind_addresses: ["::"] + type: http + x_forwarded: true + + resources: + - names: + - client + - federation + compress: false + + - port: 9090 + tls: false + bind_addresses: ["::"] + type: http + + resources: + - names: [metrics] + compress: false + + - port: 9093 + tls: false + bind_addresses: ["::"] + type: http + + resources: + - names: [replication] + compress: false + + ## Files ## + + media_store_path: "/synapse/data/media" + uploads_path: "/synapse/data/uploads" + + ## Registration ## + + enable_registration: false + + ## Metrics ### + + enable_metrics: true + + ## Signing Keys ## + + signing_key_path: "/synapse/keys/signing.key" + + # The trusted servers to download signing keys from. + trusted_key_servers: + - server_name: matrix.org + + ## Workers ## + + ## Extra config ## + + allow_guest_access: true + allow_public_rooms_without_auth: true + auto_join_rooms: + - "#othrys:live.bstein.dev" + autocreate_auto_join_rooms: true + default_room_version: "11" + experimental_features: + msc3266_enabled: true + msc4143_enabled: true + msc4222_enabled: true + max_event_delay_duration: 24h + password_config: + enabled: true + turn_uris: + - "turn:turn.live.bstein.dev:3478?transport=udp" + - "turn:turn.live.bstein.dev:3478?transport=tcp" + - "turns:turn.live.bstein.dev:5349?transport=tcp" + turn_shared_secret: "@@TURN_SECRET@@" + turn_allow_guests: true + turn_user_lifetime: 86400000 + rc_login: + address: + burst_count: 20 + per_second: 5 + account: + burst_count: 20 + per_second: 5 + failed_attempts: + burst_count: 20 + per_second: 5 + rc_message: + per_second: 0.5 + burst_count: 30 + rc_delayed_event_mgmt: + per_second: 1 + burst_count: 20 + room_list_publication_rules: + - action: allow + well_known_client: + "m.homeserver": + "base_url": "https://matrix.live.bstein.dev" + "org.matrix.msc4143.rtc_foci": + - type: "livekit" + livekit_service_url: "https://kit.live.bstein.dev/livekit/jwt" + oidc_enabled: true + oidc_providers: + - allow_existing_users: true + authorization_endpoint: https://sso.bstein.dev/realms/atlas/protocol/openid-connect/auth + client_auth_method: client_secret_post + client_id: synapse + client_secret: "@@OIDC_CLIENT_SECRET@@" + idp_id: keycloak + idp_name: Keycloak + issuer: https://sso.bstein.dev/realms/atlas + scopes: + - openid + - profile + - email + token_endpoint: https://sso.bstein.dev/realms/atlas/protocol/openid-connect/token + user_mapping_provider: + config: + display_name_template: '{{ user.name }}' + localpart_template: '{{ user.preferred_username }}' + userinfo_endpoint: https://sso.bstein.dev/realms/atlas/protocol/openid-connect/userinfo +--- +# Source: matrix-synapse/templates/pvc.yaml +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: othrys-synapse-matrix-synapse + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "50Gi" + storageClassName: "asteria" +--- +# Source: matrix-synapse/charts/redis/templates/headless-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: othrys-synapse-redis-headless + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 + annotations: + +spec: + type: ClusterIP + clusterIP: None + ports: + - name: tcp-redis + port: 6379 + targetPort: redis + selector: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/name: redis +--- +# Source: matrix-synapse/charts/redis/templates/master/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: othrys-synapse-redis-master + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 + app.kubernetes.io/component: master +spec: + type: ClusterIP + internalTrafficPolicy: Cluster + sessionAffinity: None + ports: + - name: tcp-redis + port: 6379 + targetPort: redis + nodePort: null + selector: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/name: redis + app.kubernetes.io/component: master +--- +# Source: matrix-synapse/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: othrys-synapse-matrix-synapse + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 8008 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/component: synapse + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse +--- +# Source: matrix-synapse/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: othrys-synapse-replication + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm +spec: + type: ClusterIP + ports: + - port: 9093 + targetPort: replication + protocol: TCP + name: replication + selector: + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/component: synapse +--- +# Source: matrix-synapse/charts/redis/templates/master/application.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: othrys-synapse-redis-master + namespace: "communication" + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 + app.kubernetes.io/component: master +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/name: redis + app.kubernetes.io/component: master + strategy: + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: redis + helm.sh/chart: redis-17.17.1 + app.kubernetes.io/component: master + annotations: + checksum/configmap: 86bcc953bb473748a3d3dc60b7c11f34e60c93519234d4c37f42e22ada559d47 + checksum/health: aff24913d801436ea469d8d374b2ddb3ec4c43ee7ab24663d5f8ff1a1b6991a9 + checksum/scripts: 560c33ff34d845009b51830c332aa05fa211444d1877d3526d3599be7543aaa5 + checksum/secret: 44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a + spec: + + securityContext: + fsGroup: 1001 + serviceAccountName: othrys-synapse-redis + automountServiceAccountToken: true + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/name: redis + app.kubernetes.io/component: master + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + enableServiceLinks: true + terminationGracePeriodSeconds: 30 + containers: + - name: redis + image: docker.io/bitnamilegacy/redis:7.0.12-debian-11-r34 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsGroup: 0 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + command: + - /bin/bash + args: + - -c + - /opt/bitnami/scripts/start-scripts/start-master.sh + env: + - name: BITNAMI_DEBUG + value: "false" + - name: REDIS_REPLICATION_MODE + value: master + - name: ALLOW_EMPTY_PASSWORD + value: "no" + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: synapse-redis + key: redis-password + - name: REDIS_TLS_ENABLED + value: "no" + - name: REDIS_PORT + value: "6379" + ports: + - name: redis + containerPort: 6379 + livenessProbe: + initialDelaySeconds: 20 + periodSeconds: 5 + # One second longer than command timeout should prevent generation of zombie processes. + timeoutSeconds: 6 + successThreshold: 1 + failureThreshold: 5 + exec: + command: + - sh + - -c + - /health/ping_liveness_local.sh 5 + readinessProbe: + initialDelaySeconds: 20 + periodSeconds: 5 + timeoutSeconds: 2 + successThreshold: 1 + failureThreshold: 5 + exec: + command: + - sh + - -c + - /health/ping_readiness_local.sh 1 + resources: + limits: {} + requests: {} + volumeMounts: + - name: start-scripts + mountPath: /opt/bitnami/scripts/start-scripts + - name: health + mountPath: /health + - name: redis-data + mountPath: /data + - name: config + mountPath: /opt/bitnami/redis/mounted-etc + - name: redis-tmp-conf + mountPath: /opt/bitnami/redis/etc/ + - name: tmp + mountPath: /tmp + volumes: + - name: start-scripts + configMap: + name: othrys-synapse-redis-scripts + defaultMode: 0755 + - name: health + configMap: + name: othrys-synapse-redis-health + defaultMode: 0755 + - name: config + configMap: + name: othrys-synapse-redis-configuration + - name: redis-tmp-conf + emptyDir: {} + - name: tmp + emptyDir: {} + - name: redis-data + emptyDir: {} +--- +# Source: matrix-synapse/templates/deployment.yaml +# Server: live.bstein.dev +apiVersion: apps/v1 +kind: Deployment +metadata: + name: othrys-synapse-matrix-synapse + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: synapse +spec: + replicas: 1 + strategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/component: synapse + template: + metadata: + annotations: + checksum/config: manual-rtc-enable-1 + checksum/secrets: ec9f3b254a562a0f0709461eb74a8cc91b8c1a2fb06be2594a131776c2541773 + labels: + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/component: synapse + spec: + serviceAccountName: default + + securityContext: + fsGroup: 666 + runAsGroup: 666 + runAsUser: 666 + containers: + - name: synapse + command: + - sh + - -c + - | + export POSTGRES_PASSWORD=$(echo "${POSTGRES_PASSWORD:-}" | sed 's/\//\\\//g' | sed 's/\&/\\\&/g') && \ + export REDIS_PASSWORD=$(echo "${REDIS_PASSWORD:-}" | sed 's/\//\\\//g' | sed 's/\&/\\\&/g') && \ + export OIDC_CLIENT_SECRET_ESCAPED=$(echo "${OIDC_CLIENT_SECRET:-}" | sed 's/[\\/&]/\\&/g') && \ + export TURN_SECRET_ESCAPED=$(echo "${TURN_SECRET:-}" | sed 's/[\\/&]/\\&/g') && \ + cat /synapse/secrets/*.yaml | \ + sed -e "s/@@POSTGRES_PASSWORD@@/${POSTGRES_PASSWORD:-}/" \ + -e "s/@@REDIS_PASSWORD@@/${REDIS_PASSWORD:-}/" \ + > /synapse/config/conf.d/secrets.yaml + + cp /synapse/config/homeserver.yaml /synapse/runtime-config/homeserver.yaml && \ + if [ -n "${OIDC_CLIENT_SECRET_ESCAPED}" ]; then \ + sed -i "s/@@OIDC_CLIENT_SECRET@@/${OIDC_CLIENT_SECRET_ESCAPED}/g" /synapse/runtime-config/homeserver.yaml; \ + fi; \ + if [ -n "${TURN_SECRET_ESCAPED}" ]; then \ + sed -i "s/@@TURN_SECRET@@/${TURN_SECRET_ESCAPED}/g" /synapse/runtime-config/homeserver.yaml; \ + fi + exec python -B -m synapse.app.homeserver \ + -c /synapse/runtime-config/homeserver.yaml \ + -c /synapse/config/conf.d/ + env: + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: synapse-db + key: POSTGRES_PASSWORD + - name: REDIS_PASSWORD + valueFrom: + secretKeyRef: + name: synapse-redis + key: redis-password + - name: OIDC_CLIENT_SECRET + valueFrom: + secretKeyRef: + name: synapse-oidc + key: client-secret + - name: TURN_SECRET + valueFrom: + secretKeyRef: + name: turn-shared-secret + key: TURN_STATIC_AUTH_SECRET + image: "ghcr.io/element-hq/synapse:v1.144.0" + imagePullPolicy: IfNotPresent + securityContext: + {} + ports: + - name: http + containerPort: 8008 + protocol: TCP + - name: replication + containerPort: 9093 + protocol: TCP + - name: metrics + containerPort: 9090 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: http + readinessProbe: + httpGet: + path: /health + port: http + startupProbe: + failureThreshold: 12 + httpGet: + path: /health + port: http + volumeMounts: + - name: config + mountPath: /synapse/config + - name: runtime-config + mountPath: /synapse/runtime-config + - name: tmpconf + mountPath: /synapse/config/conf.d + - name: secrets + mountPath: /synapse/secrets + - name: signingkey + mountPath: /synapse/keys + - name: media + mountPath: /synapse/data + - name: tmpdir + mountPath: /tmp + resources: + limits: + cpu: "2" + memory: 3Gi + requests: + cpu: 500m + memory: 1Gi + volumes: + - name: config + configMap: + name: othrys-synapse-matrix-synapse + - name: secrets + secret: + secretName: othrys-synapse-matrix-synapse + - name: signingkey + secret: + secretName: "othrys-synapse-signingkey" + items: + - key: "signing.key" + path: signing.key + - name: tmpconf + emptyDir: {} + - name: tmpdir + emptyDir: {} + - name: runtime-config + emptyDir: {} + - name: media + persistentVolumeClaim: + claimName: othrys-synapse-matrix-synapse + nodeSelector: + hardware: rpi5 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: hardware + operator: In + values: + - rpi5 + - rpi4 + weight: 50 +--- +# Source: matrix-synapse/templates/ingress.yaml +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: othrys-synapse-matrix-synapse + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure +spec: + ingressClassName: traefik + tls: + - hosts: + - "matrix.live.bstein.dev" + - "live.bstein.dev" + secretName: matrix-live-tls + rules: + - host: "live.bstein.dev" + http: + paths: + - path: /_matrix + backend: + service: + name: othrys-synapse-matrix-synapse + port: + number: 8008 + pathType: Prefix + - path: /.well-known/matrix + backend: + service: + name: othrys-synapse-matrix-synapse + port: + number: 8008 + pathType: Prefix + - host: "matrix.live.bstein.dev" + http: + paths: + - path: /_matrix + backend: + service: + name: othrys-synapse-matrix-synapse + port: + number: 8008 + pathType: Prefix + - path: /_synapse + backend: + service: + name: othrys-synapse-matrix-synapse + port: + number: 8008 + pathType: Prefix + - host: "bstein.dev" + http: + paths: + - path: /.well-known/matrix + backend: + service: + name: othrys-synapse-matrix-synapse + port: + number: 8008 + pathType: Prefix +--- +# Source: matrix-synapse/templates/signing-key-job.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: othrys-synapse-signingkey-job + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: signingkey-job + annotations: + helm.sh/hook: pre-install + helm.sh/hook-delete-policy: hook-succeeded +--- +# Source: matrix-synapse/templates/signing-key-job.yaml +# Create secret if signing key job is enabled, or if we're running in ArgoCD and we don't have an existing secret +apiVersion: v1 +kind: Secret +metadata: + annotations: + helm.sh/hook: pre-install + helm.sh/hook-delete-policy: never + helm.sh/resource-policy: keep + # If for some reason we didn't detect ArgoCD, but are running in it, we want to make sure we don't delete the secret + argocd.argoproj.io/hook: Skip + name: othrys-synapse-signingkey + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: signingkey-job +--- +# Source: matrix-synapse/templates/signing-key-job.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: othrys-synapse-matrix-synapse-scripts + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + annotations: + helm.sh/hook: pre-install + helm.sh/hook-delete-policy: hook-succeeded +data: + signing-key.sh: | + #!/bin/sh + + set -eu + + check_key() { + set +e + + echo "Checking for existing signing key..." + key="$(kubectl get secret "$SECRET_NAME" -o jsonpath="{.data['signing\.key']}" 2> /dev/null)" + [ $? -ne 0 ] && return 1 + [ -z "$key" ] && return 2 + return 0 + } + + create_key() { + echo "Waiting for new signing key to be generated..." + begin=$(date +%s) + end=$((begin + 300)) # 5 minutes + while true; do + [ -f /synapse/keys/signing.key ] && return 0 + [ "$(date +%s)" -gt $end ] && return 1 + sleep 5 + done + } + + store_key() { + echo "Storing signing key in Kubernetes secret..." + kubectl patch secret "$SECRET_NAME" -p "{\"data\":{\"signing.key\":\"$(base64 /synapse/keys/signing.key | tr -d '\n')\"}}" + } + + if check_key; then + echo "Key already in place, exiting." + exit + fi + + if ! create_key; then + echo "Timed out waiting for a signing key to appear." + exit 1 + fi + + store_key +--- +# Source: matrix-synapse/templates/signing-key-job.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: othrys-synapse-signingkey-job + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: signingkey-job + annotations: + helm.sh/hook: pre-install + helm.sh/hook-delete-policy: hook-succeeded +rules: + - apiGroups: + - "" + resources: + - secrets + resourceNames: + - othrys-synapse-signingkey + verbs: + - get + - update + - patch +--- +# Source: matrix-synapse/templates/signing-key-job.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: othrys-synapse-signingkey-job + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: signingkey-job + annotations: + helm.sh/hook: pre-install + helm.sh/hook-delete-policy: hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: othrys-synapse-signingkey-job +subjects: + - kind: ServiceAccount + name: othrys-synapse-signingkey-job + namespace: communication +--- +# Source: matrix-synapse/templates/tests/test-connection.yaml +apiVersion: v1 +kind: Pod +metadata: + name: "othrys-synapse-matrix-synapse-test-connection" + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": test-success +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['othrys-synapse-matrix-synapse:8008/_matrix/client/versions'] + restartPolicy: Never +--- +# Source: matrix-synapse/templates/signing-key-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: othrys-synapse-signingkey-job + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: signingkey-job + annotations: + helm.sh/hook: pre-install + helm.sh/hook-delete-policy: hook-succeeded +spec: + ttlSecondsAfterFinished: 0 + template: + metadata: + labels: + helm.sh/chart: matrix-synapse-3.12.17 + app.kubernetes.io/name: matrix-synapse + app.kubernetes.io/instance: othrys-synapse + app.kubernetes.io/version: "1.144.0" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: signingkey-job + spec: + containers: + - command: + - sh + - -c + - | + echo "Generating signing key..." + if which generate_signing_key.py >/dev/null; then + generate_signing_key.py -o /synapse/keys/signing.key + else + generate_signing_key -o /synapse/keys/signing.key + fi + image: "matrixdotorg/synapse:latest" + imagePullPolicy: IfNotPresent + name: signing-key-generate + resources: + {} + securityContext: + {} + volumeMounts: + - mountPath: /synapse/keys + name: matrix-synapse-keys + - command: + - sh + - -c + - | + printf "Checking rights to update secret... " + kubectl auth can-i update secret/${SECRET_NAME} + /scripts/signing-key.sh + env: + - name: SECRET_NAME + value: othrys-synapse-signingkey + image: "bitnami/kubectl:latest" + imagePullPolicy: IfNotPresent + name: signing-key-upload + resources: + {} + securityContext: + {} + volumeMounts: + - mountPath: /scripts + name: scripts + readOnly: true + - mountPath: /synapse/keys + name: matrix-synapse-keys + readOnly: true + securityContext: + {} + restartPolicy: Never + serviceAccount: othrys-synapse-signingkey-job + volumes: + - name: scripts + configMap: + name: othrys-synapse-matrix-synapse-scripts + defaultMode: 0755 + - name: matrix-synapse-keys + emptyDir: {} + parallelism: 1 + completions: 1 + backoffLimit: 1 diff --git a/services/communication/values-element.yaml b/services/communication/values-element.yaml new file mode 100644 index 0000000..9ab91de --- /dev/null +++ b/services/communication/values-element.yaml @@ -0,0 +1,59 @@ +# services/communication/values-element.yaml +replicaCount: 1 + +defaultServer: + url: https://matrix.live.bstein.dev + name: live.bstein.dev + +config: + default_theme: dark + brand: Othrys + disable_custom_urls: true + disable_login_language_selector: true + disable_guests: false + show_labs_settings: true + features: + feature_group_calls: true + feature_video_rooms: true + feature_element_call_video_rooms: true + room_directory: + servers: + - live.bstein.dev + jitsi: {} + element_call: + url: https://call.live.bstein.dev + participant_limit: 16 + brand: Othrys Call + +ingress: + enabled: true + className: traefik + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure + hosts: + - live.bstein.dev + tls: + - secretName: live-othrys-tls + hosts: [live.bstein.dev] + +resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + +nodeSelector: + hardware: rpi5 + +affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] diff --git a/services/communication/values-synapse.yaml b/services/communication/values-synapse.yaml new file mode 100644 index 0000000..7df16b6 --- /dev/null +++ b/services/communication/values-synapse.yaml @@ -0,0 +1,132 @@ +# services/communication/values-synapse.yaml +serverName: live.bstein.dev +publicServerName: matrix.live.bstein.dev + +config: + publicBaseurl: https://matrix.live.bstein.dev + +externalPostgresql: + host: postgres-service.postgres.svc.cluster.local + port: 5432 + username: synapse + existingSecret: synapse-db + existingSecretPasswordKey: POSTGRES_PASSWORD + database: synapse + +redis: + enabled: true + auth: + enabled: true + existingSecret: synapse-redis + existingSecretPasswordKey: redis-password + +postgresql: + enabled: false + +persistence: + enabled: true + storageClass: asteria + accessMode: ReadWriteOnce + size: 50Gi + +synapse: + podSecurityContext: + fsGroup: 666 + runAsUser: 666 + runAsGroup: 666 + resources: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: "2" + memory: 3Gi + nodeSelector: + hardware: rpi5 + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 50 + preference: + matchExpressions: + - key: hardware + operator: In + values: ["rpi5","rpi4"] + +ingress: + enabled: true + className: traefik + annotations: + cert-manager.io/cluster-issuer: letsencrypt + traefik.ingress.kubernetes.io/router.entrypoints: websecure + csHosts: + - matrix.live.bstein.dev + hosts: + - matrix.live.bstein.dev + wkHosts: + - live.bstein.dev + - bstein.dev + tls: + - secretName: matrix-live-tls + hosts: + - matrix.live.bstein.dev + - live.bstein.dev + +extraConfig: + allow_guest_access: true + allow_public_rooms_without_auth: true + auto_join_rooms: + - "#othrys:live.bstein.dev" + autocreate_auto_join_rooms: true + default_room_version: "11" + experimental_features: + msc3266_enabled: true + msc4143_enabled: true + msc4222_enabled: true + max_event_delay_duration: 24h + password_config: + enabled: true + oidc_enabled: true + oidc_providers: + - idp_id: keycloak + idp_name: Keycloak + issuer: https://sso.bstein.dev/realms/atlas + client_id: synapse + client_secret: "@@OIDC_CLIENT_SECRET@@" + client_auth_method: client_secret_post + scopes: ["openid", "profile", "email"] + authorization_endpoint: https://sso.bstein.dev/realms/atlas/protocol/openid-connect/auth + token_endpoint: https://sso.bstein.dev/realms/atlas/protocol/openid-connect/token + userinfo_endpoint: https://sso.bstein.dev/realms/atlas/protocol/openid-connect/userinfo + user_mapping_provider: + config: + localpart_template: "{{ user.preferred_username }}" + display_name_template: "{{ user.name }}" + allow_existing_users: true + rc_message: + per_second: 0.5 + burst_count: 30 + rc_delayed_event_mgmt: + per_second: 1 + burst_count: 20 + rc_login: + address: + burst_count: 20 + per_second: 5 + account: + burst_count: 20 + per_second: 5 + failed_attempts: + burst_count: 20 + per_second: 5 + room_list_publication_rules: + - action: allow + well_known_client: + "m.homeserver": + "base_url": "https://matrix.live.bstein.dev" + "org.matrix.msc4143.rtc_foci": + - type: "livekit" + livekit_service_url: "https://kit.live.bstein.dev/livekit/jwt" + +worker: + enabled: false diff --git a/services/communication/wellknown.yaml b/services/communication/wellknown.yaml new file mode 100644 index 0000000..655746a --- /dev/null +++ b/services/communication/wellknown.yaml @@ -0,0 +1,109 @@ +# services/communication/wellknown.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: matrix-wellknown + namespace: communication +data: + client.json: | + { + "m.homeserver": { + "base_url": "https://matrix.live.bstein.dev" + }, + "org.matrix.msc4143.rtc_foci": [ + { + "type": "livekit", + "livekit_service_url": "https://kit.live.bstein.dev/livekit/jwt" + } + ] + } + server.json: | + { + "m.server": "live.bstein.dev:443" + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: matrix-wellknown + namespace: communication + labels: + app: matrix-wellknown +spec: + replicas: 1 + selector: + matchLabels: + app: matrix-wellknown + template: + metadata: + labels: + app: matrix-wellknown + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: wellknown + mountPath: /usr/share/nginx/html/.well-known/matrix/client + subPath: client.json + - name: wellknown + mountPath: /usr/share/nginx/html/.well-known/matrix/server + subPath: server.json + volumes: + - name: wellknown + configMap: + name: matrix-wellknown + items: + - key: client.json + path: client.json + - key: server.json + path: server.json +--- +apiVersion: v1 +kind: Service +metadata: + name: matrix-wellknown + namespace: communication +spec: + selector: + app: matrix-wellknown + ports: + - name: http + port: 80 + targetPort: 80 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: matrix-wellknown + namespace: communication + annotations: + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + traefik.ingress.kubernetes.io/router.tls: "true" + cert-manager.io/cluster-issuer: letsencrypt +spec: + tls: + - hosts: + - live.bstein.dev + secretName: live-othrys-tls + rules: + - host: live.bstein.dev + http: + paths: + - path: /.well-known/matrix/client + pathType: Prefix + backend: + service: + name: matrix-wellknown + port: + number: 80 + - path: /.well-known/matrix/server + pathType: Prefix + backend: + service: + name: matrix-wellknown + port: + number: 80 diff --git a/services/mailu/front-lb.yaml b/services/mailu/front-lb.yaml new file mode 100644 index 0000000..ada16b0 --- /dev/null +++ b/services/mailu/front-lb.yaml @@ -0,0 +1,42 @@ +# services/mailu/front-lb.yaml +apiVersion: v1 +kind: Service +metadata: + name: mailu-front-lb + namespace: mailu-mailserver + annotations: + metallb.universe.tf/address-pool: communication-pool +spec: + type: LoadBalancer + loadBalancerClass: metallb + loadBalancerIP: 192.168.22.4 + externalTrafficPolicy: Cluster + selector: + app.kubernetes.io/component: front + app.kubernetes.io/instance: mailu + app.kubernetes.io/name: mailu + ports: + - name: smtp + port: 25 + targetPort: 25 + protocol: TCP + - name: smtps + port: 465 + targetPort: 465 + protocol: TCP + - name: submission + port: 587 + targetPort: 587 + protocol: TCP + - name: imaps + port: 993 + targetPort: 993 + protocol: TCP + - name: pop3s + port: 995 + targetPort: 995 + protocol: TCP + - name: sieve + port: 4190 + targetPort: 4190 + protocol: TCP diff --git a/services/mailu/kustomization.yaml b/services/mailu/kustomization.yaml index 2df7440..a23e0b1 100644 --- a/services/mailu/kustomization.yaml +++ b/services/mailu/kustomization.yaml @@ -13,6 +13,7 @@ resources: - mailu-sync-job.yaml - mailu-sync-cronjob.yaml - mailu-sync-listener.yaml + - front-lb.yaml configMapGenerator: - name: mailu-sync-script