From 294879e17f7203d9e6cd7241c4e4a2e5725775b4 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Sun, 24 May 2026 21:23:34 +0100 Subject: [PATCH 1/5] Move configuration files for Loki components to subdirectories --- .../namespaces/loki/{ => alloy}/alloy_values.yml | 0 .../namespaces/loki/{ => loki}/alert_rules.yml | 0 .../namespaces/loki/{ => loki}/loki_values.yml | 0 .../namespaces/loki/{ => loki}/recording_rules.yml | 0 kubernetes/namespaces/loki/{ => loki}/secrets.yaml | Bin 5 files changed, 0 insertions(+), 0 deletions(-) rename kubernetes/namespaces/loki/{ => alloy}/alloy_values.yml (100%) rename kubernetes/namespaces/loki/{ => loki}/alert_rules.yml (100%) rename kubernetes/namespaces/loki/{ => loki}/loki_values.yml (100%) rename kubernetes/namespaces/loki/{ => loki}/recording_rules.yml (100%) rename kubernetes/namespaces/loki/{ => loki}/secrets.yaml (100%) diff --git a/kubernetes/namespaces/loki/alloy_values.yml b/kubernetes/namespaces/loki/alloy/alloy_values.yml similarity index 100% rename from kubernetes/namespaces/loki/alloy_values.yml rename to kubernetes/namespaces/loki/alloy/alloy_values.yml diff --git a/kubernetes/namespaces/loki/alert_rules.yml b/kubernetes/namespaces/loki/loki/alert_rules.yml similarity index 100% rename from kubernetes/namespaces/loki/alert_rules.yml rename to kubernetes/namespaces/loki/loki/alert_rules.yml diff --git a/kubernetes/namespaces/loki/loki_values.yml b/kubernetes/namespaces/loki/loki/loki_values.yml similarity index 100% rename from kubernetes/namespaces/loki/loki_values.yml rename to kubernetes/namespaces/loki/loki/loki_values.yml diff --git a/kubernetes/namespaces/loki/recording_rules.yml b/kubernetes/namespaces/loki/loki/recording_rules.yml similarity index 100% rename from kubernetes/namespaces/loki/recording_rules.yml rename to kubernetes/namespaces/loki/loki/recording_rules.yml diff --git a/kubernetes/namespaces/loki/secrets.yaml b/kubernetes/namespaces/loki/loki/secrets.yaml similarity index 100% rename from kubernetes/namespaces/loki/secrets.yaml rename to kubernetes/namespaces/loki/loki/secrets.yaml From 4e07f026b9a6c9c270e4a357a61443b4d3243569 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Sun, 24 May 2026 21:23:57 +0100 Subject: [PATCH 2/5] Switch from deploying Kubewatch manually to Helm deployment --- .../namespaces/monitoring/kubewatch/README.md | 6 ++++ .../monitoring/kubewatch/configmap.yaml | 34 ------------------ .../monitoring/kubewatch/deployment.yaml | 32 ----------------- .../kubewatch/kubewatch_values.yaml | 24 +++++++++++++ .../monitoring/kubewatch/secrets.yaml | Bin 316 -> 0 bytes .../monitoring/kubewatch/service-account.yaml | 30 ---------------- 6 files changed, 30 insertions(+), 96 deletions(-) delete mode 100644 kubernetes/namespaces/monitoring/kubewatch/configmap.yaml delete mode 100644 kubernetes/namespaces/monitoring/kubewatch/deployment.yaml create mode 100644 kubernetes/namespaces/monitoring/kubewatch/kubewatch_values.yaml delete mode 100644 kubernetes/namespaces/monitoring/kubewatch/secrets.yaml delete mode 100644 kubernetes/namespaces/monitoring/kubewatch/service-account.yaml diff --git a/kubernetes/namespaces/monitoring/kubewatch/README.md b/kubernetes/namespaces/monitoring/kubewatch/README.md index 294c6664..4d2a4ac2 100644 --- a/kubernetes/namespaces/monitoring/kubewatch/README.md +++ b/kubernetes/namespaces/monitoring/kubewatch/README.md @@ -1,3 +1,9 @@ # Kubewatch > **kubewatch** is a Kubernetes watcher that currently publishes notification to available collaboration hubs/notification channels. Run it in your k8s cluster, and you will get event notifications through webhooks. + +``` +$ helm repo add robusta https://robusta-charts.storage.googleapis.com +$ helm repo update +$ helm upgrade --install -n monitoring --values kubewatch_values.yaml kubewatch robusta/kubewatch +``` diff --git a/kubernetes/namespaces/monitoring/kubewatch/configmap.yaml b/kubernetes/namespaces/monitoring/kubewatch/configmap.yaml deleted file mode 100644 index 82ec3908..00000000 --- a/kubernetes/namespaces/monitoring/kubewatch/configmap.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: kubewatch-config - namespace: monitoring -data: - .kubewatch.yaml: | - namespace: "" - handler: - discord: - webhook: "" - ignores: - - pixels-discord-channel - - cert-manager-cainjector-leader-election - - cert-manager-controller - - ingress-controller-leader-nginx - - cluster-autoscaler-status - - ingress-controller-leader - resource: - replicationcontroller: true - clusterrole: true - serviceaccount: true - persistentvolume: true - configmap: true - namespace: true - secret: true - deployment: false - replicaset: false - daemonset: false - services: false - pod: false - job: false - node: false - ingress: false diff --git a/kubernetes/namespaces/monitoring/kubewatch/deployment.yaml b/kubernetes/namespaces/monitoring/kubewatch/deployment.yaml deleted file mode 100644 index a6746481..00000000 --- a/kubernetes/namespaces/monitoring/kubewatch/deployment.yaml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: kubewatch - namespace: monitoring -spec: - selector: - matchLabels: - app: kubewatch - template: - metadata: - labels: - app: kubewatch - spec: - serviceAccountName: kubewatch - containers: - - image: ghcr.io/python-discord/kubewatch:latest - imagePullPolicy: Always - name: kubewatch - volumeMounts: - - name: config-volume - mountPath: /root - envFrom: - - secretRef: - name: kubewatch-secrets - securityContext: - readOnlyRootFilesystem: true - restartPolicy: Always - volumes: - - name: config-volume - configMap: - name: kubewatch-config diff --git a/kubernetes/namespaces/monitoring/kubewatch/kubewatch_values.yaml b/kubernetes/namespaces/monitoring/kubewatch/kubewatch_values.yaml new file mode 100644 index 00000000..f608b409 --- /dev/null +++ b/kubernetes/namespaces/monitoring/kubewatch/kubewatch_values.yaml @@ -0,0 +1,24 @@ +rbac: + create: true + customRoles: + - apiGroups: ["events.k8s.io"] + resources: ["events"] + verbs: ["get", "list", "watch"] + +slack: + enabled: false + +webhook: + enabled: true + url: "http://vector.loki:6000" + +resourcesToWatch: + deployment: true + replicationcontroller: false + replicaset: false + daemonset: false + services: true + pod: true + job: true + persistentvolume: true + event: true diff --git a/kubernetes/namespaces/monitoring/kubewatch/secrets.yaml b/kubernetes/namespaces/monitoring/kubewatch/secrets.yaml deleted file mode 100644 index 7427da2a0130061a5a0239a3c57f8f933cb9f04a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 316 zcmV-C0mJ?PM@dveQdv+`0NXcjb?W2Dq};Ha{vdzP-I;dV;^H5cB+{9T$=WJ$K zu3Q=kHMw%B(X=`cGl_tfcAb5Al!Yr_cJkkcU zZsy3Dj?2D-qO=t49moh+hB?RjG-q_)+NU%fo)8*``kl3pfeH$75#VaaX1&?CmFj*k}Sq*T_4VA3~Tj!Pg)E4 zc*Y)cl2zRlj+xi^-dNO;jpOzszQwKVgtOx!m6kp?rydDL$zM;jxzkGVe3oWU)3r5- O#?$-f-l}(-fUP!Oqn@z< diff --git a/kubernetes/namespaces/monitoring/kubewatch/service-account.yaml b/kubernetes/namespaces/monitoring/kubewatch/service-account.yaml deleted file mode 100644 index f0748ba4..00000000 --- a/kubernetes/namespaces/monitoring/kubewatch/service-account.yaml +++ /dev/null @@ -1,30 +0,0 @@ ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: kubewatch - namespace: monitoring -rules: -- apiGroups: ["", "extensions", "apps", "batch", "rbac.authorization.k8s.io", ] - resources: ["*"] - verbs: ["get", "watch", "list"] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: kubewatch - namespace: monitoring ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubewatch - namespace: monitoring -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubewatch -subjects: - - kind: ServiceAccount - name: kubewatch - namespace: monitoring From b586e5a1053419e9cda2b8a80a5203e58d39fc7a Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Sun, 24 May 2026 21:24:07 +0100 Subject: [PATCH 3/5] Add Vector configuration for log shipping --- kubernetes/namespaces/loki/README.md | 22 +++++- .../namespaces/loki/vector/vector_values.yml | 67 +++++++++++++++++++ 2 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 kubernetes/namespaces/loki/vector/vector_values.yml diff --git a/kubernetes/namespaces/loki/README.md b/kubernetes/namespaces/loki/README.md index 26c158de..214244ee 100644 --- a/kubernetes/namespaces/loki/README.md +++ b/kubernetes/namespaces/loki/README.md @@ -11,7 +11,7 @@ Once this is deployed, the service `loki-gateway.loki.svc.cluster.local` will po ``` $ helm repo add grafana https://grafana.github.io/helm-charts $ helm repo update -$ helm upgrade --install -n loki --values loki_values.yml loki grafana/loki +$ helm upgrade --install -n loki --values loki/loki_values.yml loki grafana/loki ``` ## Alloy @@ -21,5 +21,23 @@ $ helm upgrade --install -n loki --values loki_values.yml loki grafana/loki This requires no additional configuration, just make sure Loki is up and Alloy will start shipping logs. ``` -$ helm upgrade --install -n loki --values alloy_values.yml alloy grafana/alloy +$ helm upgrade --install -n loki --values alloy/alloy_values.yml alloy grafana/alloy ``` + +## Vector + +Vector is a log shipping tool that we use to accept various sources and ships them to Loki. + +``` +$ helm repo add vector https://helm.vector.dev +$ helm repo update +$ helm upgrade --install -n loki --values vector/vector_values.yml vector vector/vector +``` + +### Vector Port Allocations + +Log data will be tagged and retained based on the Vector endpoint it lands in. + +| Port | Sending Service | +| ---- | --------------- | +| 6000 | Kubewatch | diff --git a/kubernetes/namespaces/loki/vector/vector_values.yml b/kubernetes/namespaces/loki/vector/vector_values.yml new file mode 100644 index 00000000..736d97ba --- /dev/null +++ b/kubernetes/namespaces/loki/vector/vector_values.yml @@ -0,0 +1,67 @@ +# role -- [Role](https://vector.dev/docs/setup/deployment/roles/) for this Vector instance, valid options are: +# "Agent", "Aggregator", and "Stateless-Aggregator". + +# Each role is created with the following workloads: +# Agent = DaemonSet +# Aggregator = StatefulSet +# Stateless-Aggregator = Deployment +role: "Stateless-Aggregator" + +# customConfig -- Override Vector's default configs, if used **all** options need to be specified. This section supports +# using helm templates to populate dynamic values. See Vector's [configuration documentation](https://vector.dev/docs/reference/configuration/) +# for all options. +customConfig: + data_dir: /vector-data-dir + + api: + enabled: true + address: 127.0.0.1:8686 + + sources: + kubewatch: + address: 0.0.0.0:6000 + type: http_server + auth: + strategy: custom + source: |- + true + decoding: + codec: json + framing: + method: bytes + + transforms: + kubewatch_prepared: + type: remap + inputs: [kubewatch] + source: |- + .job = "kubewatch" + .kind = .eventmeta.kind + .name = .eventmeta.name + .namespace = .eventmeta.namespace + .reason = .eventmeta.reason + + .labels = { + "namespace": .eventmeta.namespace, + "kind": .eventmeta.kind, + } + + del(.source_type) + del(.path) + del(.eventmeta) + + sinks: + loki: + type: loki + inputs: [kubewatch_prepared] + endpoint: http://loki:3100 + labels: + job: |- + {{`vector.{{ job }}`}} + "*": |- + {{`{{ labels }}`}} + encoding: + codec: json + +# Log level for Vector. +logLevel: "info" From 0e17902e2bb73a24508ea4c612fab14a7bd08109 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Mon, 25 May 2026 18:56:37 +0100 Subject: [PATCH 4/5] Use underscore for job name instead of period --- kubernetes/namespaces/loki/vector/vector_values.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/namespaces/loki/vector/vector_values.yml b/kubernetes/namespaces/loki/vector/vector_values.yml index 736d97ba..e822ffb9 100644 --- a/kubernetes/namespaces/loki/vector/vector_values.yml +++ b/kubernetes/namespaces/loki/vector/vector_values.yml @@ -57,7 +57,7 @@ customConfig: endpoint: http://loki:3100 labels: job: |- - {{`vector.{{ job }}`}} + {{`vector_{{ job }}`}} "*": |- {{`{{ labels }}`}} encoding: From 19f4bde7732568410d6273dbdc808872f8af0572 Mon Sep 17 00:00:00 2001 From: Joe Banks Date: Mon, 25 May 2026 18:58:16 +0100 Subject: [PATCH 5/5] Update delete_request_cancel_period to 10s --- kubernetes/namespaces/loki/loki/loki_values.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/namespaces/loki/loki/loki_values.yml b/kubernetes/namespaces/loki/loki/loki_values.yml index f94dbe72..172b04fb 100644 --- a/kubernetes/namespaces/loki/loki/loki_values.yml +++ b/kubernetes/namespaces/loki/loki/loki_values.yml @@ -46,7 +46,7 @@ loki: retention_enabled: true delete_request_store: s3 compaction_interval: 10m - delete_request_cancel_period: 1h + delete_request_cancel_period: 10s retention_delete_worker_count: 150 retention_delete_delay: 10s