Skip to content

Commit 847fc12

Browse files
hoalongnatsuhoalongnatsu
authored
feat(kubernetes)add connector setup and live Pod logs
feat(kubernetes)add connector setup and live Pod logs
2 parents 17494d8 + 89a64b9 commit 847fc12

81 files changed

Lines changed: 6890 additions & 1140 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cmd/main.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -209,12 +209,13 @@ func main() {
209209
// flush interval, so racing it away drops everything learned since the last
210210
// one.
211211
var agentDone <-chan struct{}
212-
toolAvailability := agent.NewToolAvailabilityService(cfg.Agent, store)
213-
kubernetesService, kubernetesErr := agent.NewKubernetesService(cfg.Agent.Tools.Kubernetes, tenancy.DefaultOrgScope())
212+
toolAvailability := agent.NewToolAvailabilityService(cfg.Agent, store, cfg.Connectors)
213+
kubernetesService, kubernetesErr := agent.NewKubernetesService(cfg.Connectors.Kubernetes, tenancy.DefaultOrgScope())
214214
if kubernetesErr != nil {
215215
log.Printf("kubernetes: connector unavailable: %v", kubernetesErr)
216216
}
217217
if kubernetesService != nil {
218+
kubernetesService.SetLogStreamContext(rootCtx)
218219
kubernetesService.SetChangeStorage(store)
219220
}
220221
toolAvailability.BindIntegrationConstruction("kubernetes", kubernetesService != nil)
@@ -236,7 +237,7 @@ func main() {
236237
}
237238
}
238239

239-
cat, done, err := startAgent(rootCtx, app, cfg.Agent, cfg.GatewaySecret, store, rdb, toolAvailability, kubernetesService, healthManager)
240+
cat, done, err := startAgent(rootCtx, app, cfg.Agent, cfg.Connectors, cfg.GatewaySecret, store, rdb, toolAvailability, kubernetesService, healthManager)
240241
if err != nil {
241242
log.Fatalf("agent: failed to start: %v", err)
242243
}
@@ -301,7 +302,7 @@ func registerToolAvailabilityController(app *fiber.App, availability *agent.Tool
301302
// admin routes on the fiber app. It returns the catalog so the caller can
302303
// hold a reference (and so future hot-reload code has a handle to it), plus a
303304
// channel that closes when the worker has finished its shutdown flush.
304-
func startAgent(ctx context.Context, app *fiber.App, cfg c.AgentConfig, gatewaySecret string, store storage.Provider, rdb redis.UniversalClient, toolAvailability *agent.ToolAvailabilityService, kubernetesService *kubernetes.Service, healthManager *servicehealth.Manager) (*agent.Catalog, <-chan struct{}, error) {
305+
func startAgent(ctx context.Context, app *fiber.App, cfg c.AgentConfig, connectors c.ConnectorsConfig, gatewaySecret string, store storage.Provider, rdb redis.UniversalClient, toolAvailability *agent.ToolAvailabilityService, kubernetesService *kubernetes.Service, healthManager *servicehealth.Manager) (*agent.Catalog, <-chan struct{}, error) {
305306
// On the Postgres backend, install the typed signal-table
306307
// catalog store so the log catalog reads/writes the explicit
307308
// vs_patterns/vs_logs/vs_services tables (searchable, indexed) instead of
@@ -402,7 +403,7 @@ func startAgent(ctx context.Context, app *fiber.App, cfg c.AgentConfig, gatewayS
402403
log.Printf("agent: tailing dedup sets persisted through Redis for %d source(s)", n)
403404
}
404405

405-
aiBundle := agent.BuildAIsWithKubernetes(cfg, catalog, store, nil, kubernetesService)
406+
aiBundle := agent.BuildAIsWithKubernetes(cfg, catalog, store, nil, kubernetesService, connectors)
406407
toolAvailability.BindLiveSnapshot(aiBundle.ToolSnapshot)
407408
if aiBundle.Detect != nil {
408409
log.Printf("agent: AI SRE enabled provider=%s model=%s rate_limit=%d/hr",

‎cmd/tool_availability_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -23,16 +23,16 @@ func TestToolCatalogGETDoesNotConstructDisabledAgentDependencies(t *testing.T) {
2323
Enable: false,
2424
AI: config.AgentAIConfig{Enable: aiEnabled, Provider: "invalid-provider", Model: "invalid-model"},
2525
Sources: []config.AgentSourceConfig{{Name: "broken", Type: "invalid-source", Enable: true}},
26-
Tools: config.ToolsConfig{Kubernetes: config.KubernetesToolConfig{
27-
Endpoint: "https://cluster.example",
28-
Auth: config.KubernetesAuthConfig{Mode: "invalid"},
29-
}},
3026
}
27+
connectors := config.ConnectorsConfig{Kubernetes: config.KubernetesToolConfig{
28+
Endpoint: "https://cluster.example",
29+
Auth: config.KubernetesAuthConfig{Mode: "invalid"},
30+
}}
3131
app := fiber.New()
3232
app.Use(func(ctx *fiber.Ctx) error { middleware.MarkAuthorized(ctx); return ctx.Next() })
3333
app.Use(middleware.OrgInjector())
34-
availability := agent.NewToolAvailabilityService(cfg, storage.NewMemory())
35-
service, constructionErr := agent.NewKubernetesService(cfg.Tools.Kubernetes, tenancy.DefaultOrgScope())
34+
availability := agent.NewToolAvailabilityService(cfg, storage.NewMemory(), connectors)
35+
service, constructionErr := agent.NewKubernetesService(connectors.Kubernetes, tenancy.DefaultOrgScope())
3636
if constructionErr == nil {
3737
t.Fatal("invalid Kubernetes authentication constructed")
3838
}

‎config/connectors.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
connectors:
2+
kubernetes:
3+
auth:
4+
mode: ""
5+
actions:
6+
enable: false
7+
auth:
8+
mode: ""

‎config/tools.yaml‎

Lines changed: 0 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -15,117 +15,6 @@ tools:
1515
tool_timeout: 20s # cap per tool dispatch; empty/"0"/invalid = 20s default
1616
parallel_tools: false # run tool calls in one model turn concurrently (off = sequential)
1717

18-
# Kubernetes is disabled by default. Select exactly one auth.mode. Every mode
19-
# is implemented in Go; no cloud CLI or Kubernetes auth plugin is executed.
20-
# Except for in_cluster, the endpoint and CA are operator-provided. Private
21-
# endpoints also require allow_private_networks or endpoint_cidrs.
22-
# Examples use ca_data expanded from an environment-backed Secret. Set
23-
# KUBERNETES_CA_DATA to the base64-encoded CA certificate. A mounted PEM file
24-
# is also supported: replace ca_data with ca_file: /run/kubernetes/ca.crt.
25-
# Configure only one of ca_data or ca_file.
26-
kubernetes:
27-
auth:
28-
mode: ""
29-
30-
# In-cluster ServiceAccount token and CA:
31-
# auth:
32-
# mode: in_cluster
33-
34-
# Rotating token file:
35-
# endpoint: https://api.example
36-
# ca_data: ${KUBERNETES_CA_DATA}
37-
# auth:
38-
# mode: token_file
39-
# token_file: /run/secrets/kubernetes/token
40-
41-
# Static token, normally expanded from an environment-backed Secret:
42-
# endpoint: https://api.example
43-
# ca_data: ${KUBERNETES_CA_DATA}
44-
# auth:
45-
# mode: token
46-
# token: ${KUBERNETES_TOKEN}
47-
48-
# Client certificate files (certificate_data/key_data are also supported):
49-
# endpoint: https://api.example
50-
# ca_data: ${KUBERNETES_CA_DATA}
51-
# auth:
52-
# mode: client_certificate
53-
# client_certificate:
54-
# certificate_file: /run/secrets/kubernetes/client.crt
55-
# key_file: /run/secrets/kubernetes/client.key
56-
57-
# Safe kubeconfig. Only token, tokenFile, or client cert/key users are read:
58-
# auth:
59-
# mode: kubeconfig
60-
# kubeconfig:
61-
# path: /run/secrets/kubernetes/config
62-
# context: production
63-
64-
# EKS IAM uses a closed native chain: environment credentials, IRSA,
65-
# static credentials from the selected profile, constrained container
66-
# credentials, then fixed IPv4 IMDSv2. Credential processes, SSO, profile
67-
# role/source chains, web-identity directives, plugins, and endpoint/service
68-
# overrides are rejected. Benign CLI output/pager/retry settings are ignored.
69-
# Optional YAML role_arn uses native STS AssumeRole.
70-
# endpoint: https://API_ID.eks.us-east-1.amazonaws.com
71-
# ca_data: ${KUBERNETES_CA_DATA}
72-
# auth:
73-
# mode: eks
74-
# eks:
75-
# cluster_name: production
76-
# region: us-east-1
77-
# role_arn: arn:aws:iam::123456789012:role/versus-kubernetes-reader
78-
# profile: ""
79-
80-
# AKS workload identity:
81-
# endpoint: https://production.example.azmk8s.io
82-
# ca_data: ${KUBERNETES_CA_DATA}
83-
# auth:
84-
# mode: aks
85-
# aks:
86-
# credential_mode: workload_identity
87-
# environment: public
88-
# server_id: api://AKS_SERVER_APP_ID
89-
# tenant_id: TENANT_ID
90-
# client_id: CLIENT_ID
91-
# federated_token_file: /var/run/secrets/azure/tokens/azure-identity-token
92-
93-
# AKS client secret:
94-
# endpoint: https://production.example.azmk8s.io
95-
# ca_data: ${KUBERNETES_CA_DATA}
96-
# auth:
97-
# mode: aks
98-
# aks:
99-
# credential_mode: client_secret
100-
# environment: public
101-
# server_id: api://AKS_SERVER_APP_ID
102-
# tenant_id: TENANT_ID
103-
# client_id: CLIENT_ID
104-
# client_secret: ${KUBERNETES_AKS_CLIENT_SECRET}
105-
106-
# AKS managed identity:
107-
# endpoint: https://production.example.azmk8s.io
108-
# ca_data: ${KUBERNETES_CA_DATA}
109-
# auth:
110-
# mode: aks
111-
# aks:
112-
# credential_mode: managed_identity
113-
# environment: public
114-
# server_id: api://AKS_SERVER_APP_ID
115-
# client_id: OPTIONAL_USER_ASSIGNED_IDENTITY_CLIENT_ID
116-
117-
# GKE uses only the fixed metadata IP for Workload Identity when
118-
# credentials_file is empty. Set credentials_file explicitly for supported
119-
# service-account or file-backed external-account JSON. Ambient
120-
# GOOGLE_APPLICATION_CREDENTIALS and GCE_METADATA_HOST are rejected in
121-
# every mode, and well-known gcloud ADC files are never discovered.
122-
# endpoint: https://GKE_CONTROL_PLANE_ENDPOINT
123-
# ca_data: ${KUBERNETES_CA_DATA}
124-
# auth:
125-
# mode: gke
126-
# gke:
127-
# credentials_file: /run/secrets/google/credentials.json
128-
12918
# ---------------------------------------------------------------------------
13019
# recent_changes — correlate an incident with recent deploys / config changes
13120
# ---------------------------------------------------------------------------

‎helm/versus-incident/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ The agent's read-only Kubernetes connector supports native in-cluster, token,
66
client-certificate, safe kubeconfig, EKS IAM, AKS Entra, and closed GKE
77
service-account, file-workload-identity, or metadata-workload-identity
88
authentication. It does not require or execute cloud CLIs or auth plugins. See
9-
[Kubernetes Connector](https://docs.versusincident.com/#/agent/tools/kubernetes)
9+
[Kubernetes Connector](https://docs.versusincident.com/#/agent/connectors/kubernetes)
1010
for complete values and cloud prerequisites.
1111

1212
## Requirements

‎helm/versus-incident/templates/_connector-auth.tpl‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{{- define "versus-incident.connectorAuth" -}}
2+
{{- $auth := .auth -}}
3+
{{- $mode := $auth.mode | default "" -}}
4+
mode: {{ $mode | quote }}
5+
{{- if eq $mode "token" }}
6+
token: {{ printf "${%s_TOKEN}" .prefix | quote }}
7+
{{- else if eq $mode "token_file" }}
8+
token_file: {{ $auth.tokenFile | default "" | quote }}
9+
{{- else if eq $mode "client_certificate" }}
10+
client_certificate:
11+
certificate_file: {{ ($auth.clientCertificate | default dict).certificateFile | default "" | quote }}
12+
key_file: {{ ($auth.clientCertificate | default dict).keyFile | default "" | quote }}
13+
certificate_data: {{ printf "${%s_CLIENT_CERTIFICATE_DATA}" .prefix | quote }}
14+
key_data: {{ printf "${%s_CLIENT_KEY_DATA}" .prefix | quote }}
15+
{{- else if eq $mode "kubeconfig" }}
16+
kubeconfig:
17+
path: {{ ($auth.kubeconfig | default dict).path | default "" | quote }}
18+
context: {{ ($auth.kubeconfig | default dict).context | default "" | quote }}
19+
{{- else if eq $mode "eks" }}
20+
eks:
21+
cluster_name: {{ ($auth.eks | default dict).clusterName | default "" | quote }}
22+
region: {{ ($auth.eks | default dict).region | default "" | quote }}
23+
role_arn: {{ ($auth.eks | default dict).roleARN | default "" | quote }}
24+
profile: {{ ($auth.eks | default dict).profile | default "" | quote }}
25+
{{- else if eq $mode "aks" }}
26+
{{- $aks := $auth.aks | default dict }}
27+
{{- $credentialMode := $aks.credentialMode | default "" }}
28+
aks:
29+
credential_mode: {{ $credentialMode | quote }}
30+
server_id: {{ $aks.serverID | default "" | quote }}
31+
client_id: {{ $aks.clientID | default "" | quote }}
32+
{{- if or (eq $credentialMode "workload_identity") (eq $credentialMode "client_secret") }}
33+
tenant_id: {{ $aks.tenantID | default "" | quote }}
34+
{{- end }}
35+
{{- if eq $credentialMode "client_secret" }}
36+
client_secret: {{ printf "${%s_AKS_CLIENT_SECRET}" .prefix | quote }}
37+
{{- else if eq $credentialMode "workload_identity" }}
38+
federated_token_file: {{ $aks.federatedTokenFile | default "" | quote }}
39+
{{- end }}
40+
environment: {{ $aks.environment | default "" | quote }}
41+
{{- else if eq $mode "gke" }}
42+
gke:
43+
credentials_file: {{ ($auth.gke | default dict).credentialsFile | default "" | quote }}
44+
{{- end }}
45+
{{- end -}}

‎helm/versus-incident/templates/configmap.yaml‎

Lines changed: 23 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -324,21 +324,13 @@ data:
324324
{{- end }}
325325
{{- end }}
326326

327-
{{- if .Values.agent.enable }}
328-
{{- $tools := .Values.agent.tools | default dict }}
329-
{{- $kubernetes := $tools.kubernetes | default dict }}
327+
{{- if hasKey (.Values.agent.tools | default dict) "kubernetes" }}
328+
{{- fail "Kubernetes configuration must use connectors.kubernetes" }}
329+
{{- end }}
330+
{{- $kubernetes := (.Values.connectors | default dict).kubernetes | default dict }}
330331
{{- $kubernetesAuth := $kubernetes.auth | default dict }}
331-
{{- $recentChanges := $tools.recentChanges | default dict }}
332-
{{- $git := $recentChanges.git | default dict }}
333-
{{- $describeDeps := $tools.describeDependencies | default dict }}
334-
{{- $findRunbook := $tools.findRunbook | default dict }}
335-
tools.yaml: |
336-
# Per-tool DATA config for the analyze tools (rendered by Helm).
337-
# Credentials are NOT written here — they are injected as env vars from
338-
# the chart Secret and expanded by the loader at boot.
339-
tools:
340-
tool_timeout: {{ $tools.toolTimeout | default "20s" | quote }}
341-
parallel_tools: {{ dig "parallelTools" false $tools }}
332+
connectors.yaml: |
333+
connectors:
342334
kubernetes:
343335
{{- $kubernetesAuthMode := $kubernetesAuth.mode | default "" }}
344336
{{- if and (ne $kubernetesAuthMode "in_cluster") (ne $kubernetesAuthMode "kubeconfig") }}
@@ -395,6 +387,23 @@ data:
395387
gke:
396388
credentials_file: {{ ($kubernetesAuth.gke | default dict).credentialsFile | default "" | quote }}
397389
{{- end }}
390+
{{- $actions := $kubernetes.actions | default dict }}
391+
actions:
392+
enable: {{ dig "enable" false $actions }}
393+
max_replicas: {{ $actions.maxReplicas | default 20 }}
394+
timeout: {{ $actions.timeout | default "15s" | quote }}
395+
auth:
396+
{{- include "versus-incident.connectorAuth" (dict "auth" ($actions.auth | default dict) "prefix" "KUBERNETES_ACTOR") | nindent 12 }}
397+
{{- if .Values.agent.enable }}
398+
{{- $tools := .Values.agent.tools | default dict }}
399+
{{- $recentChanges := $tools.recentChanges | default dict }}
400+
{{- $git := $recentChanges.git | default dict }}
401+
{{- $describeDeps := $tools.describeDependencies | default dict }}
402+
{{- $findRunbook := $tools.findRunbook | default dict }}
403+
tools.yaml: |
404+
tools:
405+
tool_timeout: {{ $tools.toolTimeout | default "20s" | quote }}
406+
parallel_tools: {{ dig "parallelTools" false $tools }}
398407
recent_changes:
399408
git:
400409
auth:

‎helm/versus-incident/templates/deployment.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ spec:
171171
key: git_auth_token_{{ $i }}
172172
{{- end }}
173173
{{- end }}
174+
{{- end }}
174175
- name: KUBERNETES_TOKEN
175176
valueFrom:
176177
secretKeyRef:
@@ -196,6 +197,12 @@ spec:
196197
secretKeyRef:
197198
name: {{ include "versus-incident.fullname" . }}-secrets
198199
key: kubernetes_aks_client_secret
200+
{{- range $key := list "token" "client_certificate_data" "client_key_data" "aks_client_secret" }}
201+
- name: KUBERNETES_ACTOR_{{ $key | upper }}
202+
valueFrom:
203+
secretKeyRef:
204+
name: {{ include "versus-incident.fullname" $ }}-secrets
205+
key: kubernetes_actor_{{ $key }}
199206
{{- end }}
200207

201208
{{- if .Values.alert.slack.enable }}
@@ -589,6 +596,9 @@ spec:
589596
- name: config-volume
590597
mountPath: /app/config/config.yaml
591598
subPath: config.yaml
599+
- name: config-volume
600+
mountPath: /app/config/connectors.yaml
601+
subPath: connectors.yaml
592602
{{- if .Values.agent.enable }}
593603
- name: config-volume
594604
mountPath: /app/config/agent_sources.yaml

‎helm/versus-incident/templates/secret.yaml‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -40,24 +40,29 @@ data:
4040
as env vars that tools.yaml expands via ${VAR}. */}}
4141
{{- if .Values.agent.enable }}
4242
{{- $tools := .Values.agent.tools | default dict }}
43-
{{- $kubernetes := $tools.kubernetes | default dict }}
44-
{{- $kubernetesAuth := $kubernetes.auth | default dict }}
4543
{{- $git := (($tools.recentChanges | default dict).git | default dict) }}
4644
git_auth_token: {{ ($git.auth | default dict).token | default "" | b64enc | quote }}
4745
{{- range $i, $repo := ($git.repos | default list) }}
4846
{{- if $repo.auth }}
4947
git_auth_token_{{ $i }}: {{ $repo.auth.token | default "" | b64enc | quote }}
5048
{{- end }}
5149
{{- end }}
50+
{{- end }}
51+
{{- $kubernetes := (.Values.connectors | default dict).kubernetes | default dict }}
52+
{{- $kubernetesAuth := $kubernetes.auth | default dict }}
53+
{{- $actorAuth := ($kubernetes.actions | default dict).auth | default dict }}
5254
{{- dict
5355
"kubernetes_token" ($kubernetesAuth.token | default "" | b64enc)
5456
"kubernetes_ca_data" ($kubernetes.caData | default "" | b64enc)
5557
"kubernetes_client_certificate_data" (($kubernetesAuth.clientCertificate | default dict).certificateData | default "" | b64enc)
5658
"kubernetes_client_key_data" (($kubernetesAuth.clientCertificate | default dict).keyData | default "" | b64enc)
5759
"kubernetes_aks_client_secret" (($kubernetesAuth.aks | default dict).clientSecret | default "" | b64enc)
60+
"kubernetes_actor_token" ($actorAuth.token | default "" | b64enc)
61+
"kubernetes_actor_client_certificate_data" (($actorAuth.clientCertificate | default dict).certificateData | default "" | b64enc)
62+
"kubernetes_actor_client_key_data" (($actorAuth.clientCertificate | default dict).keyData | default "" | b64enc)
63+
"kubernetes_actor_aks_client_secret" (($actorAuth.aks | default dict).clientSecret | default "" | b64enc)
5864
| toYaml | nindent 2
5965
}}
60-
{{- end }}
6166

6267
{{- /* Proxy configuration secrets */}}
6368
{{- if .Values.proxy.url }}

0 commit comments

Comments
 (0)