Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions api/v1alpha1/agentworkload_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,10 @@ type AgentWorkloadStatus struct {
// +optional
ModelRoutingOperationID string `json:"modelRoutingOperationID,omitempty"`

// sandboxExecutionUID identifies the execution that last produced sandbox evidence.
// +optional
SandboxExecutionUID string `json:"sandboxExecutionUID,omitempty"`

// conditions represent the current state of the AgentWorkload resource.
// Each condition has a unique type and reflects the status of a specific aspect of the resource.
//
Expand All @@ -386,6 +390,14 @@ type AgentWorkloadStatus struct {
AgentStatuses []AgentInstanceStatus `json:"agentStatuses,omitempty"`
}

const (
ConditionTypeSandboxEnforced = "SandboxEnforced"
SandboxEnforcedVerified = "Verified"
SandboxEnforcedRuntimeClassMissing = "RuntimeClassMissing"
SandboxEnforcedNoReadyNode = "NoReadyNode"
SandboxEnforcedBestEffortUnsandboxed = "BestEffortUnsandboxed"
)

// AgentInstanceStatus reports the status of a single agent in a collaborative workload
type AgentInstanceStatus struct {
// name matches the AgentCard name from spec.agentRefs[]
Expand Down
15 changes: 12 additions & 3 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ import (
"github.com/go-logr/logr"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth"
Expand All @@ -38,6 +40,8 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/healthz"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
Expand All @@ -50,6 +54,7 @@ import (
"github.com/Clawdlinux/agentic-operator-core/internal/controller"
"github.com/Clawdlinux/agentic-operator-core/pkg/evaluation"
"github.com/Clawdlinux/agentic-operator-core/pkg/finops"
"github.com/Clawdlinux/agentic-operator-core/pkg/governance"
"github.com/Clawdlinux/agentic-operator-core/pkg/multitenancy"
"github.com/Clawdlinux/agentic-operator-core/pkg/otel/genai"
// +kubebuilder:scaffold:imports
Expand Down Expand Up @@ -204,8 +209,12 @@ func main() {
metricsServerOptions.KeyName = metricsCertKey
}

sandboxConfig := runtimeadmission.RuntimeClassInjectionConfigFromEnv()
mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
Scheme: scheme,
Scheme: scheme,
Cache: cache.Options{ByObject: map[client.Object]cache.ByObject{
&corev1.Pod{}: {Label: labels.SelectorFromSet(labels.Set{governance.ManagedByLabelKey: governance.ManagedByLabelValue})},
}},
Metrics: metricsServerOptions,
WebhookServer: webhookServer,
HealthProbeBindAddress: probeAddr,
Expand Down Expand Up @@ -264,6 +273,8 @@ func main() {
workloadReconciler.QuotaMgr = quotaMgr // Phase 7: Quota enforcement
workloadReconciler.SLAMonitor = slaMonitor // Phase 7: SLA tracking
workloadReconciler.TenantRes = tenantResolver // Phase 7: Tenant isolation
workloadReconciler.SandboxClass = sandboxConfig.RuntimeClassName
workloadReconciler.Recorder = mgr.GetEventRecorder("agentworkload-controller")

if err := workloadReconciler.SetupWithManager(mgr); err != nil {
setupLog.Error(err, "Failed to create controller", "controller", "AgentWorkload")
Expand All @@ -283,8 +294,6 @@ func main() {
setupLog.Error(err, "Failed to setup webhook", "webhook", "AgentWorkload")
os.Exit(1)
}
sandboxConfig := runtimeadmission.RuntimeClassInjectionConfigFromEnv()

mgr.GetWebhookServer().Register("/mutate-v1-pod-runtimeclass", &webhook.Admission{
Handler: &runtimeadmission.RuntimeClassInjector{
Config: sandboxConfig,
Expand Down
2 changes: 2 additions & 0 deletions config/crd/agentworkload_crd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,8 @@ spec:
type: string
modelRoutingOperationID:
type: string
sandboxExecutionUID:
type: string
workflowArtifacts:
type: object
additionalProperties:
Expand Down
6 changes: 5 additions & 1 deletion config/crd/bases/agentic.clawdlinux.org_agentworkloads.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.20.1
controller-gen.kubebuilder.io/version: v0.22.0
name: agentworkloads.agentic.clawdlinux.org
spec:
group: agentic.clawdlinux.org
Expand Down Expand Up @@ -546,6 +546,10 @@ spec:
description: readyAgents is the number of agents ready to execute
format: int32
type: integer
sandboxExecutionUID:
description: sandboxExecutionUID identifies the execution that last
produced sandbox evidence.
type: string
workflowArtifacts:
additionalProperties:
type: string
Expand Down
8 changes: 8 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,14 @@ rules:
- patch
- update
- watch
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
Expand Down
15 changes: 11 additions & 4 deletions internal/admission/runtimeclass_injector.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,17 @@ import (

corev1 "k8s.io/api/core/v1"
ctrladmission "sigs.k8s.io/controller-runtime/pkg/webhook/admission"

"github.com/Clawdlinux/agentic-operator-core/pkg/governance"
)

const (
DefaultRuntimeClassName = "gvisor"
DefaultRuntimeLabelKey = "agentic.clawdlinux.org/runtime-sandbox"
DefaultRuntimeLabelValue = "gvisor"
DefaultRuntimeLabelKey = governance.RuntimeSandboxLabelKey
DefaultRuntimeLabelValue = governance.RuntimeSandboxLabelValue
EnforcementModeStrict = "strict"
EnforcementModeBestEffort = "best-effort"
SandboxDenialPrefix = "sandbox readiness unverified: "
)

type RuntimeClassInjectionConfig struct {
Expand Down Expand Up @@ -98,9 +101,13 @@ func (i *RuntimeClassInjector) Handle(ctx context.Context, req ctrladmission.Req

func (i *RuntimeClassInjector) readinessFailureResponse(reason string) ctrladmission.Response {
if enforcementMode(i.Config.EnforcementMode) == EnforcementModeBestEffort {
return ctrladmission.Allowed("sandbox readiness unverified: " + reason)
return ctrladmission.Allowed(SandboxDenialPrefix + reason)
}
return ctrladmission.Denied("sandbox readiness unverified: " + reason)
return ctrladmission.Denied(SandboxDenialPrefix + reason)
}

func IsSandboxDenial(message string) bool {
return strings.Contains(message, SandboxDenialPrefix)
}

func enforcementMode(value string) string {
Expand Down
9 changes: 9 additions & 0 deletions internal/admission/runtimeclass_injector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,15 @@ func TestRuntimeClassInjectorHandleRejectsInvalidPodJSON(t *testing.T) {
}
}

func TestIsSandboxDenial(t *testing.T) {
if !IsSandboxDenial("pod adapter execute: create pod: admission webhook denied the request: " + SandboxDenialPrefix + SandboxReadinessNoReadyNode) {
t.Fatal("wrapped sandbox denial was not recognized")
}
if IsSandboxDenial("admission webhook denied the request: another reason") {
t.Fatal("non-sandbox denial was recognized")
}
}

func mustMarshalPod(t *testing.T, pod *corev1.Pod) []byte {
t.Helper()
raw, err := json.Marshal(pod)
Expand Down
7 changes: 7 additions & 0 deletions internal/controller/agentworkload_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/events"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
Expand Down Expand Up @@ -77,6 +78,8 @@ type AgentWorkloadReconciler struct {
Metrics *metrics.RoutingMetrics // Singleton metrics recorder (initialized once)
RetryConfig *resilience.RetryConfig // optional test seam; nil uses production defaults
RuntimeRegistry *runtimeadapter.Registry // runtime adapter registry; nil lazily defaults to Argo
SandboxClass string // RuntimeClass required for sandbox enforcement
Recorder events.EventRecorder // Optional Kubernetes event recorder
}

type quotaChecker interface {
Expand Down Expand Up @@ -148,6 +151,7 @@ func (r *AgentWorkloadReconciler) ensureRuntimeDefaults() {
// +kubebuilder:rbac:groups=agentic.clawdlinux.org,resources=agentworkloads/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=agentic.clawdlinux.org,resources=agentworkloads/finalizers,verbs=update
// +kubebuilder:rbac:groups="",resources=namespaces,verbs=get;list;watch;patch;update
// +kubebuilder:rbac:groups="",resources=pods,verbs=get;list;watch

// Reconcile reconciles the AgentWorkload by:
// 1. Fetching the AgentWorkload CR
Expand Down Expand Up @@ -741,6 +745,7 @@ func (r *AgentWorkloadReconciler) reconcileViaRuntime(ctx context.Context, workl

// Update phase based on normalized execution status
workload.Status.ArgoPhase = execStatus.Phase
r.observeSandboxStatus(ctx, workload, execStatus.Message)
switch execStatus.Phase {
case "Succeeded":
workload.Status.Phase = "Completed"
Expand All @@ -766,6 +771,7 @@ func (r *AgentWorkloadReconciler) reconcileViaRuntime(ctx context.Context, workl
log.Error(err, "failed to create execution")
workload.Status.Phase = "Failed"
workload.Status.ArgoPhase = "Error"
r.observeSandboxStatus(ctx, workload, err.Error())
if err := r.Status().Update(ctx, workload); err != nil {
log.Error(err, "failed to update status")
}
Expand All @@ -784,6 +790,7 @@ func (r *AgentWorkloadReconciler) reconcileViaRuntime(ctx context.Context, workl
UID: execStatus.UID,
CreatedAt: &metav1.Time{Time: time.Now()},
}
r.observeSandboxStatus(ctx, workload, execStatus.Message)

if err := r.Status().Update(ctx, workload); err != nil {
log.Error(err, "failed to update status with execution reference")
Expand Down
69 changes: 69 additions & 0 deletions internal/controller/sandbox_observer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
package controller

import (
"context"
"strings"
"testing"

corev1 "k8s.io/api/core/v1"
apiMeta "k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/tools/events"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

agenticv1alpha1 "github.com/Clawdlinux/agentic-operator-core/api/v1alpha1"
"github.com/Clawdlinux/agentic-operator-core/pkg/governance"
)

func TestObserveSandboxStatus_SelectsPodsByWorkloadUID(t *testing.T) {
gvisor := "gvisor"
workload := &agenticv1alpha1.AgentWorkload{
ObjectMeta: metav1.ObjectMeta{Name: "research", Namespace: "tenant-a", UID: types.UID("workload-uid"), Generation: 2},
Status: agenticv1alpha1.AgentWorkloadStatus{ArgoWorkflow: &agenticv1alpha1.ArgoWorkflowRef{Name: "run", UID: "execution-uid"}},
}
matchingPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "run-1", Namespace: "argo-workflows", Labels: map[string]string{governance.WorkloadUIDLabelKey: "workload-uid", governance.ManagedByLabelKey: governance.ManagedByLabelValue}}, Spec: corev1.PodSpec{RuntimeClassName: &gvisor}}
otherPod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "other", Namespace: "argo-workflows", Labels: map[string]string{governance.WorkloadUIDLabelKey: "other-uid", governance.ManagedByLabelKey: governance.ManagedByLabelValue}}}
reconciler := &AgentWorkloadReconciler{Client: fake.NewClientBuilder().WithScheme(newControllerTestScheme(t)).WithObjects(matchingPod, otherPod).Build()}

reconciler.observeSandboxStatus(context.Background(), workload, "")
condition := apiMeta.FindStatusCondition(workload.Status.Conditions, agenticv1alpha1.ConditionTypeSandboxEnforced)
if condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != agenticv1alpha1.SandboxEnforcedVerified {
t.Fatalf("sandbox condition = %#v, want verified", condition)
}
if workload.Status.SandboxExecutionUID != "execution-uid" {
t.Fatalf("sandbox execution UID = %q, want execution-uid", workload.Status.SandboxExecutionUID)
}
}

func TestObserveSandboxStatus_EmitsWarningOnlyOnFalseTransition(t *testing.T) {
workload := &agenticv1alpha1.AgentWorkload{
ObjectMeta: metav1.ObjectMeta{Name: "research", Namespace: "tenant-a", UID: types.UID("workload-uid")},
Status: agenticv1alpha1.AgentWorkloadStatus{ArgoWorkflow: &agenticv1alpha1.ArgoWorkflowRef{Name: "run"}},
}
recorder := events.NewFakeRecorder(1)
reconciler := &AgentWorkloadReconciler{Client: fake.NewClientBuilder().WithScheme(newControllerTestScheme(t)).WithObjects(&corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "run-1", Labels: map[string]string{governance.WorkloadUIDLabelKey: "workload-uid", governance.ManagedByLabelKey: governance.ManagedByLabelValue}}}).Build(), Recorder: recorder}

reconciler.observeSandboxStatus(context.Background(), workload, "")
select {
case event := <-recorder.Events:
if !strings.Contains(event, agenticv1alpha1.SandboxEnforcedBestEffortUnsandboxed) {
t.Fatalf("event = %q, want unsandboxed reason", event)
}
default:
t.Fatal("expected warning event")
}
reconciler.observeSandboxStatus(context.Background(), workload, "")
select {
case event := <-recorder.Events:
t.Fatalf("unexpected repeated warning event: %q", event)
default:
}
workload.Status.Conditions[0].Reason = "PreviousUnsandboxedReason"
reconciler.observeSandboxStatus(context.Background(), workload, "")
select {
case event := <-recorder.Events:
t.Fatalf("unexpected warning for false condition reason change: %q", event)
default:
}
}
Loading
Loading