From ccdf585f4f957e1ff3c4e635b5a5e052b9f20e1a Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:43:33 +0330 Subject: [PATCH 01/31] Chore: Add Missing Usage type(scheduler just dropped it) --- persys-scheduler/internal/scheduler/workload_projection.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/persys-scheduler/internal/scheduler/workload_projection.go b/persys-scheduler/internal/scheduler/workload_projection.go index 967e5e8..674f855 100644 --- a/persys-scheduler/internal/scheduler/workload_projection.go +++ b/persys-scheduler/internal/scheduler/workload_projection.go @@ -38,6 +38,7 @@ type workloadStatus struct { Metadata map[string]interface{} `json:"metadata,omitempty"` Retry models.RetryState `json:"retry"` StatusInfo models.WorkloadStatusInfo `json:"statusInfo"` + Usage *models.WorkloadUsage `json:"usage,omitempty"` } func workloadSpecFromWorkload(w models.Workload) workloadSpec { @@ -79,5 +80,6 @@ func workloadStatusFromWorkload(w models.Workload) workloadStatus { Metadata: w.Metadata, Retry: w.Retry, StatusInfo: w.StatusInfo, + Usage: w.Usage, } } From f6f976054753b66c250a620003a472da7f7ae6e9 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:44:34 +0330 Subject: [PATCH 02/31] Feat: Optimize workload status handling and event emission process --- .../internal/scheduler/state_store.go | 175 +++++++++++++++--- 1 file changed, 149 insertions(+), 26 deletions(-) diff --git a/persys-scheduler/internal/scheduler/state_store.go b/persys-scheduler/internal/scheduler/state_store.go index efc2b5c..b9f740a 100644 --- a/persys-scheduler/internal/scheduler/state_store.go +++ b/persys-scheduler/internal/scheduler/state_store.go @@ -23,7 +23,6 @@ const ( reconciliationPrefix = "/reconciliation/" retriesPrefix = "/retries/" driftsPrefix = "/drifts/" - eventsPrefix = "/events/" managedStorageStateKey = "managed_storage_state" ) @@ -35,7 +34,6 @@ func attachmentPrefix() string { return attachmentsPrefix } func assignmentKey(workloadID string) string { return assignmentsPrefix + workloadID } func reconciliationKey(workloadID string) string { return reconciliationPrefix + workloadID } func retryKey(workloadID string) string { return retriesPrefix + workloadID } -func eventKey(eventID string) string { return eventsPrefix + eventID } func volumeAttachmentKey(nodeID, workloadID, volumeID string) string { return attachmentsPrefix + sanitizeKeySegment(nodeID) + "/" + sanitizeKeySegment(workloadID) + "/" + sanitizeKeySegment(volumeID) } @@ -130,11 +128,11 @@ func (s *Scheduler) saveWorkload(workload models.Workload) error { } metricspkg.IncStateStoreWrite("status") s.cacheWorkload(workload) - retryPayload, err := json.Marshal(workload.Retry) - if err == nil { - _ = s.RetryableEtcdPut(retryKey(workload.ID), string(retryPayload)) - metricspkg.IncStateStoreWrite("retry") - } + // NOTE: retry state is embedded in the status projection above + // (workloadStatus.Retry) and read back from there — the separate + // /retries/{id} key that used to be written here was never read by + // anything in this codebase, so the write was pure overhead and has + // been removed. if shouldSyncManagedStorage(workload) { if err := s.syncWorkloadManagedStorage(workload); err != nil { return fmt.Errorf("sync managed storage state for workload %s: %w", workload.ID, err) @@ -143,6 +141,119 @@ func (s *Scheduler) saveWorkload(workload models.Workload) error { return nil } +// maxWorkloadStatusCASRetries bounds how many times updateWorkloadStatusCAS +// will re-read and re-apply a mutation after losing a compare-and-swap race +// before giving up. Mirrors maxCASConflictRetries for nodes; contention on +// a single workload's status key is normally limited to that workload's own +// agent (via heartbeat) and the reconciler, so this should rarely be +// exhausted even under load. +const maxWorkloadStatusCASRetries = 5 + +// getWorkloadWithStatusRevision reads a workload the same way GetWorkloadByID +// does (merging the spec and status projections into a full models.Workload), +// but also returns the etcd ModRevision of the status key so callers can +// compare-and-swap against it. modRevision is 0 if the status key doesn't +// exist yet (brand new workload) or the record was found via the legacy +// full-object compatibility path, in which case a CAS put behaves like +// "only succeed if the status key still doesn't exist." +func (s *Scheduler) getWorkloadWithStatusRevision(workloadID string) (models.Workload, int64, error) { + specResp, err := s.RetryableEtcdGet(workloadSpecKey(workloadID)) + if err != nil { + return models.Workload{}, 0, fmt.Errorf("failed to get workload %s: %v", workloadID, err) + } + + if specResp == nil || len(specResp.Kvs) == 0 { + // Legacy compatibility shim, same as GetWorkloadByID. + resp, legacyErr := s.RetryableEtcdGet("/workloads/" + workloadID) + if legacyErr != nil || resp == nil || len(resp.Kvs) == 0 { + return models.Workload{}, 0, fmt.Errorf("workload %s not found", workloadID) + } + var legacy models.Workload + if err := json.Unmarshal(resp.Kvs[0].Value, &legacy); err != nil { + return models.Workload{}, 0, fmt.Errorf("failed to unmarshal legacy workload %s: %v", workloadID, err) + } + return legacy, 0, nil + } + + var spec workloadSpec + if err := json.Unmarshal(specResp.Kvs[0].Value, &spec); err != nil { + return models.Workload{}, 0, fmt.Errorf("failed to unmarshal workload spec %s: %v", workloadID, err) + } + + statusResp, _ := s.RetryableEtcdGet(workloadStatusKey(workloadID)) + var st workloadStatus + var modRevision int64 + if statusResp != nil && len(statusResp.Kvs) > 0 { + _ = json.Unmarshal(statusResp.Kvs[0].Value, &st) + modRevision = statusResp.Kvs[0].ModRevision + } + + workload := models.Workload{ + ID: spec.ID, Name: spec.Name, Type: spec.Type, RevisionID: spec.RevisionID, Image: spec.Image, Command: spec.Command, + CommandList: spec.CommandList, Compose: spec.Compose, ComposeYAML: spec.ComposeYAML, ProjectName: spec.ProjectName, + GitRepo: spec.GitRepo, GitBranch: spec.GitBranch, GitToken: spec.GitToken, EnvVars: spec.EnvVars, Resources: spec.Resources, + DesiredState: spec.DesiredState, Labels: spec.Labels, LocalPath: spec.LocalPath, Ports: spec.Ports, Volumes: spec.Volumes, + Network: spec.Network, RestartPolicy: spec.RestartPolicy, VM: spec.VM, + } + if st.ID != "" { + workload.AssignedNode = st.AssignedNode + workload.NodeID = st.NodeID + workload.Status = st.Status + workload.Logs = st.Logs + workload.Metadata = st.Metadata + workload.Retry = st.Retry + workload.StatusInfo = st.StatusInfo + workload.Usage = st.Usage + } + return workload, modRevision, nil +} + +// updateWorkloadStatusCAS reads a workload, lets mutate modify the in-memory +// copy, and persists only the status projection (/workloads-status/{id}) +// via an etcd compare-and-swap on that key's ModRevision — re-reading and +// re-applying the mutation (up to maxWorkloadStatusCASRetries times) if +// another writer updated the status key in between, instead of silently +// overwriting whatever that writer just changed. This is the workload-side +// counterpart to updateNodeCAS: the concurrent writers here are typically a +// workload's own agent (via heartbeat) and the reconciler, both of which +// can touch the same workload's status around the same time. +// +// mutate returns false to signal "nothing to change" after inspecting the +// freshest read (mirroring the no-op-skip checks the four callers already +// had) — in that case this returns without writing. +func (s *Scheduler) updateWorkloadStatusCAS(workloadID string, mutate func(*models.Workload) bool) (models.Workload, error) { + if err := s.requireWritable(); err != nil { + return models.Workload{}, err + } + statusKey := workloadStatusKey(workloadID) + var lastConflictErr error + for attempt := 0; attempt < maxWorkloadStatusCASRetries; attempt++ { + workload, modRevision, err := s.getWorkloadWithStatusRevision(workloadID) + if err != nil { + return models.Workload{}, err + } + if !mutate(&workload) { + return workload, nil + } + payload, err := json.Marshal(workloadStatusFromWorkload(workload)) + if err != nil { + return models.Workload{}, fmt.Errorf("marshal workload status %s: %w", workloadID, err) + } + ok, err := s.RetryableEtcdCASPut(statusKey, string(payload), modRevision) + if err != nil { + return models.Workload{}, err + } + if !ok { + lastConflictErr = fmt.Errorf("workload %s status changed concurrently (attempt %d)", workloadID, attempt+1) + continue + } + metricspkg.IncStateStoreWrite("status") + s.cacheWorkload(workload) + return workload, nil + } + return models.Workload{}, fmt.Errorf("workload %s: too many concurrent status update conflicts: %w", workloadID, lastConflictErr) +} + func (s *Scheduler) writeAssignment(workloadID, nodeID, reason string) error { rec := models.AssignmentRecord{ WorkloadID: workloadID, @@ -167,7 +278,28 @@ func (s *Scheduler) writeReconciliationRecord(workloadID, action string, success metricspkg.IncStateStoreWrite("reconciliation") } +// emitEvent records a cluster-wide scheduler event (node lost, workload +// scheduled, drift detected, etc) to the shared Redis Stream +// (schedulerEventsStreamKey, redis_store.go). +// +// This intentionally does NOT write to etcd. Events are high-churn, +// ephemeral, observability-oriented data — exactly the kind of data etcd +// is a poor fit for at scale: every event write would be an etcd PUT (and, +// with a watch-based consumer, a watch-fanout notification) landing on the +// same datastore that holds authoritative cluster state, competing for +// write throughput with heartbeats and CAS-retried reconciliation writes +// at precisely the moments (incidents, node flapping, mass retries) when +// event volume — and etcd load from everything else — is highest. Redis +// Streams give bounded, O(1)-amortized retention (MAXLEN ~) for free, +// versus the scan-and-delete sweep an etcd-backed version would need. +// +// If Redis is unavailable, the event is dropped (logged, not retried, +// no etcd fallback) — acceptable for best-effort observability data that +// nothing else in the scheduler depends on for correctness. func (s *Scheduler) emitEvent(eventType, workloadID, nodeID, reason string, details map[string]interface{}) { + if eventType == "" || !isKnownSchedulerEventType(eventType) { + redisLogger.WithField("event_type", eventType).Warn("unknown scheduler event type emitted") + } event := models.SchedulerEvent{ ID: uuid.NewString(), Type: eventType, @@ -181,11 +313,11 @@ func (s *Scheduler) emitEvent(eventType, workloadID, nodeID, reason string, deta if err != nil { return } - if s.writeEventTelemetry(payload) { - metricspkg.IncStateStoreWrite("event") + if _, err := s.writeEventToStream(payload); err != nil { + redisLogger.WithError(err).WithField("event_type", eventType).Warn("failed to emit cluster event to redis; event dropped") + metricspkg.IncStateStoreWrite("event_dropped") return } - _ = s.RetryableEtcdPut(eventKey(event.ID), string(payload)) metricspkg.IncStateStoreWrite("event") } @@ -201,23 +333,14 @@ func (s *Scheduler) clearDriftRecord(nodeID, workloadID, driftType string) { _ = s.RetryableEtcdDelete(driftKey(nodeID, workloadID, driftType)) } +// ListSchedulerEvents returns the most recent cluster-wide events, newest +// first, from the shared Redis Stream (see emitEvent/writeEventToStream). +// limit <= 0 returns up to the configured max-entries retention bound. +// Returns an empty slice (never an error) if Redis is unavailable — see +// readEventsFromStream's doc comment for why this degrades gracefully +// instead of failing the caller. func (s *Scheduler) ListSchedulerEvents(limit int64) ([]models.SchedulerEvent, error) { - opts := []clientv3.OpOption{clientv3.WithPrefix()} - if limit > 0 { - opts = append(opts, clientv3.WithLimit(limit)) - } - resp, err := s.RetryableEtcdGet(eventsPrefix, opts...) - if err != nil { - return nil, err - } - events := make([]models.SchedulerEvent, 0, len(resp.Kvs)) - for _, kv := range resp.Kvs { - var event models.SchedulerEvent - if err := json.Unmarshal(kv.Value, &event); err != nil { - continue - } - events = append(events, event) - } + events, _ := s.readEventsFromStream(limit) return events, nil } From 0be9ac86b164b76474054ce04244f95c7d1c6e20 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:45:19 +0330 Subject: [PATCH 03/31] Chore: Update Protobuf implementation(make proto) --- pkg/agent/api/v1/agent.pb.go | 9 +- pkg/agent/control/v1/control.pb.go | 1295 ++++++++++++++++++++--- pkg/agent/control/v1/control_grpc.pb.go | 230 ++++ 3 files changed, 1397 insertions(+), 137 deletions(-) diff --git a/pkg/agent/api/v1/agent.pb.go b/pkg/agent/api/v1/agent.pb.go index eb9ee71..d9eb750 100644 --- a/pkg/agent/api/v1/agent.pb.go +++ b/pkg/agent/api/v1/agent.pb.go @@ -28,6 +28,8 @@ const ( WorkloadType_WORKLOAD_TYPE_CONTAINER WorkloadType = 1 WorkloadType_WORKLOAD_TYPE_COMPOSE WorkloadType = 2 WorkloadType_WORKLOAD_TYPE_VM WorkloadType = 3 + // Firecracker microVM. Shares WorkloadSpec.vm oneof with KVM VMs. + WorkloadType_WORKLOAD_TYPE_MICROVM WorkloadType = 4 ) // Enum value maps for WorkloadType. @@ -37,12 +39,14 @@ var ( 1: "WORKLOAD_TYPE_CONTAINER", 2: "WORKLOAD_TYPE_COMPOSE", 3: "WORKLOAD_TYPE_VM", + 4: "WORKLOAD_TYPE_MICROVM", } WorkloadType_value = map[string]int32{ "WORKLOAD_TYPE_UNSPECIFIED": 0, "WORKLOAD_TYPE_CONTAINER": 1, "WORKLOAD_TYPE_COMPOSE": 2, "WORKLOAD_TYPE_VM": 3, + "WORKLOAD_TYPE_MICROVM": 4, } ) @@ -2386,12 +2390,13 @@ const file_agent_proto_rawDesc = "" + "netTxBytes\x12!\n" + "\fcollected_at\x18\t \x01(\x03R\vcollectedAt\x12\x16\n" + "\x06source\x18\n" + - " \x01(\tR\x06source*{\n" + + " \x01(\tR\x06source*\x96\x01\n" + "\fWorkloadType\x12\x1d\n" + "\x19WORKLOAD_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17WORKLOAD_TYPE_CONTAINER\x10\x01\x12\x19\n" + "\x15WORKLOAD_TYPE_COMPOSE\x10\x02\x12\x14\n" + - "\x10WORKLOAD_TYPE_VM\x10\x03*c\n" + + "\x10WORKLOAD_TYPE_VM\x10\x03\x12\x19\n" + + "\x15WORKLOAD_TYPE_MICROVM\x10\x04*c\n" + "\fDesiredState\x12\x1d\n" + "\x19DESIRED_STATE_UNSPECIFIED\x10\x00\x12\x19\n" + "\x15DESIRED_STATE_RUNNING\x10\x01\x12\x19\n" + diff --git a/pkg/agent/control/v1/control.pb.go b/pkg/agent/control/v1/control.pb.go index e882714..583bcbf 100644 --- a/pkg/agent/control/v1/control.pb.go +++ b/pkg/agent/control/v1/control.pb.go @@ -4882,6 +4882,914 @@ func (x *DiskView) GetUpdatedAt() *timestamppb.Timestamp { return nil } +type CreateBucketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` + Versioning bool `protobuf:"varint,3,opt,name=versioning,proto3" json:"versioning,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBucketRequest) Reset() { + *x = CreateBucketRequest{} + mi := &file_control_proto_msgTypes[69] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBucketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBucketRequest) ProtoMessage() {} + +func (x *CreateBucketRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[69] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBucketRequest.ProtoReflect.Descriptor instead. +func (*CreateBucketRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{69} +} + +func (x *CreateBucketRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CreateBucketRequest) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *CreateBucketRequest) GetVersioning() bool { + if x != nil { + return x.Versioning + } + return false +} + +type CreateBucketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bucket *BucketView `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + Access *BucketAccess `protobuf:"bytes,2,opt,name=access,proto3" json:"access,omitempty"` // credentials returned once on create + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBucketResponse) Reset() { + *x = CreateBucketResponse{} + mi := &file_control_proto_msgTypes[70] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBucketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBucketResponse) ProtoMessage() {} + +func (x *CreateBucketResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[70] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBucketResponse.ProtoReflect.Descriptor instead. +func (*CreateBucketResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{70} +} + +func (x *CreateBucketResponse) GetBucket() *BucketView { + if x != nil { + return x.Bucket + } + return nil +} + +func (x *CreateBucketResponse) GetAccess() *BucketAccess { + if x != nil { + return x.Access + } + return nil +} + +type ListBucketsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketsRequest) Reset() { + *x = ListBucketsRequest{} + mi := &file_control_proto_msgTypes[71] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketsRequest) ProtoMessage() {} + +func (x *ListBucketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[71] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketsRequest.ProtoReflect.Descriptor instead. +func (*ListBucketsRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{71} +} + +type ListBucketsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Buckets []*BucketView `protobuf:"bytes,1,rep,name=buckets,proto3" json:"buckets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketsResponse) Reset() { + *x = ListBucketsResponse{} + mi := &file_control_proto_msgTypes[72] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketsResponse) ProtoMessage() {} + +func (x *ListBucketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[72] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketsResponse.ProtoReflect.Descriptor instead. +func (*ListBucketsResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{72} +} + +func (x *ListBucketsResponse) GetBuckets() []*BucketView { + if x != nil { + return x.Buckets + } + return nil +} + +type GetBucketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` // id or name + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketRequest) Reset() { + *x = GetBucketRequest{} + mi := &file_control_proto_msgTypes[73] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketRequest) ProtoMessage() {} + +func (x *GetBucketRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[73] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketRequest.ProtoReflect.Descriptor instead. +func (*GetBucketRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{73} +} + +func (x *GetBucketRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +type GetBucketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bucket *BucketView `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketResponse) Reset() { + *x = GetBucketResponse{} + mi := &file_control_proto_msgTypes[74] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketResponse) ProtoMessage() {} + +func (x *GetBucketResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[74] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketResponse.ProtoReflect.Descriptor instead. +func (*GetBucketResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{74} +} + +func (x *GetBucketResponse) GetBucket() *BucketView { + if x != nil { + return x.Bucket + } + return nil +} + +type DeleteBucketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` + Force bool `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBucketRequest) Reset() { + *x = DeleteBucketRequest{} + mi := &file_control_proto_msgTypes[75] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBucketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBucketRequest) ProtoMessage() {} + +func (x *DeleteBucketRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[75] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBucketRequest.ProtoReflect.Descriptor instead. +func (*DeleteBucketRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{75} +} + +func (x *DeleteBucketRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +func (x *DeleteBucketRequest) GetForce() bool { + if x != nil { + return x.Force + } + return false +} + +type DeleteBucketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBucketResponse) Reset() { + *x = DeleteBucketResponse{} + mi := &file_control_proto_msgTypes[76] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBucketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBucketResponse) ProtoMessage() {} + +func (x *DeleteBucketResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[76] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBucketResponse.ProtoReflect.Descriptor instead. +func (*DeleteBucketResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{76} +} + +func (x *DeleteBucketResponse) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +func (x *DeleteBucketResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +type GetBucketAccessRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketAccessRequest) Reset() { + *x = GetBucketAccessRequest{} + mi := &file_control_proto_msgTypes[77] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketAccessRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketAccessRequest) ProtoMessage() {} + +func (x *GetBucketAccessRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[77] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketAccessRequest.ProtoReflect.Descriptor instead. +func (*GetBucketAccessRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{77} +} + +func (x *GetBucketAccessRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +type GetBucketAccessResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Access *BucketAccess `protobuf:"bytes,1,opt,name=access,proto3" json:"access,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketAccessResponse) Reset() { + *x = GetBucketAccessResponse{} + mi := &file_control_proto_msgTypes[78] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketAccessResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketAccessResponse) ProtoMessage() {} + +func (x *GetBucketAccessResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[78] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketAccessResponse.ProtoReflect.Descriptor instead. +func (*GetBucketAccessResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{78} +} + +func (x *GetBucketAccessResponse) GetAccess() *BucketAccess { + if x != nil { + return x.Access + } + return nil +} + +type ListBucketObjectsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` + Prefix string `protobuf:"bytes,2,opt,name=prefix,proto3" json:"prefix,omitempty"` + ContinuationToken string `protobuf:"bytes,3,opt,name=continuation_token,json=continuationToken,proto3" json:"continuation_token,omitempty"` + MaxKeys int32 `protobuf:"varint,4,opt,name=max_keys,json=maxKeys,proto3" json:"max_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketObjectsRequest) Reset() { + *x = ListBucketObjectsRequest{} + mi := &file_control_proto_msgTypes[79] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketObjectsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketObjectsRequest) ProtoMessage() {} + +func (x *ListBucketObjectsRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[79] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketObjectsRequest.ProtoReflect.Descriptor instead. +func (*ListBucketObjectsRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{79} +} + +func (x *ListBucketObjectsRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +func (x *ListBucketObjectsRequest) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +func (x *ListBucketObjectsRequest) GetContinuationToken() string { + if x != nil { + return x.ContinuationToken + } + return "" +} + +func (x *ListBucketObjectsRequest) GetMaxKeys() int32 { + if x != nil { + return x.MaxKeys + } + return 0 +} + +type ListBucketObjectsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Objects []*ObjectInfo `protobuf:"bytes,1,rep,name=objects,proto3" json:"objects,omitempty"` + NextContinuationToken string `protobuf:"bytes,2,opt,name=next_continuation_token,json=nextContinuationToken,proto3" json:"next_continuation_token,omitempty"` + IsTruncated bool `protobuf:"varint,3,opt,name=is_truncated,json=isTruncated,proto3" json:"is_truncated,omitempty"` + Prefix string `protobuf:"bytes,4,opt,name=prefix,proto3" json:"prefix,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketObjectsResponse) Reset() { + *x = ListBucketObjectsResponse{} + mi := &file_control_proto_msgTypes[80] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketObjectsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketObjectsResponse) ProtoMessage() {} + +func (x *ListBucketObjectsResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[80] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketObjectsResponse.ProtoReflect.Descriptor instead. +func (*ListBucketObjectsResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{80} +} + +func (x *ListBucketObjectsResponse) GetObjects() []*ObjectInfo { + if x != nil { + return x.Objects + } + return nil +} + +func (x *ListBucketObjectsResponse) GetNextContinuationToken() string { + if x != nil { + return x.NextContinuationToken + } + return "" +} + +func (x *ListBucketObjectsResponse) GetIsTruncated() bool { + if x != nil { + return x.IsTruncated + } + return false +} + +func (x *ListBucketObjectsResponse) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +type BucketView struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Region string `protobuf:"bytes,3,opt,name=region,proto3" json:"region,omitempty"` + Owner string `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"` + Endpoint string `protobuf:"bytes,5,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + Versioning bool `protobuf:"varint,6,opt,name=versioning,proto3" json:"versioning,omitempty"` + ObjectCount int64 `protobuf:"varint,7,opt,name=object_count,json=objectCount,proto3" json:"object_count,omitempty"` + SizeBytes int64 `protobuf:"varint,8,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + Phase string `protobuf:"bytes,9,opt,name=phase,proto3" json:"phase,omitempty"` + LastError string `protobuf:"bytes,10,opt,name=last_error,json=lastError,proto3" json:"last_error,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BucketView) Reset() { + *x = BucketView{} + mi := &file_control_proto_msgTypes[81] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BucketView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketView) ProtoMessage() {} + +func (x *BucketView) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[81] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketView.ProtoReflect.Descriptor instead. +func (*BucketView) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{81} +} + +func (x *BucketView) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *BucketView) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *BucketView) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *BucketView) GetOwner() string { + if x != nil { + return x.Owner + } + return "" +} + +func (x *BucketView) GetEndpoint() string { + if x != nil { + return x.Endpoint + } + return "" +} + +func (x *BucketView) GetVersioning() bool { + if x != nil { + return x.Versioning + } + return false +} + +func (x *BucketView) GetObjectCount() int64 { + if x != nil { + return x.ObjectCount + } + return 0 +} + +func (x *BucketView) GetSizeBytes() int64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *BucketView) GetPhase() string { + if x != nil { + return x.Phase + } + return "" +} + +func (x *BucketView) GetLastError() string { + if x != nil { + return x.LastError + } + return "" +} + +func (x *BucketView) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *BucketView) GetUpdatedAt() *timestamppb.Timestamp { + if x != nil { + return x.UpdatedAt + } + return nil +} + +type BucketAccess struct { + state protoimpl.MessageState `protogen:"open.v1"` + Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` + Bucket string `protobuf:"bytes,3,opt,name=bucket,proto3" json:"bucket,omitempty"` + AccessKey string `protobuf:"bytes,4,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` + SecretKey string `protobuf:"bytes,5,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"` + VaultPath string `protobuf:"bytes,6,opt,name=vault_path,json=vaultPath,proto3" json:"vault_path,omitempty"` // when secrets live in Vault + S3Url string `protobuf:"bytes,7,opt,name=s3_url,json=s3Url,proto3" json:"s3_url,omitempty"` // e.g. s3://bucket + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BucketAccess) Reset() { + *x = BucketAccess{} + mi := &file_control_proto_msgTypes[82] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BucketAccess) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketAccess) ProtoMessage() {} + +func (x *BucketAccess) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[82] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketAccess.ProtoReflect.Descriptor instead. +func (*BucketAccess) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{82} +} + +func (x *BucketAccess) GetEndpoint() string { + if x != nil { + return x.Endpoint + } + return "" +} + +func (x *BucketAccess) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *BucketAccess) GetBucket() string { + if x != nil { + return x.Bucket + } + return "" +} + +func (x *BucketAccess) GetAccessKey() string { + if x != nil { + return x.AccessKey + } + return "" +} + +func (x *BucketAccess) GetSecretKey() string { + if x != nil { + return x.SecretKey + } + return "" +} + +func (x *BucketAccess) GetVaultPath() string { + if x != nil { + return x.VaultPath + } + return "" +} + +func (x *BucketAccess) GetS3Url() string { + if x != nil { + return x.S3Url + } + return "" +} + +type ObjectInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + SizeBytes int64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + LastModified string `protobuf:"bytes,4,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"` + StorageClass string `protobuf:"bytes,5,opt,name=storage_class,json=storageClass,proto3" json:"storage_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectInfo) Reset() { + *x = ObjectInfo{} + mi := &file_control_proto_msgTypes[83] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectInfo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectInfo) ProtoMessage() {} + +func (x *ObjectInfo) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[83] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectInfo.ProtoReflect.Descriptor instead. +func (*ObjectInfo) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{83} +} + +func (x *ObjectInfo) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *ObjectInfo) GetSizeBytes() int64 { + if x != nil { + return x.SizeBytes + } + return 0 +} + +func (x *ObjectInfo) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *ObjectInfo) GetLastModified() string { + if x != nil { + return x.LastModified + } + return "" +} + +func (x *ObjectInfo) GetStorageClass() string { + if x != nil { + return x.StorageClass + } + return "" +} + var File_control_proto protoreflect.FileDescriptor const file_control_proto_rawDesc = "" + @@ -5298,7 +6206,83 @@ const file_control_proto_rawDesc = "" + "\n" + "created_at\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + "\n" + - "updated_at\x18\x11 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt*\xda\x01\n" + + "updated_at\x18\x11 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\"a\n" + + "\x13CreateBucketRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x06region\x18\x02 \x01(\tR\x06region\x12\x1e\n" + + "\n" + + "versioning\x18\x03 \x01(\bR\n" + + "versioning\"\x86\x01\n" + + "\x14CreateBucketResponse\x125\n" + + "\x06bucket\x18\x01 \x01(\v2\x1d.persys.control.v1.BucketViewR\x06bucket\x127\n" + + "\x06access\x18\x02 \x01(\v2\x1f.persys.control.v1.BucketAccessR\x06access\"\x14\n" + + "\x12ListBucketsRequest\"N\n" + + "\x13ListBucketsResponse\x127\n" + + "\abuckets\x18\x01 \x03(\v2\x1d.persys.control.v1.BucketViewR\abuckets\"/\n" + + "\x10GetBucketRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\"J\n" + + "\x11GetBucketResponse\x125\n" + + "\x06bucket\x18\x01 \x01(\v2\x1d.persys.control.v1.BucketViewR\x06bucket\"H\n" + + "\x13DeleteBucketRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\x12\x14\n" + + "\x05force\x18\x02 \x01(\bR\x05force\"U\n" + + "\x14DeleteBucketResponse\x12\x18\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\x12#\n" + + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage\"5\n" + + "\x16GetBucketAccessRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\"R\n" + + "\x17GetBucketAccessResponse\x127\n" + + "\x06access\x18\x01 \x01(\v2\x1f.persys.control.v1.BucketAccessR\x06access\"\x99\x01\n" + + "\x18ListBucketObjectsRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\x12\x16\n" + + "\x06prefix\x18\x02 \x01(\tR\x06prefix\x12-\n" + + "\x12continuation_token\x18\x03 \x01(\tR\x11continuationToken\x12\x19\n" + + "\bmax_keys\x18\x04 \x01(\x05R\amaxKeys\"\xc7\x01\n" + + "\x19ListBucketObjectsResponse\x127\n" + + "\aobjects\x18\x01 \x03(\v2\x1d.persys.control.v1.ObjectInfoR\aobjects\x126\n" + + "\x17next_continuation_token\x18\x02 \x01(\tR\x15nextContinuationToken\x12!\n" + + "\fis_truncated\x18\x03 \x01(\bR\visTruncated\x12\x16\n" + + "\x06prefix\x18\x04 \x01(\tR\x06prefix\"\x87\x03\n" + + "\n" + + "BucketView\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x16\n" + + "\x06region\x18\x03 \x01(\tR\x06region\x12\x14\n" + + "\x05owner\x18\x04 \x01(\tR\x05owner\x12\x1a\n" + + "\bendpoint\x18\x05 \x01(\tR\bendpoint\x12\x1e\n" + + "\n" + + "versioning\x18\x06 \x01(\bR\n" + + "versioning\x12!\n" + + "\fobject_count\x18\a \x01(\x03R\vobjectCount\x12\x1d\n" + + "\n" + + "size_bytes\x18\b \x01(\x03R\tsizeBytes\x12\x14\n" + + "\x05phase\x18\t \x01(\tR\x05phase\x12\x1d\n" + + "\n" + + "last_error\x18\n" + + " \x01(\tR\tlastError\x129\n" + + "\n" + + "created_at\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + + "\n" + + "updated_at\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\"\xce\x01\n" + + "\fBucketAccess\x12\x1a\n" + + "\bendpoint\x18\x01 \x01(\tR\bendpoint\x12\x16\n" + + "\x06region\x18\x02 \x01(\tR\x06region\x12\x16\n" + + "\x06bucket\x18\x03 \x01(\tR\x06bucket\x12\x1d\n" + + "\n" + + "access_key\x18\x04 \x01(\tR\taccessKey\x12\x1d\n" + + "\n" + + "secret_key\x18\x05 \x01(\tR\tsecretKey\x12\x1d\n" + + "\n" + + "vault_path\x18\x06 \x01(\tR\tvaultPath\x12\x15\n" + + "\x06s3_url\x18\a \x01(\tR\x05s3Url\"\x9b\x01\n" + + "\n" + + "ObjectInfo\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x1d\n" + + "\n" + + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12#\n" + + "\rlast_modified\x18\x04 \x01(\tR\flastModified\x12#\n" + + "\rstorage_class\x18\x05 \x01(\tR\fstorageClass*\xda\x01\n" + "\x14AutomationActionType\x12&\n" + "\"AUTOMATION_ACTION_TYPE_UNSPECIFIED\x10\x00\x12'\n" + "#AUTOMATION_ACTION_SET_DESIRED_STATE\x10\x01\x12$\n" + @@ -5314,7 +6298,7 @@ const file_control_proto_rawDesc = "" + "\rRUNTIME_ERROR\x10\x05\x12\x11\n" + "\rNETWORK_ERROR\x10\x06\x12\x11\n" + "\rSTORAGE_ERROR\x10\a\x12\x12\n" + - "\x0eVM_BOOT_FAILED\x10\b2\x8a\x12\n" + + "\x0eVM_BOOT_FAILED\x10\b2\xdc\x16\n" + "\fAgentControl\x12_\n" + "\fRegisterNode\x12&.persys.control.v1.RegisterNodeRequest\x1a'.persys.control.v1.RegisterNodeResponse\x12V\n" + "\tHeartbeat\x12#.persys.control.v1.HeartbeatRequest\x1a$.persys.control.v1.HeartbeatResponse\x12b\n" + @@ -5342,7 +6326,13 @@ const file_control_proto_rawDesc = "" + "\tListDisks\x12#.persys.control.v1.ListDisksRequest\x1a$.persys.control.v1.ListDisksResponse\x12P\n" + "\aGetDisk\x12!.persys.control.v1.GetDiskRequest\x1a\".persys.control.v1.GetDiskResponse\x12Y\n" + "\n" + - "DeleteDisk\x12$.persys.control.v1.DeleteDiskRequest\x1a%.persys.control.v1.DeleteDiskResponseB7Z5github.com/persys-dev/persys/api/control/v1;controlv1b\x06proto3" + "DeleteDisk\x12$.persys.control.v1.DeleteDiskRequest\x1a%.persys.control.v1.DeleteDiskResponse\x12_\n" + + "\fCreateBucket\x12&.persys.control.v1.CreateBucketRequest\x1a'.persys.control.v1.CreateBucketResponse\x12\\\n" + + "\vListBuckets\x12%.persys.control.v1.ListBucketsRequest\x1a&.persys.control.v1.ListBucketsResponse\x12V\n" + + "\tGetBucket\x12#.persys.control.v1.GetBucketRequest\x1a$.persys.control.v1.GetBucketResponse\x12_\n" + + "\fDeleteBucket\x12&.persys.control.v1.DeleteBucketRequest\x1a'.persys.control.v1.DeleteBucketResponse\x12h\n" + + "\x0fGetBucketAccess\x12).persys.control.v1.GetBucketAccessRequest\x1a*.persys.control.v1.GetBucketAccessResponse\x12n\n" + + "\x11ListBucketObjects\x12+.persys.control.v1.ListBucketObjectsRequest\x1a,.persys.control.v1.ListBucketObjectsResponseB7Z5github.com/persys-dev/persys/api/control/v1;controlv1b\x06proto3" var ( file_control_proto_rawDescOnce sync.Once @@ -5357,7 +6347,7 @@ func file_control_proto_rawDescGZIP() []byte { } var file_control_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_control_proto_msgTypes = make([]protoimpl.MessageInfo, 75) +var file_control_proto_msgTypes = make([]protoimpl.MessageInfo, 90) var file_control_proto_goTypes = []any{ (AutomationActionType)(0), // 0: persys.control.v1.AutomationActionType (FailureReason)(0), // 1: persys.control.v1.FailureReason @@ -5430,138 +6420,173 @@ var file_control_proto_goTypes = []any{ (*DeleteDiskRequest)(nil), // 68: persys.control.v1.DeleteDiskRequest (*DeleteDiskResponse)(nil), // 69: persys.control.v1.DeleteDiskResponse (*DiskView)(nil), // 70: persys.control.v1.DiskView - nil, // 71: persys.control.v1.RegisterNodeRequest.LabelsEntry - nil, // 72: persys.control.v1.WorkloadSpec.MetadataEntry - nil, // 73: persys.control.v1.ContainerSpec.EnvEntry - nil, // 74: persys.control.v1.ComposeSpec.EnvEntry - nil, // 75: persys.control.v1.NodeView.LabelsEntry - nil, // 76: persys.control.v1.SchedulerEventView.DetailsEntry - (*timestamppb.Timestamp)(nil), // 77: google.protobuf.Timestamp + (*CreateBucketRequest)(nil), // 71: persys.control.v1.CreateBucketRequest + (*CreateBucketResponse)(nil), // 72: persys.control.v1.CreateBucketResponse + (*ListBucketsRequest)(nil), // 73: persys.control.v1.ListBucketsRequest + (*ListBucketsResponse)(nil), // 74: persys.control.v1.ListBucketsResponse + (*GetBucketRequest)(nil), // 75: persys.control.v1.GetBucketRequest + (*GetBucketResponse)(nil), // 76: persys.control.v1.GetBucketResponse + (*DeleteBucketRequest)(nil), // 77: persys.control.v1.DeleteBucketRequest + (*DeleteBucketResponse)(nil), // 78: persys.control.v1.DeleteBucketResponse + (*GetBucketAccessRequest)(nil), // 79: persys.control.v1.GetBucketAccessRequest + (*GetBucketAccessResponse)(nil), // 80: persys.control.v1.GetBucketAccessResponse + (*ListBucketObjectsRequest)(nil), // 81: persys.control.v1.ListBucketObjectsRequest + (*ListBucketObjectsResponse)(nil), // 82: persys.control.v1.ListBucketObjectsResponse + (*BucketView)(nil), // 83: persys.control.v1.BucketView + (*BucketAccess)(nil), // 84: persys.control.v1.BucketAccess + (*ObjectInfo)(nil), // 85: persys.control.v1.ObjectInfo + nil, // 86: persys.control.v1.RegisterNodeRequest.LabelsEntry + nil, // 87: persys.control.v1.WorkloadSpec.MetadataEntry + nil, // 88: persys.control.v1.ContainerSpec.EnvEntry + nil, // 89: persys.control.v1.ComposeSpec.EnvEntry + nil, // 90: persys.control.v1.NodeView.LabelsEntry + nil, // 91: persys.control.v1.SchedulerEventView.DetailsEntry + (*timestamppb.Timestamp)(nil), // 92: google.protobuf.Timestamp } var file_control_proto_depIdxs = []int32{ - 0, // 0: persys.control.v1.AutomationSuggestion.action_type:type_name -> persys.control.v1.AutomationActionType - 77, // 1: persys.control.v1.AutomationSuggestion.suggested_at:type_name -> google.protobuf.Timestamp - 2, // 2: persys.control.v1.SubmitAutomationSuggestionRequest.suggestion:type_name -> persys.control.v1.AutomationSuggestion - 77, // 3: persys.control.v1.SubmitAutomationSuggestionResponse.decided_at:type_name -> google.protobuf.Timestamp - 6, // 4: persys.control.v1.RegisterNodeRequest.capabilities:type_name -> persys.control.v1.NodeCapabilities - 71, // 5: persys.control.v1.RegisterNodeRequest.labels:type_name -> persys.control.v1.RegisterNodeRequest.LabelsEntry - 77, // 6: persys.control.v1.RegisterNodeRequest.timestamp:type_name -> google.protobuf.Timestamp - 7, // 7: persys.control.v1.NodeCapabilities.storage_pools:type_name -> persys.control.v1.StoragePool - 77, // 8: persys.control.v1.RegisterNodeResponse.lease_expires_at:type_name -> google.protobuf.Timestamp - 10, // 9: persys.control.v1.HeartbeatRequest.usage:type_name -> persys.control.v1.NodeUsage - 29, // 10: persys.control.v1.HeartbeatRequest.workload_statuses:type_name -> persys.control.v1.WorkloadStatus - 77, // 11: persys.control.v1.HeartbeatRequest.timestamp:type_name -> google.protobuf.Timestamp - 27, // 12: persys.control.v1.HeartbeatRequest.workload_usage:type_name -> persys.control.v1.WorkloadUsageSnapshot - 77, // 13: persys.control.v1.HeartbeatResponse.lease_expires_at:type_name -> google.protobuf.Timestamp - 16, // 14: persys.control.v1.ApplyWorkloadRequest.spec:type_name -> persys.control.v1.WorkloadSpec - 1, // 15: persys.control.v1.ApplyWorkloadResponse.failure_reason:type_name -> persys.control.v1.FailureReason - 17, // 16: persys.control.v1.WorkloadSpec.resources:type_name -> persys.control.v1.ResourceRequirements - 18, // 17: persys.control.v1.WorkloadSpec.container:type_name -> persys.control.v1.ContainerSpec - 21, // 18: persys.control.v1.WorkloadSpec.compose:type_name -> persys.control.v1.ComposeSpec - 22, // 19: persys.control.v1.WorkloadSpec.vm:type_name -> persys.control.v1.VMSpec - 72, // 20: persys.control.v1.WorkloadSpec.metadata:type_name -> persys.control.v1.WorkloadSpec.MetadataEntry - 73, // 21: persys.control.v1.ContainerSpec.env:type_name -> persys.control.v1.ContainerSpec.EnvEntry - 19, // 22: persys.control.v1.ContainerSpec.volumes:type_name -> persys.control.v1.VolumeMount - 20, // 23: persys.control.v1.ContainerSpec.ports:type_name -> persys.control.v1.Port - 26, // 24: persys.control.v1.ContainerSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec - 74, // 25: persys.control.v1.ComposeSpec.env:type_name -> persys.control.v1.ComposeSpec.EnvEntry - 23, // 26: persys.control.v1.VMSpec.disks:type_name -> persys.control.v1.DiskConfig - 24, // 27: persys.control.v1.VMSpec.networks:type_name -> persys.control.v1.NetworkConfig - 25, // 28: persys.control.v1.VMSpec.cloud_init:type_name -> persys.control.v1.CloudInitConfig - 26, // 29: persys.control.v1.VMSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec - 77, // 30: persys.control.v1.WorkloadUsageSnapshot.collected_at:type_name -> google.protobuf.Timestamp - 77, // 31: persys.control.v1.ReasonDetail.last_transition:type_name -> google.protobuf.Timestamp - 77, // 32: persys.control.v1.ReasonDetail.next_retry_at:type_name -> google.protobuf.Timestamp - 1, // 33: persys.control.v1.WorkloadStatus.failure_reason:type_name -> persys.control.v1.FailureReason - 77, // 34: persys.control.v1.WorkloadStatus.last_transition:type_name -> google.protobuf.Timestamp - 28, // 35: persys.control.v1.WorkloadStatus.reason:type_name -> persys.control.v1.ReasonDetail - 27, // 36: persys.control.v1.WorkloadStatus.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot - 49, // 37: persys.control.v1.DrainNodeResponse.node:type_name -> persys.control.v1.NodeView - 49, // 38: persys.control.v1.UndrainNodeResponse.node:type_name -> persys.control.v1.NodeView - 36, // 39: persys.control.v1.TaintNodeRequest.taint:type_name -> persys.control.v1.NodeTaint - 49, // 40: persys.control.v1.TaintNodeResponse.node:type_name -> persys.control.v1.NodeView - 49, // 41: persys.control.v1.UntaintNodeResponse.node:type_name -> persys.control.v1.NodeView - 49, // 42: persys.control.v1.SetNodeLabelResponse.node:type_name -> persys.control.v1.NodeView - 49, // 43: persys.control.v1.DeleteNodeLabelResponse.node:type_name -> persys.control.v1.NodeView - 49, // 44: persys.control.v1.ListNodesResponse.nodes:type_name -> persys.control.v1.NodeView - 49, // 45: persys.control.v1.GetNodeResponse.node:type_name -> persys.control.v1.NodeView - 77, // 46: persys.control.v1.NodeView.status_updated_at:type_name -> google.protobuf.Timestamp - 77, // 47: persys.control.v1.NodeView.last_heartbeat:type_name -> google.protobuf.Timestamp - 75, // 48: persys.control.v1.NodeView.labels:type_name -> persys.control.v1.NodeView.LabelsEntry - 36, // 49: persys.control.v1.NodeView.taints:type_name -> persys.control.v1.NodeTaint - 77, // 50: persys.control.v1.SchedulerEventView.timestamp:type_name -> google.protobuf.Timestamp - 76, // 51: persys.control.v1.SchedulerEventView.details:type_name -> persys.control.v1.SchedulerEventView.DetailsEntry - 51, // 52: persys.control.v1.ListEventsResponse.events:type_name -> persys.control.v1.SchedulerEventView - 58, // 53: persys.control.v1.ListWorkloadsResponse.workloads:type_name -> persys.control.v1.WorkloadView - 58, // 54: persys.control.v1.GetWorkloadResponse.workload:type_name -> persys.control.v1.WorkloadView - 77, // 55: persys.control.v1.WorkloadView.retry_next_at:type_name -> google.protobuf.Timestamp - 77, // 56: persys.control.v1.WorkloadView.last_updated:type_name -> google.protobuf.Timestamp - 28, // 57: persys.control.v1.WorkloadView.reason:type_name -> persys.control.v1.ReasonDetail - 27, // 58: persys.control.v1.WorkloadView.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot - 77, // 59: persys.control.v1.WorkloadView.created_at:type_name -> google.protobuf.Timestamp - 77, // 60: persys.control.v1.GetClusterSummaryResponse.generated_at:type_name -> google.protobuf.Timestamp - 5, // 61: persys.control.v1.ControlMessage.register:type_name -> persys.control.v1.RegisterNodeRequest - 9, // 62: persys.control.v1.ControlMessage.heartbeat:type_name -> persys.control.v1.HeartbeatRequest - 12, // 63: persys.control.v1.ControlMessage.apply:type_name -> persys.control.v1.ApplyWorkloadRequest - 14, // 64: persys.control.v1.ControlMessage.delete:type_name -> persys.control.v1.DeleteWorkloadRequest - 70, // 65: persys.control.v1.CreateDiskResponse.disk:type_name -> persys.control.v1.DiskView - 70, // 66: persys.control.v1.ListDisksResponse.disks:type_name -> persys.control.v1.DiskView - 70, // 67: persys.control.v1.GetDiskResponse.disk:type_name -> persys.control.v1.DiskView - 77, // 68: persys.control.v1.DiskView.created_at:type_name -> google.protobuf.Timestamp - 77, // 69: persys.control.v1.DiskView.updated_at:type_name -> google.protobuf.Timestamp - 5, // 70: persys.control.v1.AgentControl.RegisterNode:input_type -> persys.control.v1.RegisterNodeRequest - 9, // 71: persys.control.v1.AgentControl.Heartbeat:input_type -> persys.control.v1.HeartbeatRequest - 12, // 72: persys.control.v1.AgentControl.ApplyWorkload:input_type -> persys.control.v1.ApplyWorkloadRequest - 14, // 73: persys.control.v1.AgentControl.DeleteWorkload:input_type -> persys.control.v1.DeleteWorkloadRequest - 30, // 74: persys.control.v1.AgentControl.RetryWorkload:input_type -> persys.control.v1.RetryWorkloadRequest - 32, // 75: persys.control.v1.AgentControl.DrainNode:input_type -> persys.control.v1.DrainNodeRequest - 34, // 76: persys.control.v1.AgentControl.UndrainNode:input_type -> persys.control.v1.UndrainNodeRequest - 37, // 77: persys.control.v1.AgentControl.TaintNode:input_type -> persys.control.v1.TaintNodeRequest - 39, // 78: persys.control.v1.AgentControl.UntaintNode:input_type -> persys.control.v1.UntaintNodeRequest - 41, // 79: persys.control.v1.AgentControl.SetNodeLabel:input_type -> persys.control.v1.SetNodeLabelRequest - 43, // 80: persys.control.v1.AgentControl.DeleteNodeLabel:input_type -> persys.control.v1.DeleteNodeLabelRequest - 3, // 81: persys.control.v1.AgentControl.SubmitAutomationSuggestion:input_type -> persys.control.v1.SubmitAutomationSuggestionRequest - 45, // 82: persys.control.v1.AgentControl.ListNodes:input_type -> persys.control.v1.ListNodesRequest - 46, // 83: persys.control.v1.AgentControl.GetNode:input_type -> persys.control.v1.GetNodeRequest - 50, // 84: persys.control.v1.AgentControl.ListWorkloads:input_type -> persys.control.v1.ListWorkloadsRequest - 55, // 85: persys.control.v1.AgentControl.GetWorkload:input_type -> persys.control.v1.GetWorkloadRequest - 59, // 86: persys.control.v1.AgentControl.GetClusterSummary:input_type -> persys.control.v1.GetClusterSummaryRequest - 52, // 87: persys.control.v1.AgentControl.ListEvents:input_type -> persys.control.v1.ListEventsRequest - 54, // 88: persys.control.v1.AgentControl.WatchEvents:input_type -> persys.control.v1.WatchEventsRequest - 61, // 89: persys.control.v1.AgentControl.ControlStream:input_type -> persys.control.v1.ControlMessage - 62, // 90: persys.control.v1.AgentControl.CreateDisk:input_type -> persys.control.v1.CreateDiskRequest - 64, // 91: persys.control.v1.AgentControl.ListDisks:input_type -> persys.control.v1.ListDisksRequest - 66, // 92: persys.control.v1.AgentControl.GetDisk:input_type -> persys.control.v1.GetDiskRequest - 68, // 93: persys.control.v1.AgentControl.DeleteDisk:input_type -> persys.control.v1.DeleteDiskRequest - 8, // 94: persys.control.v1.AgentControl.RegisterNode:output_type -> persys.control.v1.RegisterNodeResponse - 11, // 95: persys.control.v1.AgentControl.Heartbeat:output_type -> persys.control.v1.HeartbeatResponse - 13, // 96: persys.control.v1.AgentControl.ApplyWorkload:output_type -> persys.control.v1.ApplyWorkloadResponse - 15, // 97: persys.control.v1.AgentControl.DeleteWorkload:output_type -> persys.control.v1.DeleteWorkloadResponse - 31, // 98: persys.control.v1.AgentControl.RetryWorkload:output_type -> persys.control.v1.RetryWorkloadResponse - 33, // 99: persys.control.v1.AgentControl.DrainNode:output_type -> persys.control.v1.DrainNodeResponse - 35, // 100: persys.control.v1.AgentControl.UndrainNode:output_type -> persys.control.v1.UndrainNodeResponse - 38, // 101: persys.control.v1.AgentControl.TaintNode:output_type -> persys.control.v1.TaintNodeResponse - 40, // 102: persys.control.v1.AgentControl.UntaintNode:output_type -> persys.control.v1.UntaintNodeResponse - 42, // 103: persys.control.v1.AgentControl.SetNodeLabel:output_type -> persys.control.v1.SetNodeLabelResponse - 44, // 104: persys.control.v1.AgentControl.DeleteNodeLabel:output_type -> persys.control.v1.DeleteNodeLabelResponse - 4, // 105: persys.control.v1.AgentControl.SubmitAutomationSuggestion:output_type -> persys.control.v1.SubmitAutomationSuggestionResponse - 47, // 106: persys.control.v1.AgentControl.ListNodes:output_type -> persys.control.v1.ListNodesResponse - 48, // 107: persys.control.v1.AgentControl.GetNode:output_type -> persys.control.v1.GetNodeResponse - 56, // 108: persys.control.v1.AgentControl.ListWorkloads:output_type -> persys.control.v1.ListWorkloadsResponse - 57, // 109: persys.control.v1.AgentControl.GetWorkload:output_type -> persys.control.v1.GetWorkloadResponse - 60, // 110: persys.control.v1.AgentControl.GetClusterSummary:output_type -> persys.control.v1.GetClusterSummaryResponse - 53, // 111: persys.control.v1.AgentControl.ListEvents:output_type -> persys.control.v1.ListEventsResponse - 51, // 112: persys.control.v1.AgentControl.WatchEvents:output_type -> persys.control.v1.SchedulerEventView - 61, // 113: persys.control.v1.AgentControl.ControlStream:output_type -> persys.control.v1.ControlMessage - 63, // 114: persys.control.v1.AgentControl.CreateDisk:output_type -> persys.control.v1.CreateDiskResponse - 65, // 115: persys.control.v1.AgentControl.ListDisks:output_type -> persys.control.v1.ListDisksResponse - 67, // 116: persys.control.v1.AgentControl.GetDisk:output_type -> persys.control.v1.GetDiskResponse - 69, // 117: persys.control.v1.AgentControl.DeleteDisk:output_type -> persys.control.v1.DeleteDiskResponse - 94, // [94:118] is the sub-list for method output_type - 70, // [70:94] is the sub-list for method input_type - 70, // [70:70] is the sub-list for extension type_name - 70, // [70:70] is the sub-list for extension extendee - 0, // [0:70] is the sub-list for field type_name + 0, // 0: persys.control.v1.AutomationSuggestion.action_type:type_name -> persys.control.v1.AutomationActionType + 92, // 1: persys.control.v1.AutomationSuggestion.suggested_at:type_name -> google.protobuf.Timestamp + 2, // 2: persys.control.v1.SubmitAutomationSuggestionRequest.suggestion:type_name -> persys.control.v1.AutomationSuggestion + 92, // 3: persys.control.v1.SubmitAutomationSuggestionResponse.decided_at:type_name -> google.protobuf.Timestamp + 6, // 4: persys.control.v1.RegisterNodeRequest.capabilities:type_name -> persys.control.v1.NodeCapabilities + 86, // 5: persys.control.v1.RegisterNodeRequest.labels:type_name -> persys.control.v1.RegisterNodeRequest.LabelsEntry + 92, // 6: persys.control.v1.RegisterNodeRequest.timestamp:type_name -> google.protobuf.Timestamp + 7, // 7: persys.control.v1.NodeCapabilities.storage_pools:type_name -> persys.control.v1.StoragePool + 92, // 8: persys.control.v1.RegisterNodeResponse.lease_expires_at:type_name -> google.protobuf.Timestamp + 10, // 9: persys.control.v1.HeartbeatRequest.usage:type_name -> persys.control.v1.NodeUsage + 29, // 10: persys.control.v1.HeartbeatRequest.workload_statuses:type_name -> persys.control.v1.WorkloadStatus + 92, // 11: persys.control.v1.HeartbeatRequest.timestamp:type_name -> google.protobuf.Timestamp + 27, // 12: persys.control.v1.HeartbeatRequest.workload_usage:type_name -> persys.control.v1.WorkloadUsageSnapshot + 92, // 13: persys.control.v1.HeartbeatResponse.lease_expires_at:type_name -> google.protobuf.Timestamp + 16, // 14: persys.control.v1.ApplyWorkloadRequest.spec:type_name -> persys.control.v1.WorkloadSpec + 1, // 15: persys.control.v1.ApplyWorkloadResponse.failure_reason:type_name -> persys.control.v1.FailureReason + 17, // 16: persys.control.v1.WorkloadSpec.resources:type_name -> persys.control.v1.ResourceRequirements + 18, // 17: persys.control.v1.WorkloadSpec.container:type_name -> persys.control.v1.ContainerSpec + 21, // 18: persys.control.v1.WorkloadSpec.compose:type_name -> persys.control.v1.ComposeSpec + 22, // 19: persys.control.v1.WorkloadSpec.vm:type_name -> persys.control.v1.VMSpec + 87, // 20: persys.control.v1.WorkloadSpec.metadata:type_name -> persys.control.v1.WorkloadSpec.MetadataEntry + 88, // 21: persys.control.v1.ContainerSpec.env:type_name -> persys.control.v1.ContainerSpec.EnvEntry + 19, // 22: persys.control.v1.ContainerSpec.volumes:type_name -> persys.control.v1.VolumeMount + 20, // 23: persys.control.v1.ContainerSpec.ports:type_name -> persys.control.v1.Port + 26, // 24: persys.control.v1.ContainerSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec + 89, // 25: persys.control.v1.ComposeSpec.env:type_name -> persys.control.v1.ComposeSpec.EnvEntry + 23, // 26: persys.control.v1.VMSpec.disks:type_name -> persys.control.v1.DiskConfig + 24, // 27: persys.control.v1.VMSpec.networks:type_name -> persys.control.v1.NetworkConfig + 25, // 28: persys.control.v1.VMSpec.cloud_init:type_name -> persys.control.v1.CloudInitConfig + 26, // 29: persys.control.v1.VMSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec + 92, // 30: persys.control.v1.WorkloadUsageSnapshot.collected_at:type_name -> google.protobuf.Timestamp + 92, // 31: persys.control.v1.ReasonDetail.last_transition:type_name -> google.protobuf.Timestamp + 92, // 32: persys.control.v1.ReasonDetail.next_retry_at:type_name -> google.protobuf.Timestamp + 1, // 33: persys.control.v1.WorkloadStatus.failure_reason:type_name -> persys.control.v1.FailureReason + 92, // 34: persys.control.v1.WorkloadStatus.last_transition:type_name -> google.protobuf.Timestamp + 28, // 35: persys.control.v1.WorkloadStatus.reason:type_name -> persys.control.v1.ReasonDetail + 27, // 36: persys.control.v1.WorkloadStatus.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot + 49, // 37: persys.control.v1.DrainNodeResponse.node:type_name -> persys.control.v1.NodeView + 49, // 38: persys.control.v1.UndrainNodeResponse.node:type_name -> persys.control.v1.NodeView + 36, // 39: persys.control.v1.TaintNodeRequest.taint:type_name -> persys.control.v1.NodeTaint + 49, // 40: persys.control.v1.TaintNodeResponse.node:type_name -> persys.control.v1.NodeView + 49, // 41: persys.control.v1.UntaintNodeResponse.node:type_name -> persys.control.v1.NodeView + 49, // 42: persys.control.v1.SetNodeLabelResponse.node:type_name -> persys.control.v1.NodeView + 49, // 43: persys.control.v1.DeleteNodeLabelResponse.node:type_name -> persys.control.v1.NodeView + 49, // 44: persys.control.v1.ListNodesResponse.nodes:type_name -> persys.control.v1.NodeView + 49, // 45: persys.control.v1.GetNodeResponse.node:type_name -> persys.control.v1.NodeView + 92, // 46: persys.control.v1.NodeView.status_updated_at:type_name -> google.protobuf.Timestamp + 92, // 47: persys.control.v1.NodeView.last_heartbeat:type_name -> google.protobuf.Timestamp + 90, // 48: persys.control.v1.NodeView.labels:type_name -> persys.control.v1.NodeView.LabelsEntry + 36, // 49: persys.control.v1.NodeView.taints:type_name -> persys.control.v1.NodeTaint + 92, // 50: persys.control.v1.SchedulerEventView.timestamp:type_name -> google.protobuf.Timestamp + 91, // 51: persys.control.v1.SchedulerEventView.details:type_name -> persys.control.v1.SchedulerEventView.DetailsEntry + 51, // 52: persys.control.v1.ListEventsResponse.events:type_name -> persys.control.v1.SchedulerEventView + 58, // 53: persys.control.v1.ListWorkloadsResponse.workloads:type_name -> persys.control.v1.WorkloadView + 58, // 54: persys.control.v1.GetWorkloadResponse.workload:type_name -> persys.control.v1.WorkloadView + 92, // 55: persys.control.v1.WorkloadView.retry_next_at:type_name -> google.protobuf.Timestamp + 92, // 56: persys.control.v1.WorkloadView.last_updated:type_name -> google.protobuf.Timestamp + 28, // 57: persys.control.v1.WorkloadView.reason:type_name -> persys.control.v1.ReasonDetail + 27, // 58: persys.control.v1.WorkloadView.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot + 92, // 59: persys.control.v1.WorkloadView.created_at:type_name -> google.protobuf.Timestamp + 92, // 60: persys.control.v1.GetClusterSummaryResponse.generated_at:type_name -> google.protobuf.Timestamp + 5, // 61: persys.control.v1.ControlMessage.register:type_name -> persys.control.v1.RegisterNodeRequest + 9, // 62: persys.control.v1.ControlMessage.heartbeat:type_name -> persys.control.v1.HeartbeatRequest + 12, // 63: persys.control.v1.ControlMessage.apply:type_name -> persys.control.v1.ApplyWorkloadRequest + 14, // 64: persys.control.v1.ControlMessage.delete:type_name -> persys.control.v1.DeleteWorkloadRequest + 70, // 65: persys.control.v1.CreateDiskResponse.disk:type_name -> persys.control.v1.DiskView + 70, // 66: persys.control.v1.ListDisksResponse.disks:type_name -> persys.control.v1.DiskView + 70, // 67: persys.control.v1.GetDiskResponse.disk:type_name -> persys.control.v1.DiskView + 92, // 68: persys.control.v1.DiskView.created_at:type_name -> google.protobuf.Timestamp + 92, // 69: persys.control.v1.DiskView.updated_at:type_name -> google.protobuf.Timestamp + 83, // 70: persys.control.v1.CreateBucketResponse.bucket:type_name -> persys.control.v1.BucketView + 84, // 71: persys.control.v1.CreateBucketResponse.access:type_name -> persys.control.v1.BucketAccess + 83, // 72: persys.control.v1.ListBucketsResponse.buckets:type_name -> persys.control.v1.BucketView + 83, // 73: persys.control.v1.GetBucketResponse.bucket:type_name -> persys.control.v1.BucketView + 84, // 74: persys.control.v1.GetBucketAccessResponse.access:type_name -> persys.control.v1.BucketAccess + 85, // 75: persys.control.v1.ListBucketObjectsResponse.objects:type_name -> persys.control.v1.ObjectInfo + 92, // 76: persys.control.v1.BucketView.created_at:type_name -> google.protobuf.Timestamp + 92, // 77: persys.control.v1.BucketView.updated_at:type_name -> google.protobuf.Timestamp + 5, // 78: persys.control.v1.AgentControl.RegisterNode:input_type -> persys.control.v1.RegisterNodeRequest + 9, // 79: persys.control.v1.AgentControl.Heartbeat:input_type -> persys.control.v1.HeartbeatRequest + 12, // 80: persys.control.v1.AgentControl.ApplyWorkload:input_type -> persys.control.v1.ApplyWorkloadRequest + 14, // 81: persys.control.v1.AgentControl.DeleteWorkload:input_type -> persys.control.v1.DeleteWorkloadRequest + 30, // 82: persys.control.v1.AgentControl.RetryWorkload:input_type -> persys.control.v1.RetryWorkloadRequest + 32, // 83: persys.control.v1.AgentControl.DrainNode:input_type -> persys.control.v1.DrainNodeRequest + 34, // 84: persys.control.v1.AgentControl.UndrainNode:input_type -> persys.control.v1.UndrainNodeRequest + 37, // 85: persys.control.v1.AgentControl.TaintNode:input_type -> persys.control.v1.TaintNodeRequest + 39, // 86: persys.control.v1.AgentControl.UntaintNode:input_type -> persys.control.v1.UntaintNodeRequest + 41, // 87: persys.control.v1.AgentControl.SetNodeLabel:input_type -> persys.control.v1.SetNodeLabelRequest + 43, // 88: persys.control.v1.AgentControl.DeleteNodeLabel:input_type -> persys.control.v1.DeleteNodeLabelRequest + 3, // 89: persys.control.v1.AgentControl.SubmitAutomationSuggestion:input_type -> persys.control.v1.SubmitAutomationSuggestionRequest + 45, // 90: persys.control.v1.AgentControl.ListNodes:input_type -> persys.control.v1.ListNodesRequest + 46, // 91: persys.control.v1.AgentControl.GetNode:input_type -> persys.control.v1.GetNodeRequest + 50, // 92: persys.control.v1.AgentControl.ListWorkloads:input_type -> persys.control.v1.ListWorkloadsRequest + 55, // 93: persys.control.v1.AgentControl.GetWorkload:input_type -> persys.control.v1.GetWorkloadRequest + 59, // 94: persys.control.v1.AgentControl.GetClusterSummary:input_type -> persys.control.v1.GetClusterSummaryRequest + 52, // 95: persys.control.v1.AgentControl.ListEvents:input_type -> persys.control.v1.ListEventsRequest + 54, // 96: persys.control.v1.AgentControl.WatchEvents:input_type -> persys.control.v1.WatchEventsRequest + 61, // 97: persys.control.v1.AgentControl.ControlStream:input_type -> persys.control.v1.ControlMessage + 62, // 98: persys.control.v1.AgentControl.CreateDisk:input_type -> persys.control.v1.CreateDiskRequest + 64, // 99: persys.control.v1.AgentControl.ListDisks:input_type -> persys.control.v1.ListDisksRequest + 66, // 100: persys.control.v1.AgentControl.GetDisk:input_type -> persys.control.v1.GetDiskRequest + 68, // 101: persys.control.v1.AgentControl.DeleteDisk:input_type -> persys.control.v1.DeleteDiskRequest + 71, // 102: persys.control.v1.AgentControl.CreateBucket:input_type -> persys.control.v1.CreateBucketRequest + 73, // 103: persys.control.v1.AgentControl.ListBuckets:input_type -> persys.control.v1.ListBucketsRequest + 75, // 104: persys.control.v1.AgentControl.GetBucket:input_type -> persys.control.v1.GetBucketRequest + 77, // 105: persys.control.v1.AgentControl.DeleteBucket:input_type -> persys.control.v1.DeleteBucketRequest + 79, // 106: persys.control.v1.AgentControl.GetBucketAccess:input_type -> persys.control.v1.GetBucketAccessRequest + 81, // 107: persys.control.v1.AgentControl.ListBucketObjects:input_type -> persys.control.v1.ListBucketObjectsRequest + 8, // 108: persys.control.v1.AgentControl.RegisterNode:output_type -> persys.control.v1.RegisterNodeResponse + 11, // 109: persys.control.v1.AgentControl.Heartbeat:output_type -> persys.control.v1.HeartbeatResponse + 13, // 110: persys.control.v1.AgentControl.ApplyWorkload:output_type -> persys.control.v1.ApplyWorkloadResponse + 15, // 111: persys.control.v1.AgentControl.DeleteWorkload:output_type -> persys.control.v1.DeleteWorkloadResponse + 31, // 112: persys.control.v1.AgentControl.RetryWorkload:output_type -> persys.control.v1.RetryWorkloadResponse + 33, // 113: persys.control.v1.AgentControl.DrainNode:output_type -> persys.control.v1.DrainNodeResponse + 35, // 114: persys.control.v1.AgentControl.UndrainNode:output_type -> persys.control.v1.UndrainNodeResponse + 38, // 115: persys.control.v1.AgentControl.TaintNode:output_type -> persys.control.v1.TaintNodeResponse + 40, // 116: persys.control.v1.AgentControl.UntaintNode:output_type -> persys.control.v1.UntaintNodeResponse + 42, // 117: persys.control.v1.AgentControl.SetNodeLabel:output_type -> persys.control.v1.SetNodeLabelResponse + 44, // 118: persys.control.v1.AgentControl.DeleteNodeLabel:output_type -> persys.control.v1.DeleteNodeLabelResponse + 4, // 119: persys.control.v1.AgentControl.SubmitAutomationSuggestion:output_type -> persys.control.v1.SubmitAutomationSuggestionResponse + 47, // 120: persys.control.v1.AgentControl.ListNodes:output_type -> persys.control.v1.ListNodesResponse + 48, // 121: persys.control.v1.AgentControl.GetNode:output_type -> persys.control.v1.GetNodeResponse + 56, // 122: persys.control.v1.AgentControl.ListWorkloads:output_type -> persys.control.v1.ListWorkloadsResponse + 57, // 123: persys.control.v1.AgentControl.GetWorkload:output_type -> persys.control.v1.GetWorkloadResponse + 60, // 124: persys.control.v1.AgentControl.GetClusterSummary:output_type -> persys.control.v1.GetClusterSummaryResponse + 53, // 125: persys.control.v1.AgentControl.ListEvents:output_type -> persys.control.v1.ListEventsResponse + 51, // 126: persys.control.v1.AgentControl.WatchEvents:output_type -> persys.control.v1.SchedulerEventView + 61, // 127: persys.control.v1.AgentControl.ControlStream:output_type -> persys.control.v1.ControlMessage + 63, // 128: persys.control.v1.AgentControl.CreateDisk:output_type -> persys.control.v1.CreateDiskResponse + 65, // 129: persys.control.v1.AgentControl.ListDisks:output_type -> persys.control.v1.ListDisksResponse + 67, // 130: persys.control.v1.AgentControl.GetDisk:output_type -> persys.control.v1.GetDiskResponse + 69, // 131: persys.control.v1.AgentControl.DeleteDisk:output_type -> persys.control.v1.DeleteDiskResponse + 72, // 132: persys.control.v1.AgentControl.CreateBucket:output_type -> persys.control.v1.CreateBucketResponse + 74, // 133: persys.control.v1.AgentControl.ListBuckets:output_type -> persys.control.v1.ListBucketsResponse + 76, // 134: persys.control.v1.AgentControl.GetBucket:output_type -> persys.control.v1.GetBucketResponse + 78, // 135: persys.control.v1.AgentControl.DeleteBucket:output_type -> persys.control.v1.DeleteBucketResponse + 80, // 136: persys.control.v1.AgentControl.GetBucketAccess:output_type -> persys.control.v1.GetBucketAccessResponse + 82, // 137: persys.control.v1.AgentControl.ListBucketObjects:output_type -> persys.control.v1.ListBucketObjectsResponse + 108, // [108:138] is the sub-list for method output_type + 78, // [78:108] is the sub-list for method input_type + 78, // [78:78] is the sub-list for extension type_name + 78, // [78:78] is the sub-list for extension extendee + 0, // [0:78] is the sub-list for field type_name } func init() { file_control_proto_init() } @@ -5586,7 +6611,7 @@ func file_control_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_control_proto_rawDesc), len(file_control_proto_rawDesc)), NumEnums: 2, - NumMessages: 75, + NumMessages: 90, NumExtensions: 0, NumServices: 1, }, diff --git a/pkg/agent/control/v1/control_grpc.pb.go b/pkg/agent/control/v1/control_grpc.pb.go index 9d6f739..1ccea93 100644 --- a/pkg/agent/control/v1/control_grpc.pb.go +++ b/pkg/agent/control/v1/control_grpc.pb.go @@ -43,6 +43,12 @@ const ( AgentControl_ListDisks_FullMethodName = "/persys.control.v1.AgentControl/ListDisks" AgentControl_GetDisk_FullMethodName = "/persys.control.v1.AgentControl/GetDisk" AgentControl_DeleteDisk_FullMethodName = "/persys.control.v1.AgentControl/DeleteDisk" + AgentControl_CreateBucket_FullMethodName = "/persys.control.v1.AgentControl/CreateBucket" + AgentControl_ListBuckets_FullMethodName = "/persys.control.v1.AgentControl/ListBuckets" + AgentControl_GetBucket_FullMethodName = "/persys.control.v1.AgentControl/GetBucket" + AgentControl_DeleteBucket_FullMethodName = "/persys.control.v1.AgentControl/DeleteBucket" + AgentControl_GetBucketAccess_FullMethodName = "/persys.control.v1.AgentControl/GetBucketAccess" + AgentControl_ListBucketObjects_FullMethodName = "/persys.control.v1.AgentControl/ListBucketObjects" ) // AgentControlClient is the client API for AgentControl service. @@ -91,6 +97,13 @@ type AgentControlClient interface { ListDisks(ctx context.Context, in *ListDisksRequest, opts ...grpc.CallOption) (*ListDisksResponse, error) GetDisk(ctx context.Context, in *GetDiskRequest, opts ...grpc.CallOption) (*GetDiskResponse, error) DeleteDisk(ctx context.Context, in *DeleteDiskRequest, opts ...grpc.CallOption) (*DeleteDiskResponse, error) + // Object storage (Ceph RGW / S3-compatible buckets) + CreateBucket(ctx context.Context, in *CreateBucketRequest, opts ...grpc.CallOption) (*CreateBucketResponse, error) + ListBuckets(ctx context.Context, in *ListBucketsRequest, opts ...grpc.CallOption) (*ListBucketsResponse, error) + GetBucket(ctx context.Context, in *GetBucketRequest, opts ...grpc.CallOption) (*GetBucketResponse, error) + DeleteBucket(ctx context.Context, in *DeleteBucketRequest, opts ...grpc.CallOption) (*DeleteBucketResponse, error) + GetBucketAccess(ctx context.Context, in *GetBucketAccessRequest, opts ...grpc.CallOption) (*GetBucketAccessResponse, error) + ListBucketObjects(ctx context.Context, in *ListBucketObjectsRequest, opts ...grpc.CallOption) (*ListBucketObjectsResponse, error) } type agentControlClient struct { @@ -353,6 +366,66 @@ func (c *agentControlClient) DeleteDisk(ctx context.Context, in *DeleteDiskReque return out, nil } +func (c *agentControlClient) CreateBucket(ctx context.Context, in *CreateBucketRequest, opts ...grpc.CallOption) (*CreateBucketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateBucketResponse) + err := c.cc.Invoke(ctx, AgentControl_CreateBucket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) ListBuckets(ctx context.Context, in *ListBucketsRequest, opts ...grpc.CallOption) (*ListBucketsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBucketsResponse) + err := c.cc.Invoke(ctx, AgentControl_ListBuckets_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) GetBucket(ctx context.Context, in *GetBucketRequest, opts ...grpc.CallOption) (*GetBucketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBucketResponse) + err := c.cc.Invoke(ctx, AgentControl_GetBucket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) DeleteBucket(ctx context.Context, in *DeleteBucketRequest, opts ...grpc.CallOption) (*DeleteBucketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeleteBucketResponse) + err := c.cc.Invoke(ctx, AgentControl_DeleteBucket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) GetBucketAccess(ctx context.Context, in *GetBucketAccessRequest, opts ...grpc.CallOption) (*GetBucketAccessResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBucketAccessResponse) + err := c.cc.Invoke(ctx, AgentControl_GetBucketAccess_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) ListBucketObjects(ctx context.Context, in *ListBucketObjectsRequest, opts ...grpc.CallOption) (*ListBucketObjectsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBucketObjectsResponse) + err := c.cc.Invoke(ctx, AgentControl_ListBucketObjects_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // AgentControlServer is the server API for AgentControl service. // All implementations must embed UnimplementedAgentControlServer // for forward compatibility. @@ -399,6 +472,13 @@ type AgentControlServer interface { ListDisks(context.Context, *ListDisksRequest) (*ListDisksResponse, error) GetDisk(context.Context, *GetDiskRequest) (*GetDiskResponse, error) DeleteDisk(context.Context, *DeleteDiskRequest) (*DeleteDiskResponse, error) + // Object storage (Ceph RGW / S3-compatible buckets) + CreateBucket(context.Context, *CreateBucketRequest) (*CreateBucketResponse, error) + ListBuckets(context.Context, *ListBucketsRequest) (*ListBucketsResponse, error) + GetBucket(context.Context, *GetBucketRequest) (*GetBucketResponse, error) + DeleteBucket(context.Context, *DeleteBucketRequest) (*DeleteBucketResponse, error) + GetBucketAccess(context.Context, *GetBucketAccessRequest) (*GetBucketAccessResponse, error) + ListBucketObjects(context.Context, *ListBucketObjectsRequest) (*ListBucketObjectsResponse, error) mustEmbedUnimplementedAgentControlServer() } @@ -481,6 +561,24 @@ func (UnimplementedAgentControlServer) GetDisk(context.Context, *GetDiskRequest) func (UnimplementedAgentControlServer) DeleteDisk(context.Context, *DeleteDiskRequest) (*DeleteDiskResponse, error) { return nil, status.Error(codes.Unimplemented, "method DeleteDisk not implemented") } +func (UnimplementedAgentControlServer) CreateBucket(context.Context, *CreateBucketRequest) (*CreateBucketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CreateBucket not implemented") +} +func (UnimplementedAgentControlServer) ListBuckets(context.Context, *ListBucketsRequest) (*ListBucketsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBuckets not implemented") +} +func (UnimplementedAgentControlServer) GetBucket(context.Context, *GetBucketRequest) (*GetBucketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetBucket not implemented") +} +func (UnimplementedAgentControlServer) DeleteBucket(context.Context, *DeleteBucketRequest) (*DeleteBucketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteBucket not implemented") +} +func (UnimplementedAgentControlServer) GetBucketAccess(context.Context, *GetBucketAccessRequest) (*GetBucketAccessResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetBucketAccess not implemented") +} +func (UnimplementedAgentControlServer) ListBucketObjects(context.Context, *ListBucketObjectsRequest) (*ListBucketObjectsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBucketObjects not implemented") +} func (UnimplementedAgentControlServer) mustEmbedUnimplementedAgentControlServer() {} func (UnimplementedAgentControlServer) testEmbeddedByValue() {} @@ -916,6 +1014,114 @@ func _AgentControl_DeleteDisk_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _AgentControl_CreateBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateBucketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).CreateBucket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_CreateBucket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).CreateBucket(ctx, req.(*CreateBucketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_ListBuckets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBucketsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).ListBuckets(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_ListBuckets_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).ListBuckets(ctx, req.(*ListBucketsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_GetBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBucketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).GetBucket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_GetBucket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).GetBucket(ctx, req.(*GetBucketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_DeleteBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteBucketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).DeleteBucket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_DeleteBucket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).DeleteBucket(ctx, req.(*DeleteBucketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_GetBucketAccess_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBucketAccessRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).GetBucketAccess(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_GetBucketAccess_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).GetBucketAccess(ctx, req.(*GetBucketAccessRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_ListBucketObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBucketObjectsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).ListBucketObjects(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_ListBucketObjects_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).ListBucketObjects(ctx, req.(*ListBucketObjectsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // AgentControl_ServiceDesc is the grpc.ServiceDesc for AgentControl service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -1011,6 +1217,30 @@ var AgentControl_ServiceDesc = grpc.ServiceDesc{ MethodName: "DeleteDisk", Handler: _AgentControl_DeleteDisk_Handler, }, + { + MethodName: "CreateBucket", + Handler: _AgentControl_CreateBucket_Handler, + }, + { + MethodName: "ListBuckets", + Handler: _AgentControl_ListBuckets_Handler, + }, + { + MethodName: "GetBucket", + Handler: _AgentControl_GetBucket_Handler, + }, + { + MethodName: "DeleteBucket", + Handler: _AgentControl_DeleteBucket_Handler, + }, + { + MethodName: "GetBucketAccess", + Handler: _AgentControl_GetBucketAccess_Handler, + }, + { + MethodName: "ListBucketObjects", + Handler: _AgentControl_ListBucketObjects_Handler, + }, }, Streams: []grpc.StreamDesc{ { From e18d411082c6c85fe4bcde7ae7543bd621c7b786 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:45:37 +0330 Subject: [PATCH 04/31] Feat: Implement sharding support for active-active scheduler mode --- .../internal/scheduler/sharding.go | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/sharding.go diff --git a/persys-scheduler/internal/scheduler/sharding.go b/persys-scheduler/internal/scheduler/sharding.go new file mode 100644 index 0000000..a49e1ff --- /dev/null +++ b/persys-scheduler/internal/scheduler/sharding.go @@ -0,0 +1,90 @@ +package scheduler + +import ( + "fmt" + "hash/crc32" + "strings" +) + +// haModeActiveActive is the SCHEDULER_HA_MODE value that enables sharding. +// Any other value (including empty/unset) is treated as "failover", the +// existing single-active-instance behavior from the previous round of +// this work. +const haModeActiveActive = "active-active" + +// isActiveActive reports whether this instance is configured for +// active-active (sharded) mode rather than failover mode. +func (s *Scheduler) isActiveActive() bool { + return s.cfg != nil && strings.EqualFold(strings.TrimSpace(s.cfg.SchedulerHAMode), haModeActiveActive) +} + +// shardTopology returns the effective (count, index) for this instance, +// normalizing invalid configuration (count < 1, index out of range) down +// to the safe single-shard default rather than silently misbehaving. +func (s *Scheduler) shardTopology() (count, index int) { + count = 1 + index = 0 + if s.cfg == nil { + return + } + if s.cfg.SchedulerShardCount > 1 { + count = s.cfg.SchedulerShardCount + } + if s.cfg.SchedulerShardIndex > 0 && s.cfg.SchedulerShardIndex < count { + index = s.cfg.SchedulerShardIndex + } + return +} + +// ownsNode reports whether this scheduler instance is responsible for +// driving reconciliation/monitoring/drift-detection for the given node. +// +// In failover mode (the default), exactly one scheduler instance is ever +// active cluster-wide (see leader.go), so it owns every node — this always +// returns true, matching pre-sharding behavior exactly. +// +// In active-active mode, nodes are partitioned across SCHEDULER_SHARD_COUNT +// shards by a stable hash of the node ID, and this instance only owns the +// nodes that hash to its own SCHEDULER_SHARD_INDEX. Placement itself +// (selectNodeForWorkload) is NOT gated by this — any replica can accept an +// ApplyWorkload call and assign a workload to any node cluster-wide; only +// the ongoing convergence loops (reconciliation, node/workload monitoring, +// drift detection) are partitioned, so a workload's steady-state upkeep is +// driven by whichever shard owns the node it landed on. +// +// Known caveat, worth having in mind before enabling active-active mode: +// if a node fails and RelocateWorkloadsFromNode reassigns its workloads to +// a node owned by a *different* shard, ownership of those workloads +// follows their new node immediately, but there's no explicit hand-off +// protocol between shards — the old shard simply stops seeing them on its +// next cycle (the workload's NodeID changed) and the new shard picks them +// up on its own next cycle. This is expected to self-heal within one +// reconcile interval, not a correctness bug, but it does mean a brief +// window where neither shard is actively driving that specific workload. +func (s *Scheduler) ownsNode(nodeID string) bool { + if !s.isActiveActive() { + return true + } + count, index := s.shardTopology() + if count <= 1 { + return true + } + h := crc32.ChecksumIEEE([]byte(strings.TrimSpace(nodeID))) + return int(h%uint32(count)) == index +} + +// electionKey returns the etcd key this instance campaigns on for +// leadership (see leader.go). In failover mode every replica contends on +// the same global key, so exactly one is ever active. In active-active +// mode each shard gets its own independent key, so replicas configured +// with different SCHEDULER_SHARD_INDEX values never contend with each +// other and can be active simultaneously; replicas sharing the same shard +// index still only elect one active owner for that shard, giving you HA +// within a shard if you run more than one replica per index. +func (s *Scheduler) electionKey() string { + if !s.isActiveActive() { + return leaderElectionKey + } + _, index := s.shardTopology() + return fmt.Sprintf("%s/shard-%d", leaderElectionKey, index) +} From 4ac8390cc3b4d9cdef3a5a984bf41b923935e6a0 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:46:03 +0330 Subject: [PATCH 05/31] Feat: Enhance scheduler with agent connection management and workload handling improvements --- .../internal/scheduler/scheduler.go | 534 ++++++++++++------ 1 file changed, 348 insertions(+), 186 deletions(-) diff --git a/persys-scheduler/internal/scheduler/scheduler.go b/persys-scheduler/internal/scheduler/scheduler.go index d75e05b..d641ec8 100644 --- a/persys-scheduler/internal/scheduler/scheduler.go +++ b/persys-scheduler/internal/scheduler/scheduler.go @@ -7,10 +7,12 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" "github.com/google/uuid" cfgpkg "github.com/persys-dev/persys-cloud/persys-scheduler/internal/config" + "github.com/persys-dev/persys-cloud/pkg/certmanager" "github.com/persys-dev/persys-cloud/persys-scheduler/internal/logging" "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" "github.com/redis/go-redis/v9" @@ -30,24 +32,32 @@ var schedulerLogger = logging.C("scheduler.core") // Scheduler holds the state and configuration for the cluster scheduler. type Scheduler struct { - cfg *cfgpkg.Config - etcdClient *clientv3.Client - redisClient *redis.Client - domain string - agentsDomain string - schedulerShard string - monitor *Monitor - reconciler *Reconciler - bgWG sync.WaitGroup - modeMu sync.RWMutex - mode OperatingMode - modeReasonText string - modeChangedAt time.Time - frozen *FrozenState - cacheMu sync.RWMutex - cacheNodes map[string]models.Node - cacheWorkloads map[string]models.Workload - cacheAssignments map[string]models.AssignmentRecord + cfg *cfgpkg.Config + etcdClient *clientv3.Client + redisClient *redis.Client + domain string + agentsDomain string + schedulerShard string + monitor *Monitor + reconciler *Reconciler + bgWG sync.WaitGroup + modeMu sync.RWMutex + mode OperatingMode + modeReasonText string + modeChangedAt time.Time + frozen *FrozenState + cacheMu sync.RWMutex + cacheNodes map[string]models.Node + cacheWorkloads map[string]models.Workload + cacheAssignments map[string]models.AssignmentRecord + agentConnMu sync.Mutex + agentConns map[string]*agentConnEntry + certMgr *certmanager.Manager // optional; enables ForceRotate on agent TLS failures + nodeCacheReady atomic.Bool + isLeader atomic.Bool + instanceID string + pendingMu sync.Mutex + pendingReservations map[string]map[string]pendingReservation } // NewScheduler initializes the scheduler with an etcd client and configuration. @@ -85,6 +95,8 @@ func NewScheduler(cfg *cfgpkg.Config) (*Scheduler, error) { cacheNodes: map[string]models.Node{}, cacheWorkloads: map[string]models.Workload{}, cacheAssignments: map[string]models.AssignmentRecord{}, + agentConns: map[string]*agentConnEntry{}, + instanceID: newInstanceID(), } // Initialize monitor and reconciler @@ -95,8 +107,18 @@ func NewScheduler(cfg *cfgpkg.Config) (*Scheduler, error) { return scheduler, nil } + +// SetCertManager attaches the process-wide certmanager so outbound agent +// dials can ForceRotate + retry on TLS handshake failures. Safe to call +// once after NewScheduler; nil is allowed (disables cert-aware retry). +func (s *Scheduler) SetCertManager(m *certmanager.Manager) { + s.certMgr = m +} + // Close shuts down the scheduler gracefully. func (s *Scheduler) Close() error { + s.closeAgentConns() + s.DeregisterSchedulerSelfFromCoreDNS() if s.etcdClient != nil { _ = s.etcdClient.Close() } @@ -117,24 +139,71 @@ func (s *Scheduler) RegisterNode(node models.Node) error { return fmt.Errorf("totalCPU and totalMemory must be positive") } + // Agent-advertised labels from this registration request. + agentLabels := node.Labels + if agentLabels == nil { + agentLabels = map[string]string{} + } + + // Load existing etcd record so operator-managed fields survive reconnect. + // Without this merge, RegisterNode does a full put and drops SetNodeLabel keys + // (UI / persysctl → SetNodeLabel → etcd), which is what agents see vanish after + // reconnect / re-register. + var existing *models.Node + if prev, err := s.GetNodeByID(node.NodeID); err == nil { + existing = &prev + } + node.LastHeartbeat = time.Now() node.DomainName = node.NodeID + "." + s.domain + + // Preserve operator drain/taint state across re-registration. + if existing != nil { + if strings.EqualFold(existing.Status, "Draining") || strings.EqualFold(existing.Status, "Drained") { + node.Status = existing.Status + node.StatusReason = existing.StatusReason + node.StatusUpdatedBy = existing.StatusUpdatedBy + node.StatusUpdatedAt = existing.StatusUpdatedAt + } + if len(existing.Taints) > 0 { + node.Taints = existing.Taints + } + } if node.Status == "" { node.Status = "Ready" } - node.StatusReason = "registered" - node.StatusUpdatedBy = "register" - node.StatusUpdatedAt = time.Now().UTC() + if existing == nil || (!strings.EqualFold(node.Status, "Draining") && !strings.EqualFold(node.Status, "Drained")) { + if node.StatusReason == "" { + node.StatusReason = "registered" + } + if node.StatusUpdatedBy == "" { + node.StatusUpdatedBy = "register" + } + node.StatusUpdatedAt = time.Now().UTC() + } if node.AvailableCPU == 0 { node.AvailableCPU = node.TotalCPU } if node.AvailableMemory == 0 { node.AvailableMemory = node.TotalMemory } - if node.Labels == nil { - node.Labels = make(map[string]string) + + // Label merge: + // 1) start with existing etcd labels (includes operator SetNodeLabel keys) + // 2) overlay agent-reported labels (agent is source of truth for keys it sends) + // 3) always stamp scheduler_shard + // Keys only present on the operator side are preserved when the agent omits them. + merged := make(map[string]string) + if existing != nil && existing.Labels != nil { + for k, v := range existing.Labels { + merged[k] = v + } + } + for k, v := range agentLabels { + merged[k] = v } - node.Labels["scheduler_shard"] = s.schedulerShard + merged["scheduler_shard"] = s.schedulerShard + node.Labels = merged schedulerLogger.WithFields(logrus.Fields{ "node_id": node.NodeID, @@ -147,6 +216,8 @@ func (s *Scheduler) RegisterNode(node models.Node) error { "total_memory_mb": node.TotalMemory, "available_cpu": node.AvailableCPU, "available_memory": node.AvailableMemory, + "label_count": len(node.Labels), + "re_register": existing != nil, }).Info("registering node") nodeJSON, err := json.Marshal(node) @@ -165,6 +236,11 @@ func (s *Scheduler) RegisterNode(node models.Node) error { schedulerLogger.WithField("node_id", node.NodeID).Info("updated CoreDNS record for node") } s.cacheNode(node) + s.emitEvent("NodeJoined", "", node.NodeID, "node registered", map[string]interface{}{ + "status": node.Status, + "agent_endpoint": node.AgentEndpoint, + "scheduler_shard": node.Labels["scheduler_shard"], + }) schedulerLogger.WithField("node_id", node.NodeID).Info("registered node") @@ -184,19 +260,6 @@ func matchesLabels(workloadLabels, nodeLabels map[string]string) bool { return true } -func nodeUtilizationScore(node models.Node) float64 { - cpuTotal := node.TotalCPU - memTotal := float64(node.TotalMemory) - if cpuTotal <= 0 || memTotal <= 0 { - return 1e9 - } - cpuUsed := cpuTotal - node.AvailableCPU - memUsed := memTotal - float64(node.AvailableMemory) - cpuRatio := cpuUsed / cpuTotal - memRatio := memUsed / memTotal - return (cpuRatio + memRatio) / 2.0 -} - func canonicalWorkloadType(t string) string { switch strings.ToLower(strings.TrimSpace(t)) { case "docker-container", "container": @@ -205,6 +268,8 @@ func canonicalWorkloadType(t string) string { return "compose" case "vm": return "vm" + case "microvm", "micro-vm", "firecracker": + return "microvm" default: return strings.ToLower(strings.TrimSpace(t)) } @@ -305,21 +370,25 @@ func isNodeStatusSubKey(key string) bool { return strings.HasSuffix(key, "/status") } -func (s *Scheduler) selectNodeForWorkload(workload models.Workload) (models.Node, string, error) { - if !s.isWritable() { - return models.Node{}, "", errControlPlaneFrozen +// candidateNodeSnapshot returns the node set placement decisions are made +// against. When the watch-backed node cache (node_watch.go) is populated +// and healthy, it's served from there — an in-memory map read instead of +// an etcd round-trip plus a fresh unmarshal of every node on every single +// placement decision. If the cache isn't ready yet (scheduler just +// started, or the watch stream is down and hasn't resynced), this falls +// back transparently to the original live etcd prefix scan, so placement +// never silently runs against stale or empty data. +func (s *Scheduler) candidateNodeSnapshot() ([]models.Node, error) { + if s.nodeCacheReady.Load() { + if nodes := s.getCachedNodes(); len(nodes) > 0 { + return nodes, nil + } } resp, err := s.RetryableEtcdGet(nodesPrefix, clientv3.WithPrefix()) if err != nil { - return models.Node{}, "", fmt.Errorf("failed to get nodes for scheduling: %v", err) - } - if len(resp.Kvs) == 0 { - return models.Node{}, "", fmt.Errorf("no nodes available") + return nil, fmt.Errorf("failed to get nodes for scheduling: %v", err) } - - candidates := make([]models.Node, 0) - rejections := make([]string, 0) - neededStorageDrivers := requiredStorageDrivers(workload) + nodes := make([]models.Node, 0, len(resp.Kvs)) for _, kv := range resp.Kvs { if isNodeStatusSubKey(string(kv.Key)) { continue @@ -329,6 +398,27 @@ func (s *Scheduler) selectNodeForWorkload(workload models.Workload) (models.Node schedulerLogger.WithError(err).WithField("key", string(kv.Key)).Warn("failed to unmarshal node data") continue } + nodes = append(nodes, node) + } + return nodes, nil +} + +func (s *Scheduler) selectNodeForWorkload(workload models.Workload) (models.Node, string, error) { + if !s.isWritable() { + return models.Node{}, "", errControlPlaneFrozen + } + nodes, err := s.candidateNodeSnapshot() + if err != nil { + return models.Node{}, "", err + } + if len(nodes) == 0 { + return models.Node{}, "", fmt.Errorf("no nodes available") + } + + candidates := make([]models.Node, 0) + rejections := make([]string, 0) + neededStorageDrivers := requiredStorageDrivers(workload) + for _, node := range nodes { if !strings.EqualFold(node.Status, "active") && !strings.EqualFold(node.Status, "ready") { reason := strings.TrimSpace(node.StatusReason) if reason == "" { @@ -378,11 +468,28 @@ func (s *Scheduler) selectNodeForWorkload(workload models.Workload) (models.Node return models.Node{}, "", fmt.Errorf("no suitable node available") } + counts, maxCount, err := s.workloadCountsByNode() + if err != nil { + // Non-fatal: fall back to treating spread as uniform (score + // degrades gracefully to the CPU/memory terms only) rather than + // failing the whole placement decision over a monitoring-adjacent + // read. + schedulerLogger.WithError(err).Warn("failed to compute workload counts for placement spread scoring; continuing without it") + counts = map[string]int{} + maxCount = 0 + } + sort.Slice(candidates, func(i, j int) bool { - return nodeUtilizationScore(candidates[i]) < nodeUtilizationScore(candidates[j]) + pendingCPUI, pendingMemI := s.pendingReservationFor(candidates[i].NodeID) + pendingCPUJ, pendingMemJ := s.pendingReservationFor(candidates[j].NodeID) + scoreI := nodePlacementScore(candidates[i], workload, pendingCPUI, pendingMemI, counts[candidates[i].NodeID], maxCount) + scoreJ := nodePlacementScore(candidates[j], workload, pendingCPUJ, pendingMemJ, counts[candidates[j].NodeID], maxCount) + return scoreI > scoreJ // descending: highest score (most preferred) first }) - reason := fmt.Sprintf("selected by lowest utilization score %.4f", nodeUtilizationScore(candidates[0])) + pendingCPU, pendingMem := s.pendingReservationFor(candidates[0].NodeID) + bestScore := nodePlacementScore(candidates[0], workload, pendingCPU, pendingMem, counts[candidates[0].NodeID], maxCount) + reason := fmt.Sprintf("selected by placement score %.4f (cpu/mem headroom + spread, node has %d workloads)", bestScore, counts[candidates[0].NodeID]) return candidates[0], reason, nil } @@ -407,6 +514,16 @@ func (s *Scheduler) RelocateWorkloadsFromNode(nodeID, reason string) (int, error relocated := 0 for i := range workloads { workload := workloads[i] + if isNodeLocalWorkload(workload) { + schedulerLogger.WithFields(logrus.Fields{ + "workload_id": workload.ID, + "node_id": nodeID, + }).Info("skip relocate: node-local workload hard-pinned") + s.emitEvent("RescheduleSkipped", workload.ID, nodeID, "node-local; not moving local data", map[string]interface{}{ + "persistence_class": workloadPersistenceClass(workload), + }) + continue + } nextNode, selectionReason, selErr := s.selectNodeForWorkload(workload) if selErr != nil { _ = s.UpdateWorkloadRetryOnFailure(workload.ID, fmt.Sprintf("%s; no relocation target: %v", reason, selErr)) @@ -454,6 +571,7 @@ func (s *Scheduler) assignWorkload(workload *models.Workload, node models.Node, if err := s.writeAssignment(workload.ID, node.NodeID, reason); err != nil { return err } + s.reservePlacement(node.NodeID, workload.ID, workload.Resources.CPUUsage, workload.Resources.MemoryUsage) s.emitEvent("WorkloadScheduled", workload.ID, node.NodeID, reason, nil) return nil } @@ -614,6 +732,10 @@ func (s *Scheduler) DeleteNode(nodeID string) error { schedulerLogger.WithError(err).WithField("node_id", nodeID).Warn("failed to remove CoreDNS entry") } + s.emitEvent("NodeLeft", "", nodeID, "node removed from cluster", map[string]interface{}{ + "status": "Removed", + }) + schedulerLogger.WithField("node_id", nodeID).Info("deleted node") return nil } @@ -665,6 +787,7 @@ func (s *Scheduler) GetWorkloads() ([]models.Workload, error) { workload.Metadata = st.Metadata workload.Retry = st.Retry workload.StatusInfo = st.StatusInfo + workload.Usage = st.Usage } workloads = append(workloads, workload) } @@ -735,6 +858,7 @@ func (s *Scheduler) GetWorkloadByID(workloadID string) (models.Workload, error) workload.Metadata = st.Metadata workload.Retry = st.Retry workload.StatusInfo = st.StatusInfo + workload.Usage = st.Usage } s.cacheWorkload(workload) @@ -808,28 +932,22 @@ func (s *Scheduler) DeleteWorkload(workloadID string) error { // UpdateWorkloadStatus updates the status of a workload. func (s *Scheduler) UpdateWorkloadStatus(workloadID, status string) error { - if err := s.requireWritable(); err != nil { - return err - } - workload, err := s.GetWorkloadByID(workloadID) + _, err := s.updateWorkloadStatusCAS(workloadID, func(workload *models.Workload) bool { + if strings.EqualFold(workload.Status, status) && strings.EqualFold(workload.StatusInfo.ActualState, status) { + return false + } + workload.Status = status + workload.StatusInfo.ActualState = status + workload.StatusInfo.LastUpdated = time.Now().UTC() + if workload.Metadata == nil { + workload.Metadata = map[string]interface{}{} + } + if strings.EqualFold(status, "failed") { + workload.Metadata["last_action"] = "Failed" + } + return true + }) if err != nil { - return err - } - - if strings.EqualFold(workload.Status, status) && strings.EqualFold(workload.StatusInfo.ActualState, status) { - return nil - } - - workload.Status = status - workload.StatusInfo.ActualState = status - workload.StatusInfo.LastUpdated = time.Now().UTC() - if workload.Metadata == nil { - workload.Metadata = map[string]interface{}{} - } - if strings.EqualFold(status, "failed") { - workload.Metadata["last_action"] = "Failed" - } - if err := s.saveWorkload(workload); err != nil { return fmt.Errorf("failed to update workload %s status: %v", workloadID, err) } @@ -842,29 +960,22 @@ func (s *Scheduler) UpdateWorkloadStatus(workloadID, status string) error { // UpdateWorkloadLogs updates the logs of a workload. func (s *Scheduler) UpdateWorkloadLogs(workloadID, logs string) error { - if err := s.requireWritable(); err != nil { - return err - } - workload, err := s.GetWorkloadByID(workloadID) - if err != nil { - return err - } - - // Append logs with timestamp - timestamp := time.Now().Format("2006-01-02 15:04:05") - logEntry := fmt.Sprintf("[%s] %s\n", timestamp, logs) - if strings.TrimSpace(logs) == "" { return nil } + timestamp := time.Now().Format("2006-01-02 15:04:05") + logEntry := fmt.Sprintf("[%s] %s\n", timestamp, logs) - if workload.Logs == "" { - workload.Logs = logEntry - } else { - workload.Logs += logEntry - } - workload.StatusInfo.LastUpdated = time.Now().UTC() - if err := s.saveWorkload(workload); err != nil { + _, err := s.updateWorkloadStatusCAS(workloadID, func(workload *models.Workload) bool { + if workload.Logs == "" { + workload.Logs = logEntry + } else { + workload.Logs += logEntry + } + workload.StatusInfo.LastUpdated = time.Now().UTC() + return true + }) + if err != nil { return fmt.Errorf("failed to update workload %s logs: %v", workloadID, err) } @@ -874,44 +985,40 @@ func (s *Scheduler) UpdateWorkloadLogs(workloadID, logs string) error { // UpdateWorkloadMetadata merges runtime metadata into the workload record. func (s *Scheduler) UpdateWorkloadMetadata(workloadID string, metadata map[string]string) error { - if err := s.requireWritable(); err != nil { - return err - } if len(metadata) == 0 { return nil } - workload, err := s.GetWorkloadByID(workloadID) - if err != nil { - return err - } - if workload.Metadata == nil { - workload.Metadata = map[string]interface{}{} - } - changed := false - for k, v := range metadata { - key := strings.TrimSpace(k) - if key == "" { - continue - } - cleanVal := strings.TrimSpace(v) - if existing, ok := workload.Metadata[key]; !ok || fmt.Sprintf("%v", existing) != cleanVal { - changed = true + _, err := s.updateWorkloadStatusCAS(workloadID, func(workload *models.Workload) bool { + if workload.Metadata == nil { + workload.Metadata = map[string]interface{}{} } - workload.Metadata[key] = cleanVal - if key == "container.stderr" || key == "container.runtime_error" { - if cleanVal != "" { - workload.Metadata["last_runtime_error"] = cleanVal - if strings.TrimSpace(workload.StatusInfo.FailureReason) == "" || isInfrastructureFailureReason(workload.StatusInfo.FailureReason) { - workload.StatusInfo.FailureReason = cleanVal + changed := false + for k, v := range metadata { + key := strings.TrimSpace(k) + if key == "" { + continue + } + cleanVal := strings.TrimSpace(v) + if existing, ok := workload.Metadata[key]; !ok || fmt.Sprintf("%v", existing) != cleanVal { + changed = true + } + workload.Metadata[key] = cleanVal + if key == "container.stderr" || key == "container.runtime_error" { + if cleanVal != "" { + workload.Metadata["last_runtime_error"] = cleanVal + if strings.TrimSpace(workload.StatusInfo.FailureReason) == "" || isInfrastructureFailureReason(workload.StatusInfo.FailureReason) { + workload.StatusInfo.FailureReason = cleanVal + } } } } - } - if !changed { - return nil - } - workload.StatusInfo.LastUpdated = time.Now().UTC() - if err := s.saveWorkload(workload); err != nil { + if !changed { + return false + } + workload.StatusInfo.LastUpdated = time.Now().UTC() + return true + }) + if err != nil { return fmt.Errorf("failed to update workload %s metadata: %v", workloadID, err) } return nil @@ -922,57 +1029,53 @@ func (s *Scheduler) UpdateWorkloadMetadata(workloadID string, metadata map[strin // envelope) and is used only for the meter usage-stream event; it is not // persisted on the workload record. func (s *Scheduler) UpdateWorkloadRuntimeDetails(workloadID, nodeID string, reason *models.WorkloadReason, usage *models.WorkloadUsage) error { - if err := s.requireWritable(); err != nil { - return err - } - workload, err := s.GetWorkloadByID(workloadID) - if err != nil { - return err - } - if workload.Metadata == nil { - workload.Metadata = map[string]interface{}{} - } - - if reason != nil { - copied := *reason - workload.StatusInfo.Reason = &copied - if strings.TrimSpace(copied.Message) != "" { - workload.StatusInfo.FailureReason = copied.Message - } - if strings.TrimSpace(copied.Code) != "" { - workload.Metadata["reason_code"] = copied.Code - } - if strings.TrimSpace(copied.Message) != "" { - workload.Metadata["reason_message"] = copied.Message - } - if !copied.LastTransition.IsZero() { - workload.Metadata["reason_last_transition"] = copied.LastTransition.UTC().Format(time.RFC3339) - } - if !copied.NextRetryAt.IsZero() { - workload.Metadata["reason_next_retry_at"] = copied.NextRetryAt.UTC().Format(time.RFC3339) + workload, err := s.updateWorkloadStatusCAS(workloadID, func(workload *models.Workload) bool { + if workload.Metadata == nil { + workload.Metadata = map[string]interface{}{} } - workload.Metadata["reason_retryable"] = fmt.Sprintf("%t", copied.Retryable) - } - if usage != nil { - copied := *usage - workload.Usage = &copied - workload.Metadata["usage_cpu_percent"] = fmt.Sprintf("%.4f", copied.CPUPercent) - workload.Metadata["usage_memory_bytes"] = fmt.Sprintf("%d", copied.MemoryBytes) - workload.Metadata["usage_disk_read_bytes"] = fmt.Sprintf("%d", copied.DiskReadBytes) - workload.Metadata["usage_disk_write_bytes"] = fmt.Sprintf("%d", copied.DiskWriteBytes) - workload.Metadata["usage_net_rx_bytes"] = fmt.Sprintf("%d", copied.NetRXBytes) - workload.Metadata["usage_net_tx_bytes"] = fmt.Sprintf("%d", copied.NetTXBytes) - if !copied.CollectedAt.IsZero() { - workload.Metadata["usage_collected_at"] = copied.CollectedAt.UTC().Format(time.RFC3339) + if reason != nil { + copied := *reason + workload.StatusInfo.Reason = &copied + if strings.TrimSpace(copied.Message) != "" { + workload.StatusInfo.FailureReason = copied.Message + } + if strings.TrimSpace(copied.Code) != "" { + workload.Metadata["reason_code"] = copied.Code + } + if strings.TrimSpace(copied.Message) != "" { + workload.Metadata["reason_message"] = copied.Message + } + if !copied.LastTransition.IsZero() { + workload.Metadata["reason_last_transition"] = copied.LastTransition.UTC().Format(time.RFC3339) + } + if !copied.NextRetryAt.IsZero() { + workload.Metadata["reason_next_retry_at"] = copied.NextRetryAt.UTC().Format(time.RFC3339) + } + workload.Metadata["reason_retryable"] = fmt.Sprintf("%t", copied.Retryable) } - if strings.TrimSpace(copied.Source) != "" { - workload.Metadata["usage_source"] = copied.Source + + if usage != nil { + copied := *usage + workload.Usage = &copied + workload.Metadata["usage_cpu_percent"] = fmt.Sprintf("%.4f", copied.CPUPercent) + workload.Metadata["usage_memory_bytes"] = fmt.Sprintf("%d", copied.MemoryBytes) + workload.Metadata["usage_disk_read_bytes"] = fmt.Sprintf("%d", copied.DiskReadBytes) + workload.Metadata["usage_disk_write_bytes"] = fmt.Sprintf("%d", copied.DiskWriteBytes) + workload.Metadata["usage_net_rx_bytes"] = fmt.Sprintf("%d", copied.NetRXBytes) + workload.Metadata["usage_net_tx_bytes"] = fmt.Sprintf("%d", copied.NetTXBytes) + if !copied.CollectedAt.IsZero() { + workload.Metadata["usage_collected_at"] = copied.CollectedAt.UTC().Format(time.RFC3339) + } + if strings.TrimSpace(copied.Source) != "" { + workload.Metadata["usage_source"] = copied.Source + } } - } - workload.StatusInfo.LastUpdated = time.Now().UTC() - if err := s.saveWorkload(workload); err != nil { + workload.StatusInfo.LastUpdated = time.Now().UTC() + return true + }) + if err != nil { return fmt.Errorf("failed to update workload %s runtime details: %v", workloadID, err) } @@ -1005,6 +1108,20 @@ func (s *Scheduler) GetWorkloadsByNode(nodeID string) ([]models.Workload, error) } // MonitorNodes periodically checks node health and updates status. +// backgroundLoopConcurrency returns the concurrency cap used by +// MonitorNodes, MonitorWorkloads, and detectDriftOnce. Shares the +// SCHEDULER_RECONCILE_CONCURRENCY setting with the reconciler +// (Reconciler.reconcileConcurrency) rather than introducing a separate +// knob — these loops make the same shape of per-item network call the +// reconciler does, just less frequently, so the same concurrency budget +// applies. +func (s *Scheduler) backgroundLoopConcurrency() int { + if s.cfg != nil && s.cfg.SchedulerReconcileConcurrency > 0 { + return s.cfg.SchedulerReconcileConcurrency + } + return defaultReconcileConcurrency +} + func (s *Scheduler) MonitorNodes(ctx context.Context) { ticker := time.NewTicker(1 * time.Minute) defer ticker.Stop() @@ -1022,60 +1139,105 @@ func (s *Scheduler) MonitorNodes(ctx context.Context) { schedulerLogger.WithError(err).Error("node monitoring cycle failed") continue } + owned := make([]models.Node, 0, len(nodes)) for _, node := range nodes { + if s.ownsNode(node.NodeID) { + owned = append(owned, node) + } + } + runBounded(owned, s.backgroundLoopConcurrency(), func(node models.Node) { if time.Since(node.LastHeartbeat) > 3*time.Minute { reason := fmt.Sprintf("heartbeat expired: last heartbeat %s", node.LastHeartbeat.UTC().Format(time.RFC3339)) if err := s.markNodeNotReady(node.NodeID, reason, "monitor"); err != nil { schedulerLogger.WithError(err).WithField("node_id", node.NodeID).Warn("failed to update node status") } } - } + }) } } } -// StartMonitoring starts both node monitoring and workload monitoring +// StartMonitoring starts the per-replica supervisory loop that tracks this +// instance's own etcd connectivity (used by requireWritable/isWritable +// checks throughout, including in gRPC handlers that must run on every +// replica regardless of leadership). Cluster-wide singleton work — node +// watch, node/workload monitoring, drift detection, reconciliation — is +// started separately via StartLeaderElectedBackgroundLoops, gated on +// leader election so it runs on exactly one replica at a time. func (s *Scheduler) StartMonitoring(ctx context.Context) { s.bgWG.Add(1) go func() { defer s.bgWG.Done() s.startModeSupervisor(ctx) }() +} - // Start node monitoring +// StartLeaderElectedBackgroundLoops campaigns for scheduler leadership (see +// leader.go) and, for as long as this instance holds it, runs every +// cluster-wide singleton background loop: the watch-backed node cache, +// node monitoring, workload monitoring, drift detection, and +// reconciliation. Only one scheduler replica runs these at a time; if the +// current leader dies or its etcd session lapses, another replica takes +// over automatically. Safe to call on every replica — non-leaders simply +// block campaigning until they win an election (e.g. after the previous +// leader stops). +func (s *Scheduler) StartLeaderElectedBackgroundLoops(ctx context.Context) { s.bgWG.Add(1) go func() { defer s.bgWG.Done() - s.MonitorNodes(ctx) + s.RunWithLeaderElection(ctx, s.runLeaderOnlyBackgroundLoops) + }() +} + +// runLeaderOnlyBackgroundLoops runs every singleton background loop and +// blocks until leaderCtx is cancelled — i.e. until this instance loses +// leadership (session expiry, or normal shutdown). Called by +// RunWithLeaderElection; not meant to be called directly. +func (s *Scheduler) runLeaderOnlyBackgroundLoops(leaderCtx context.Context) { + var wg sync.WaitGroup + + // Watch-backed node cache used by placement (candidateNodeSnapshot / + // selectNodeForWorkload) to avoid a full etcd scan per scheduling + // decision. + wg.Add(1) + go func() { + defer wg.Done() + s.StartNodeWatch(leaderCtx) }() - // Start workload monitoring + // Node monitoring + wg.Add(1) + go func() { + defer wg.Done() + s.MonitorNodes(leaderCtx) + }() + + // Workload monitoring if s.monitor != nil { - s.bgWG.Add(1) + wg.Add(1) go func() { - defer s.bgWG.Done() - s.monitor.MonitorWorkloads(ctx, 60*time.Second) + defer wg.Done() + s.monitor.MonitorWorkloads(leaderCtx, 60*time.Second) }() } - // Start drift detection loop (agent state vs scheduler state) - s.bgWG.Add(1) + // Drift detection (agent state vs scheduler state) + wg.Add(1) go func() { - defer s.bgWG.Done() - s.StartDriftDetection(ctx, s.driftDetectInterval()) + defer wg.Done() + s.StartDriftDetection(leaderCtx, s.driftDetectInterval()) }() -} -// StartReconciliation starts the reconciliation loop -func (s *Scheduler) StartReconciliation(ctx context.Context) { + // Reconciliation if s.reconciler != nil { - interval := s.cfg.SchedulerReconcileInterval - s.bgWG.Add(1) + wg.Add(1) go func() { - defer s.bgWG.Done() - s.reconciler.StartReconciliationLoop(ctx, interval) + defer wg.Done() + s.reconciler.StartReconciliationLoop(leaderCtx, s.cfg.SchedulerReconcileInterval) }() } + + wg.Wait() } // WaitForBackground blocks until scheduler background workers stop or timeout elapses. From 86c5d615f1647f87470210d417a0d446547465de Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:46:44 +0330 Subject: [PATCH 06/31] Feat: Implement Redis event stream for cluster-wide event handling and telemetry --- .../internal/scheduler/redis_store.go | 176 ++++++++++++++++-- 1 file changed, 161 insertions(+), 15 deletions(-) diff --git a/persys-scheduler/internal/scheduler/redis_store.go b/persys-scheduler/internal/scheduler/redis_store.go index 42b3d6c..6ab8fb4 100644 --- a/persys-scheduler/internal/scheduler/redis_store.go +++ b/persys-scheduler/internal/scheduler/redis_store.go @@ -7,6 +7,7 @@ import ( "time" "github.com/persys-dev/persys-cloud/persys-scheduler/internal/logging" + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" "github.com/redis/go-redis/v9" "github.com/sirupsen/logrus" ) @@ -69,24 +70,169 @@ func (s *Scheduler) writeReconciliationTelemetry(workloadID, action string, succ _ = s.RetryableEtcdPut(reconciliationKey(workloadID), string(payload)) } -func (s *Scheduler) writeEventTelemetry(payload []byte) bool { +// schedulerEventsStreamKey is the single Redis Stream every scheduler +// replica writes cluster-wide events to and reads/watches from. Because +// all replicas share the same Redis instance, this is exactly as +// "cluster-wide" as the etcd-backed approach it replaced: an event +// emitted by whichever replica handled the triggering request is visible +// to every replica's WatchEvents callers, not just the one that wrote it. +const schedulerEventsStreamKey = "scheduler:events" + +// eventRetentionTTL / eventMaxEntries return the configured Redis event +// retention knobs, defaulting sensibly if unset. +func (s *Scheduler) eventRetentionTTL() time.Duration { + if s.cfg != nil && s.cfg.RedisEventTTL > 0 { + return s.cfg.RedisEventTTL + } + return 24 * time.Hour +} + +func (s *Scheduler) eventMaxEntries() int64 { + if s.cfg != nil && s.cfg.RedisEventMaxEntries > 0 { + return s.cfg.RedisEventMaxEntries + } + return 1000 +} + +// writeEventToStream appends one event to the shared Redis Stream, +// trimming to eventMaxEntries (approximate — MAXLEN ~ is O(1) amortized, +// unlike a scan-and-delete sweep) and refreshing the stream key's TTL on +// every write so an idle cluster's event history eventually expires +// entirely rather than persisting forever. +// +// Returns the assigned stream ID on success. Cluster-wide events are +// explicitly NOT written to etcd (see the doc comment on emitEvent in +// state_store.go for why) — if Redis is unavailable, the event is +// dropped. This is a deliberate tradeoff: events are high-churn, +// ephemeral, observability-oriented data, not state the cluster's +// correctness depends on, and coupling them to etcd would reintroduce +// exactly the problem this design avoids. +func (s *Scheduler) writeEventToStream(payload []byte) (string, error) { + if s.redisClient == nil { + return "", fmt.Errorf("redis is not configured") + } + ctx := context.Background() + id, err := s.redisClient.XAdd(ctx, &redis.XAddArgs{ + Stream: schedulerEventsStreamKey, + MaxLen: s.eventMaxEntries(), + Approx: true, + Values: map[string]interface{}{"data": payload}, + }).Result() + if err != nil { + return "", err + } + // Best-effort TTL refresh; a failure here just means the stream lives + // a bit longer than configured, not a correctness problem. + _ = s.redisClient.Expire(ctx, schedulerEventsStreamKey, s.eventRetentionTTL()).Err() + return id, nil +} + +// readEventsFromStream returns the most recent limit events (or up to +// eventMaxEntries if limit <= 0), newest first — matching the ordering +// ListSchedulerEvents has always returned — plus the Redis stream ID of +// the newest entry returned (empty if there were none). The ID lets +// WatchEvents resume live-watching from exactly where a replay left off, +// with no gap and no duplicate delivery. +// +// Returns an empty slice (not an error) if Redis is unavailable or the +// stream doesn't exist yet, since this backs read paths (ListEvents, +// WatchEvents replay) that should degrade gracefully rather than fail a +// CLI/dashboard request outright over what is, by design, best-effort +// observability data. +func (s *Scheduler) readEventsFromStream(limit int64) (events []models.SchedulerEvent, newestID string) { + if s.redisClient == nil { + return nil, "" + } + if limit <= 0 { + limit = s.eventMaxEntries() + } + msgs, err := s.redisClient.XRevRangeN(context.Background(), schedulerEventsStreamKey, "+", "-", limit).Result() + if err != nil { + redisLogger.WithError(err).Warn("failed to read events from redis stream") + return nil, "" + } + events = make([]models.SchedulerEvent, 0, len(msgs)) + for i, msg := range msgs { + raw, ok := msg.Values["data"] + if !ok { + continue + } + var payload []byte + switch v := raw.(type) { + case string: + payload = []byte(v) + case []byte: + payload = v + default: + continue + } + var event models.SchedulerEvent + if err := json.Unmarshal(payload, &event); err != nil { + continue + } + events = append(events, event) + if i == 0 { + newestID = msg.ID + } + } + return events, newestID +} + +// watchEventStream blocks, delivering events newly added to the stream +// after fromID to onEvent, until ctx is cancelled, onEvent returns an +// error, or an unrecoverable Redis error occurs. fromID should be the +// highest ID already delivered to the caller (e.g. from a prior replay +// via readEventsFromStream, or a previous call to this function), or "$" +// to start from only new events with no replay. +// +// Always returns the highest ID actually delivered to onEvent before +// returning (even on error), so a caller retrying after a transient +// failure can resume from exactly that point instead of re-delivering +// everything from fromID again. +func (s *Scheduler) watchEventStream(ctx context.Context, fromID string, onEvent func(models.SchedulerEvent) error) (lastDeliveredID string, err error) { if s.redisClient == nil { - return false + return fromID, fmt.Errorf("redis is not configured") } - ttl := 24 * time.Hour - maxEntries := int64(2000) - if s.cfg != nil { - if s.cfg.RedisEventTTL > 0 { - ttl = s.cfg.RedisEventTTL + lastID := fromID + for { + if ctx.Err() != nil { + return lastID, ctx.Err() } - if s.cfg.RedisEventMaxEntries > 0 { - maxEntries = s.cfg.RedisEventMaxEntries + res, readErr := s.redisClient.XRead(ctx, &redis.XReadArgs{ + Streams: []string{schedulerEventsStreamKey, lastID}, + Block: 0, // block indefinitely until data arrives or ctx is cancelled + Count: 0, + }).Result() + if readErr != nil { + if readErr == redis.Nil || ctx.Err() != nil { + continue + } + return lastID, fmt.Errorf("redis XREAD failed: %w", readErr) + } + for _, stream := range res { + for _, msg := range stream.Messages { + lastID = msg.ID + raw, ok := msg.Values["data"] + if !ok { + continue + } + var payload []byte + switch v := raw.(type) { + case string: + payload = []byte(v) + case []byte: + payload = v + default: + continue + } + var event models.SchedulerEvent + if err := json.Unmarshal(payload, &event); err != nil { + continue + } + if err := onEvent(event); err != nil { + return lastID, err + } + } } } - pipe := s.redisClient.TxPipeline() - pipe.LPush(context.Background(), "events:history", payload) - pipe.LTrim(context.Background(), "events:history", 0, maxEntries-1) - pipe.Expire(context.Background(), "events:history", ttl) - _, err := pipe.Exec(context.Background()) - return err == nil } From 10004db4d448a3f9bb286fbdae971757faf9f9fc Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:47:05 +0330 Subject: [PATCH 07/31] Feat: Enhance reconciler with concurrent workload processing and node snapshot prefetching --- .../internal/scheduler/reconciler.go | 274 +++++++++++++++++- 1 file changed, 262 insertions(+), 12 deletions(-) diff --git a/persys-scheduler/internal/scheduler/reconciler.go b/persys-scheduler/internal/scheduler/reconciler.go index ae2fe0a..73c15f2 100644 --- a/persys-scheduler/internal/scheduler/reconciler.go +++ b/persys-scheduler/internal/scheduler/reconciler.go @@ -6,6 +6,7 @@ import ( "fmt" "math" "strings" + "sync" "time" agentpb "github.com/persys-dev/persys-cloud/persys-scheduler/internal/agentpb" @@ -15,6 +16,11 @@ import ( "github.com/sirupsen/logrus" ) +// defaultReconcileConcurrency bounds how many workloads (and, during +// snapshot prefetch, how many nodes) are processed concurrently in a single +// reconciliation cycle. Overridable via SCHEDULER_RECONCILE_CONCURRENCY. +const defaultReconcileConcurrency = 64 + const defaultMissingGracePeriod = 15 * time.Second const defaultNodeUnavailableGrace = 3 * time.Minute const workloadReapplyTimestampKey = "lastApplyRequestAt" @@ -29,16 +35,44 @@ const terminalFailureReasonMetadataKey = "terminal_failure_reason" var reconcilerLogger = logging.C("scheduler.reconciler") var nonRetryableFailureReasons = map[string]struct{}{ - "INVALID_IMAGE": {}, - "INVALID_SPECIFICATION": {}, - "INVALID_CONFIGURATION": {}, - "PORT_BIND_CONFLICT": {}, + "INVALID_IMAGE": {}, + "INVALID_SPECIFICATION": {}, + "INVALID_CONFIGURATION": {}, + "PORT_BIND_CONFLICT": {}, + "DISK_PRESSURE": {}, + "DOMAIN_PAUSED_IO": {}, + "RESOURCE_QUOTA": {}, + "RESOURCE_QUOTA_EXCEEDED": {}, } // Reconciler handles reconciliation between desired and actual state. type Reconciler struct { scheduler *Scheduler monitor *Monitor + + // cycleMu/cycleRunning prevent a reconciliation cycle from starting + // while the previous one is still in flight (possible once cycles run + // concurrently internally; without this guard, a slow cycle plus a + // healthy tick interval could stack overlapping full-cluster passes). + cycleMu sync.Mutex + cycleRunning bool + + // snapshot holds, for the duration of a single ReconcileAllWorkloads + // cycle, one batched workload-list response per node (see + // prefetchNodeSnapshots). getActualWorkloadState consults it before + // falling back to a live per-workload RPC, which turns the scheduler's + // fan-out to agents from O(workloads) into O(nodes) per cycle. + snapshotMu sync.RWMutex + snapshot map[string]*nodeSnapshotEntry +} + +// nodeSnapshotEntry records the outcome of one node's batched workload-list +// call for the current reconciliation cycle. ok=false means the batch call +// wasn't attempted or failed, so callers must fall back to a live per- +// workload call rather than assuming the workload is missing. +type nodeSnapshotEntry struct { + ok bool + statuses map[string]*agentpb.WorkloadStatus } // ReconciliationResult represents the result of a reconciliation operation. @@ -281,6 +315,39 @@ func (r *Reconciler) handleUnavailableAssignedNode(workload *models.Workload) (b } oldNode := workload.NodeID + + // Hard pin: node-local workloads must not relocate on node flap. + // Override: reapply with metadata persys.scheduling.allow_move=true. + if isNodeLocalWorkload(*workload) { + if workload.Metadata == nil { + workload.Metadata = map[string]interface{}{} + } + workload.Metadata[metaPinnedReason] = "node-local storage; waiting for original node" + workload.Metadata["last_action"] = "PinnedAwaitNode" + workload.Status = "Pending" + workload.StatusInfo.LastUpdated = time.Now().UTC() + if err := r.scheduler.saveWorkload(*workload); err != nil { + return false, err + } + _ = r.scheduler.UpdateWorkloadLogs(workload.ID, fmt.Sprintf( + "node %s unavailable; node-local workload pinned (reapply with metadata %s=true to force move)", + oldNode, metaAllowMove, + )) + r.scheduler.emitEvent("RescheduleSkipped", workload.ID, oldNode, + "node-local workload hard-pinned; not relocating", + map[string]interface{}{ + "persistence_class": workloadPersistenceClass(*workload), + "allow_move": false, + }, + ) + reconcilerLogger.WithFields(logrus.Fields{ + "workload_id": workload.ID, + "node_id": oldNode, + "persistence_class": workloadPersistenceClass(*workload), + }).Info("skipped failover for node-local workload") + return true, nil + } + nextNode, reason, selErr := r.scheduler.selectNodeForWorkload(*workload) if selErr != nil { reconcilerLogger.WithError(selErr).WithFields(logrus.Fields{ @@ -326,8 +393,20 @@ func (r *Reconciler) handleUnavailableAssignedNode(workload *models.Workload) (b return false, nil } -// getActualWorkloadState queries the agent to get the actual state of a workload. +// getActualWorkloadState returns the actual state of a workload as last +// reported by its assigned node. When a batched per-node snapshot for the +// current reconciliation cycle is available (see prefetchNodeSnapshots), it +// is used instead of a live RPC. Outside of a full-cluster cycle (e.g. a +// single-workload reconcile triggered by ApplyWorkload), no snapshot is set +// and this falls back to the original live per-workload call. func (r *Reconciler) getActualWorkloadState(ctx context.Context, workload models.Workload) (string, error) { + if statusResp, found, nodeSnapshotted := r.snapshotLookup(workload.NodeID, workload.ID); nodeSnapshotted { + if !found { + return "Missing", nil + } + return r.applyStatusResponse(workload, statusResp), nil + } + node, err := r.scheduler.GetNodeByID(workload.NodeID) if err != nil { return "", fmt.Errorf("failed to get node %s: %v", workload.NodeID, err) @@ -343,15 +422,129 @@ func (r *Reconciler) getActualWorkloadState(ctx context.Context, workload models if statusResp == nil { return "Unknown", nil } + return r.applyStatusResponse(workload, statusResp), nil +} + +// applyStatusResponse persists any metadata/failure-reason side effects +// carried on an agent status response and returns the mapped state string. +// Shared by both the snapshot path and the live per-workload RPC path. +func (r *Reconciler) applyStatusResponse(workload models.Workload, statusResp *agentpb.WorkloadStatus) string { if len(statusResp.GetMetadata()) > 0 { _ = r.scheduler.UpdateWorkloadMetadata(workload.ID, statusResp.GetMetadata()) } - if strings.EqualFold(mapActualStateToSchedulerStatus(statusResp.GetActualState()), "Failed") { + state := mapActualStateToSchedulerStatus(statusResp.GetActualState()) + if strings.EqualFold(state, "Failed") { if msg := strings.TrimSpace(statusResp.GetMessage()); msg != "" { _ = r.scheduler.UpdateWorkloadMetadata(workload.ID, map[string]string{"last_runtime_error": msg}) } } - return mapActualStateToSchedulerStatus(statusResp.GetActualState()), nil + return state +} + +// snapshotLookup returns the batched status for (nodeID, workloadID) if the +// current cycle's snapshot has an entry for that node. nodeSnapshotted is +// false if the node wasn't part of this cycle's batch prefetch (or the +// batch call to it failed), signalling the caller should fall back to a +// live RPC rather than assume the workload is missing. +func (r *Reconciler) snapshotLookup(nodeID, workloadID string) (status *agentpb.WorkloadStatus, found bool, nodeSnapshotted bool) { + if strings.TrimSpace(nodeID) == "" { + return nil, false, false + } + r.snapshotMu.RLock() + defer r.snapshotMu.RUnlock() + if r.snapshot == nil { + return nil, false, false + } + entry, ok := r.snapshot[nodeID] + if !ok || !entry.ok { + return nil, false, false + } + status, found = entry.statuses[workloadID] + return status, found, true +} + +// reconcileConcurrency returns the configured cap on concurrent per-workload +// reconciliation and per-node snapshot-prefetch work in a single cycle. +func (r *Reconciler) reconcileConcurrency() int { + if r.scheduler.cfg != nil && r.scheduler.cfg.SchedulerReconcileConcurrency > 0 { + return r.scheduler.cfg.SchedulerReconcileConcurrency + } + return defaultReconcileConcurrency +} + +// prefetchNodeSnapshots concurrently fetches one batched workload-list RPC +// per distinct node referenced by the given workloads, instead of the +// reconciler making one GetWorkloadStatus RPC per workload. This is what +// turns the scheduler's fan-out to agents from O(workloads) into O(nodes) +// per reconciliation cycle. Nodes that are already known NotReady are +// skipped (their workloads go through the existing failover path instead of +// a doomed status call), and any node whose batch call fails is simply left +// out of the snapshot so per-workload reconciliation falls back to a live +// call for it. +func (r *Reconciler) prefetchNodeSnapshots(ctx context.Context, workloads []models.Workload) { + nodeIDs := make(map[string]struct{}) + for _, w := range workloads { + id := strings.TrimSpace(w.NodeID) + if id == "" { + continue + } + nodeIDs[id] = struct{}{} + } + if len(nodeIDs) == 0 { + r.snapshotMu.Lock() + r.snapshot = map[string]*nodeSnapshotEntry{} + r.snapshotMu.Unlock() + return + } + + snapshot := make(map[string]*nodeSnapshotEntry, len(nodeIDs)) + var snapMu sync.Mutex + sem := make(chan struct{}, r.reconcileConcurrency()) + var wg sync.WaitGroup + + for nodeID := range nodeIDs { + nodeID := nodeID + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + + node, err := r.scheduler.GetNodeByID(nodeID) + if err != nil { + return + } + if !strings.EqualFold(node.Status, "Ready") && !strings.EqualFold(node.Status, "Active") { + return + } + list, err := r.scheduler.listWorkloadsFromNode(ctx, node) + if err != nil { + return + } + byID := make(map[string]*agentpb.WorkloadStatus, len(list)) + for _, st := range list { + byID[st.GetId()] = st + } + snapMu.Lock() + snapshot[nodeID] = &nodeSnapshotEntry{ok: true, statuses: byID} + snapMu.Unlock() + }() + } + wg.Wait() + + r.snapshotMu.Lock() + r.snapshot = snapshot + r.snapshotMu.Unlock() +} + +// clearSnapshot drops the current cycle's snapshot so that reconciliation +// calls outside of a full ReconcileAllWorkloads cycle (e.g. the immediate +// single-workload reconcile triggered by ApplyWorkload) correctly fall back +// to live per-workload RPCs instead of an empty or stale snapshot. +func (r *Reconciler) clearSnapshot() { + r.snapshotMu.Lock() + r.snapshot = nil + r.snapshotMu.Unlock() } // needsReconciliation determines if a workload needs reconciliation. @@ -836,7 +1029,20 @@ func (r *Reconciler) updateWorkloadReconciliationStatus(workloadID string, resul } } -// ReconcileAllWorkloads reconciles all workloads in the system. +// ReconcileAllWorkloads reconciles all workloads in the system. Node status +// is prefetched once per node (not once per workload) and per-workload +// reconciliation runs with bounded concurrency, so cycle time scales with +// the slower of node count / concurrency limit rather than with total +// workload count times per-workload RPC latency. +// +// NOTE: concurrent workloads that happen to share an assigned node can +// still race on that node's etcd record (e.g. two workloads on the same +// failing node both trying to mark it NotReady / trigger failover at once). +// This was already possible with the sequential loop across ticks, but +// concurrency makes it materially more likely; it's mitigated, not fixed, +// by this change. Adding etcd CAS to node/workload writes (see the scaling +// plan doc) removes the race outright and should land before pushing +// concurrency limits much higher than the default. func (r *Reconciler) ReconcileAllWorkloads(ctx context.Context) ([]*ReconciliationResult, error) { if ctx == nil { ctx = context.Background() @@ -846,16 +1052,46 @@ func (r *Reconciler) ReconcileAllWorkloads(ctx context.Context) ([]*Reconciliati return nil, fmt.Errorf("failed to get workloads: %v", err) } - var results []*ReconciliationResult + active := make([]models.Workload, 0, len(workloads)) for _, workload := range workloads { if workload.Status == "Completed" || workload.Status == "Deleted" { continue } - result, err := r.ReconcileWorkload(ctx, workload) - if err != nil { - reconcilerLogger.WithError(err).WithField("workload_id", workload.ID).Warn("failed to reconcile workload") + if !r.scheduler.ownsNode(workload.NodeID) { continue } + active = append(active, workload) + } + if len(active) == 0 { + return nil, nil + } + + r.prefetchNodeSnapshots(ctx, active) + defer r.clearSnapshot() + + resultsCh := make(chan *ReconciliationResult, len(active)) + sem := make(chan struct{}, r.reconcileConcurrency()) + var wg sync.WaitGroup + for _, workload := range active { + workload := workload + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + result, err := r.ReconcileWorkload(ctx, workload) + if err != nil { + reconcilerLogger.WithError(err).WithField("workload_id", workload.ID).Warn("failed to reconcile workload") + return + } + resultsCh <- result + }() + } + wg.Wait() + close(resultsCh) + + results := make([]*ReconciliationResult, 0, len(active)) + for result := range resultsCh { results = append(results, result) } return results, nil @@ -876,9 +1112,23 @@ func (r *Reconciler) StartReconciliationLoop(ctx context.Context, interval time. if !r.scheduler.isWritable() { continue } + r.cycleMu.Lock() + if r.cycleRunning { + r.cycleMu.Unlock() + reconcilerLogger.Warn("skipping reconciliation tick: previous cycle still in progress") + continue + } + r.cycleRunning = true + r.cycleMu.Unlock() + cycleStart := time.Now() results, err := r.ReconcileAllWorkloads(ctx) metricspkg.ObserveReconciliationCycle(time.Since(cycleStart), err) + + r.cycleMu.Lock() + r.cycleRunning = false + r.cycleMu.Unlock() + if err != nil { reconcilerLogger.WithError(err).Error("reconciliation cycle failed") continue From 184c1e33a85bbcb36d4c9c220fc76deb3c6f03ae Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:47:18 +0330 Subject: [PATCH 08/31] Feat: Implement placement scoring and resource reservation for workload scheduling --- .../internal/scheduler/placement.go | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/placement.go diff --git a/persys-scheduler/internal/scheduler/placement.go b/persys-scheduler/internal/scheduler/placement.go new file mode 100644 index 0000000..5c55fd3 --- /dev/null +++ b/persys-scheduler/internal/scheduler/placement.go @@ -0,0 +1,182 @@ +package scheduler + +import ( + "hash/fnv" + "strings" + "time" + + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" +) + +// Placement scoring weights. CPU and memory headroom dominate (a node +// that's actually short on resources should never outscore one that +// isn't), with a smaller spread term to avoid piling workloads onto an +// otherwise-tied node just because it happened to sort first. These are +// constants rather than a config knob for now because retuning them +// usefully needs operational data on real workload mixes that doesn't +// exist yet — a knob nobody has good values for is worse than no knob. +const ( + placementWeightCPU = 0.4 + placementWeightMemory = 0.4 + placementWeightSpread = 0.2 + + // pendingReservationTTL bounds how long an in-flight placement + // reservation (see below) is counted against a node before it's + // treated as stale and dropped. Chosen as a multiple of a typical + // heartbeat interval so a reservation reliably outlives the window + // where the node's own reported AvailableCPU/AvailableMemory hasn't + // caught up yet, without leaking indefinitely if a workload's status + // updates are delayed or lost. + pendingReservationTTL = 90 * time.Second +) + +// pendingReservation tracks resources committed to a workload that was +// just assigned to a node but not yet reflected in that node's own +// heartbeat-reported availability. +type pendingReservation struct { + cpu float64 + memory float64 + expiresAt time.Time +} + +// reservePlacement records that workloadID has just been assigned to +// nodeID, committing the given CPU/memory. Called from assignWorkload +// immediately after a successful assignment. +// +// This exists because node.AvailableCPU/AvailableMemory are only updated +// by that node's own heartbeat, which lags behind an assignment decision. +// Without tracking commitments locally, concurrent placement decisions — +// routine now: reconciliation and monitoring run with bounded concurrency, +// and active-active mode can have multiple scheduler replicas placing +// workloads at the same time — can all read the same stale "available" +// numbers and independently pick the same node, oversubscribing it before +// any of their heartbeats catch up. Reservations are advisory (they adjust +// scoring, not a hard admission-control gate) and self-expire via +// pendingReservationTTL rather than needing an explicit clear on heartbeat +// confirmation, trading a little scoring precision for not having to hook +// into every workload-status-confirmation path. +func (s *Scheduler) reservePlacement(nodeID, workloadID string, cpu float64, memory float64) { + if strings.TrimSpace(nodeID) == "" || strings.TrimSpace(workloadID) == "" { + return + } + s.pendingMu.Lock() + defer s.pendingMu.Unlock() + if s.pendingReservations == nil { + s.pendingReservations = make(map[string]map[string]pendingReservation) + } + byWorkload, ok := s.pendingReservations[nodeID] + if !ok { + byWorkload = make(map[string]pendingReservation) + s.pendingReservations[nodeID] = byWorkload + } + byWorkload[workloadID] = pendingReservation{ + cpu: cpu, + memory: memory, + expiresAt: time.Now().Add(pendingReservationTTL), + } +} + +// pendingReservationFor sums non-expired reservations for nodeID, pruning +// expired entries as it goes so the map doesn't grow unboundedly. +func (s *Scheduler) pendingReservationFor(nodeID string) (cpu float64, memory float64) { + s.pendingMu.Lock() + defer s.pendingMu.Unlock() + byWorkload, ok := s.pendingReservations[nodeID] + if !ok { + return 0, 0 + } + now := time.Now() + for workloadID, r := range byWorkload { + if now.After(r.expiresAt) { + delete(byWorkload, workloadID) + continue + } + cpu += r.cpu + memory += r.memory + } + if len(byWorkload) == 0 { + delete(s.pendingReservations, nodeID) + } + return cpu, memory +} + +// workloadCountsByNode returns how many non-terminal workloads are +// currently assigned to each node, plus the highest count on any single +// node, for use as the spread term in nodePlacementScore. Computed once +// per placement decision (not once per candidate node), so it costs one +// extra GetWorkloads() call per scheduling event — negligible next to +// actual scheduling event rates, unlike the per-workload agent RPCs the +// reconciler used to make on every cycle. +func (s *Scheduler) workloadCountsByNode() (counts map[string]int, maxCount int, err error) { + workloads, err := s.GetWorkloads() + if err != nil { + return nil, 0, err + } + counts = make(map[string]int) + for _, w := range workloads { + if w.Status == "Completed" || w.Status == "Deleted" { + continue + } + nodeID := strings.TrimSpace(w.NodeID) + if nodeID == "" { + continue + } + counts[nodeID]++ + if counts[nodeID] > maxCount { + maxCount = counts[nodeID] + } + } + return counts, maxCount, nil +} + +// nodePlacementScore scores a feasible candidate node for a workload; +// higher is better. It blends three signals rather than the single +// CPU+memory average the previous version used: +// +// - CPU and memory headroom, adjusted by pendingReservationFor so +// resources already committed to workloads assigned in this +// scheduling window (but not yet reflected in the node's own +// heartbeat) count against the node instead of being invisible. +// - A spread term based on how many workloads are already assigned to +// the node relative to the busiest candidate, so two nodes with +// similar CPU/memory ratios aren't treated as identical if one is +// already hosting far more workloads than the other. +// - A small deterministic jitter, hashed from the workload and node ID +// (not real randomness — reproducible for the same workload/node +// pair), to break exact ties without always resolving them to +// whichever node happens to sort first. Real ties are common in a +// mostly-homogeneous fleet; always breaking them the same way is +// itself a form of imbalance. +func nodePlacementScore(node models.Node, workload models.Workload, pendingCPU, pendingMemory float64, workloadCount, maxWorkloadCount int) float64 { + cpuTotal := node.TotalCPU + memTotal := float64(node.TotalMemory) + if cpuTotal <= 0 || memTotal <= 0 { + return -1e9 + } + + effectiveAvailableCPU := node.AvailableCPU - pendingCPU + effectiveAvailableMemory := float64(node.AvailableMemory) - pendingMemory + + cpuHeadroom := effectiveAvailableCPU / cpuTotal + memHeadroom := effectiveAvailableMemory / memTotal + + spreadTerm := 1.0 + if maxWorkloadCount > 0 { + spreadTerm = 1.0 - (float64(workloadCount) / float64(maxWorkloadCount)) + } + + score := placementWeightCPU*cpuHeadroom + placementWeightMemory*memHeadroom + placementWeightSpread*spreadTerm + score += placementTieBreakJitter(workload.ID, node.NodeID) + return score +} + +// placementTieBreakJitter returns a small, deterministic (not random) +// value in [0, 0.01) derived from the workload/node ID pair — large enough +// to reliably separate exact ties between the weighted terms above (which +// each range roughly 0-1), small enough to never override a genuine +// difference between two candidates. +func placementTieBreakJitter(workloadID, nodeID string) float64 { + h := fnv.New32a() + _, _ = h.Write([]byte(workloadID + "|" + nodeID)) + return (float64(h.Sum32()%1000) / 1000.0) * 0.01 +} From 313c035f666fc2e226d8f82d720fd8d62fe22967 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:47:45 +0330 Subject: [PATCH 09/31] Feat: Add persistence management functions for workload scheduling --- .../internal/scheduler/persistence.go | 91 +++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/persistence.go diff --git a/persys-scheduler/internal/scheduler/persistence.go b/persys-scheduler/internal/scheduler/persistence.go new file mode 100644 index 0000000..d32ab2b --- /dev/null +++ b/persys-scheduler/internal/scheduler/persistence.go @@ -0,0 +1,91 @@ +package scheduler + +import ( + "fmt" + "strings" + + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" +) + +const ( + metaAllowMove = "persys.scheduling.allow_move" + metaPersistenceClass = "persys.scheduling.persistence_class" + metaPinnedReason = "persys.scheduling.pinned_reason" + persistenceEphemeral = "ephemeral" + persistenceNodeLocal = "node-local" + persistenceShared = "shared-storage" +) + +func isMetadataTruthy(meta map[string]interface{}, key string) bool { + if meta == nil { + return false + } + v, ok := meta[key] + if !ok || v == nil { + return false + } + switch t := v.(type) { + case bool: + return t + case string: + s := strings.ToLower(strings.TrimSpace(t)) + return s == "true" || s == "1" || s == "yes" + default: + return strings.EqualFold(strings.TrimSpace(fmt.Sprint(t)), "true") + } +} + +func metaStringRaw(meta map[string]interface{}, key string) string { + if meta == nil { + return "" + } + v, ok := meta[key] + if !ok || v == nil { + return "" + } + if s, ok := v.(string); ok { + return strings.TrimSpace(s) + } + return strings.TrimSpace(fmt.Sprint(v)) +} + +func workloadAllowsMove(w models.Workload) bool { + return isMetadataTruthy(w.Metadata, metaAllowMove) || isMetadataTruthy(w.Metadata, "allow_move") +} + +func workloadPersistenceClass(w models.Workload) string { + if explicit := strings.ToLower(metaStringRaw(w.Metadata, metaPersistenceClass)); explicit != "" { + switch explicit { + case persistenceEphemeral, persistenceNodeLocal, persistenceShared: + return explicit + } + } + t := strings.ToLower(strings.TrimSpace(w.Type)) + if t == "vm" || t == "microvm" { + if isMetadataTruthy(w.Metadata, "persys.storage.shared") { + return persistenceShared + } + return persistenceNodeLocal + } + if isMetadataTruthy(w.Metadata, "persys.storage.local") { + return persistenceNodeLocal + } + if isMetadataTruthy(w.Metadata, "persys.storage.shared") { + return persistenceShared + } + if w.Metadata != nil { + for k, v := range w.Metadata { + ks := strings.ToLower(k) + if strings.Contains(ks, "host_path") || strings.Contains(ks, "bind_mount") { + if strings.TrimSpace(fmt.Sprint(v)) != "" { + return persistenceNodeLocal + } + } + } + } + return persistenceEphemeral +} + +func isNodeLocalWorkload(w models.Workload) bool { + return workloadPersistenceClass(w) == persistenceNodeLocal && !workloadAllowsMove(w) +} From fa281b6456d1aed96cb3f73293de7b2f8bc09c77 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:48:01 +0330 Subject: [PATCH 10/31] Feat: Implement node watch functionality for dynamic node cache updates --- .../internal/scheduler/node_watch.go | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/node_watch.go diff --git a/persys-scheduler/internal/scheduler/node_watch.go b/persys-scheduler/internal/scheduler/node_watch.go new file mode 100644 index 0000000..aa9b80f --- /dev/null +++ b/persys-scheduler/internal/scheduler/node_watch.go @@ -0,0 +1,121 @@ +package scheduler + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/logging" + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" + clientv3 "go.etcd.io/etcd/client/v3" +) + +var nodeWatchLogger = logging.C("scheduler.node_watch") + +// nodeWatchResyncBackoff is how long StartNodeWatch waits before retrying +// after the watch stream ends (connection loss, etcd compaction ahead of +// our revision, etc.) or a resync attempt fails outright. +const nodeWatchResyncBackoff = 2 * time.Second + +// StartNodeWatch runs a loop that keeps the in-memory node cache +// (Scheduler.cacheNodes) continuously up to date via an etcd watch on the +// node prefix, instead of the cache only being populated as a side effect +// of whatever GetNodes()/GetNodeByID() calls happen to occur. This is what +// candidateNodeSnapshot (used by selectNodeForWorkload) relies on to avoid +// a full etcd scan + unmarshal-every-node on every single placement +// decision. +// +// Like the scheduler's other Start* background loops (StartDriftDetection, +// Reconciler.StartReconciliationLoop), this call blocks until ctx is +// cancelled — callers run it in its own goroutine, tracked by the same +// wait group as the other background loops. It resyncs and re-watches +// automatically if the underlying watch stream ends for any reason. +func (s *Scheduler) StartNodeWatch(ctx context.Context) { + nodeWatchLogger.Info("starting node watch") + for { + select { + case <-ctx.Done(): + nodeWatchLogger.Info("stopping node watch") + return + default: + } + + if err := s.watchNodesOnce(ctx); err != nil && ctx.Err() == nil { + nodeWatchLogger.WithError(err).Warn("node watch stream ended, will resync") + } + // The cache is now stale relative to reality until the next + // successful resync below; candidateNodeSnapshot falls back to a + // live scan while nodeCacheReady is false, so placement keeps + // working (just without the fast path) during the gap. + s.nodeCacheReady.Store(false) + + select { + case <-ctx.Done(): + nodeWatchLogger.Info("stopping node watch") + return + case <-time.After(nodeWatchResyncBackoff): + } + } +} + +// watchNodesOnce does a full resync of the node cache from etcd, then +// watches from that revision forward, applying each event to the cache as +// it arrives. Returns when the watch channel closes or errors; the caller +// (runNodeWatch) handles backoff and resync. +func (s *Scheduler) watchNodesOnce(ctx context.Context) error { + resp, err := s.RetryableEtcdGet(nodesPrefix, clientv3.WithPrefix()) + if err != nil { + return fmt.Errorf("initial node resync failed: %w", err) + } + + fresh := make(map[string]models.Node, len(resp.Kvs)) + for _, kv := range resp.Kvs { + if isNodeStatusSubKey(string(kv.Key)) { + continue + } + var node models.Node + if err := json.Unmarshal(kv.Value, &node); err != nil { + nodeWatchLogger.WithError(err).WithField("key", string(kv.Key)).Warn("failed to unmarshal node during resync") + continue + } + fresh[node.NodeID] = node + } + s.withCacheLock(func() { + // Replace wholesale rather than merge: this is a full resync, so + // any node that no longer exists in etcd should no longer exist + // in the cache either. + s.cacheNodes = fresh + }) + s.nodeCacheReady.Store(true) + nodeWatchLogger.WithField("node_count", len(fresh)).Debug("node cache resynced") + + watchCh := s.etcdClient.Watch(ctx, nodesPrefix, clientv3.WithPrefix(), clientv3.WithRev(resp.Header.Revision+1)) + for wresp := range watchCh { + if err := wresp.Err(); err != nil { + return fmt.Errorf("node watch error: %w", err) + } + for _, ev := range wresp.Events { + key := string(ev.Kv.Key) + if isNodeStatusSubKey(key) { + continue + } + nodeID := strings.TrimPrefix(key, nodesPrefix) + switch ev.Type { + case clientv3.EventTypePut: + var node models.Node + if err := json.Unmarshal(ev.Kv.Value, &node); err != nil { + nodeWatchLogger.WithError(err).WithField("node_id", nodeID).Warn("failed to unmarshal node watch event") + continue + } + s.cacheNode(node) + case clientv3.EventTypeDelete: + s.withCacheLock(func() { delete(s.cacheNodes, nodeID) }) + } + } + } + // Channel closed without an error surfaced through wresp.Err() — treat + // as a stream end like any other, so the caller resyncs. + return fmt.Errorf("node watch channel closed") +} From 2b12d9bed279a80b8febced3fd945d8845bedd6a Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:48:18 +0330 Subject: [PATCH 11/31] Feat: Refactor node update logic to use CAS for safer concurrent modifications --- .../internal/scheduler/node_control.go | 407 ++++++++++-------- 1 file changed, 237 insertions(+), 170 deletions(-) diff --git a/persys-scheduler/internal/scheduler/node_control.go b/persys-scheduler/internal/scheduler/node_control.go index cd3da23..4453364 100644 --- a/persys-scheduler/internal/scheduler/node_control.go +++ b/persys-scheduler/internal/scheduler/node_control.go @@ -14,60 +14,39 @@ import ( var nodeLogger = logging.C("scheduler.node_control") func (s *Scheduler) UpdateNodeHeartbeat(nodeID, status string, availableCPU float64, availableMemory int64) error { - if err := s.requireWritable(); err != nil { - return err - } - resp, err := s.RetryableEtcdGet("/nodes/" + nodeID) - if err != nil { - return fmt.Errorf("failed to get node %s from etcd: %w", nodeID, err) - } - if resp == nil || len(resp.Kvs) == 0 { - return fmt.Errorf("node %s not found", nodeID) - } - - var node models.Node - if err := json.Unmarshal(resp.Kvs[0].Value, &node); err != nil { - return fmt.Errorf("failed to unmarshal node %s: %w", nodeID, err) - } - - node.LastHeartbeat = time.Now().UTC() - if strings.TrimSpace(status) != "" { - previousStatus := node.Status - if strings.EqualFold(previousStatus, "Draining") && strings.EqualFold(status, "Ready") { - status = "Draining" - } - node.Status = status - if strings.EqualFold(status, "Ready") { - if strings.EqualFold(previousStatus, "Ready") { - node.StatusReason = "heartbeat received" - } else { - node.StatusReason = fmt.Sprintf("status transition %s -> %s via heartbeat", previousStatus, status) - nodeLogger.WithField("node_id", nodeID).Info("node recovered to Ready via heartbeat") + _, err := s.updateNodeCAS(nodeID, func(node *models.Node) { + node.LastHeartbeat = time.Now().UTC() + if strings.TrimSpace(status) != "" { + previousStatus := node.Status + if strings.EqualFold(previousStatus, "Draining") && strings.EqualFold(status, "Ready") { + status = "Draining" + } + node.Status = status + if strings.EqualFold(status, "Ready") { + if strings.EqualFold(previousStatus, "Ready") { + node.StatusReason = "heartbeat received" + } else { + node.StatusReason = fmt.Sprintf("status transition %s -> %s via heartbeat", previousStatus, status) + nodeLogger.WithField("node_id", nodeID).Info("node recovered to Ready via heartbeat") + } + node.StatusUpdatedBy = "heartbeat" + node.StatusUpdatedAt = time.Now().UTC() + } else if !strings.EqualFold(previousStatus, status) { + node.StatusReason = "heartbeat status transition" + node.StatusUpdatedBy = "heartbeat" + node.StatusUpdatedAt = time.Now().UTC() } - node.StatusUpdatedBy = "heartbeat" - node.StatusUpdatedAt = time.Now().UTC() - } else if !strings.EqualFold(previousStatus, status) { - node.StatusReason = "heartbeat status transition" - node.StatusUpdatedBy = "heartbeat" - node.StatusUpdatedAt = time.Now().UTC() } - } - if availableCPU >= 0 { - node.AvailableCPU = availableCPU - } - if availableMemory >= 0 { - node.AvailableMemory = availableMemory - } - - updatedNodeJSON, err := json.Marshal(node) + if availableCPU >= 0 { + node.AvailableCPU = availableCPU + } + if availableMemory >= 0 { + node.AvailableMemory = availableMemory + } + }) if err != nil { - return fmt.Errorf("failed to marshal node %s: %w", nodeID, err) - } - if err := s.RetryableEtcdPut("/nodes/"+nodeID, string(updatedNodeJSON)); err != nil { return fmt.Errorf("failed to update node %s heartbeat: %w", nodeID, err) } - _ = s.RetryableEtcdPut("/nodes/"+nodeID+"/status", node.Status) - s.cacheNode(node) return nil } @@ -81,6 +60,9 @@ func (s *Scheduler) MarkNodeDraining(nodeID, reason, source string) (int, error) if err != nil { return 0, err } + s.emitEvent("NodeDraining", "", nodeID, defaultString(reason, "node drain requested"), map[string]interface{}{ + "source": defaultString(source, "operator"), + }) relocated, err := s.RelocateWorkloadsFromNode(node.NodeID, "node draining") if err != nil { return relocated, err @@ -95,7 +77,13 @@ func (s *Scheduler) MarkNodeReady(nodeID, reason, source string) error { node.StatusUpdatedBy = defaultString(source, "operator") node.StatusUpdatedAt = time.Now().UTC() }) - return err + if err != nil { + return err + } + s.emitEvent("NodeReady", "", nodeID, defaultString(reason, "node returned to service"), map[string]interface{}{ + "source": defaultString(source, "operator"), + }) + return nil } func (s *Scheduler) TaintNode(nodeID string, taint models.NodeTaint, source string) error { @@ -117,77 +105,156 @@ func (s *Scheduler) TaintNode(nodeID string, taint models.NodeTaint, source stri node.StatusUpdatedBy = defaultString(source, "operator") node.StatusUpdatedAt = time.Now().UTC() }) - return err + if err != nil { + return err + } + s.emitEvent("NodeTainted", "", nodeID, "node taint applied", map[string]interface{}{ + "source": defaultString(source, "operator"), + "key": strings.TrimSpace(taint.Key), + "effect": normalizeTaintEffect(taint.Effect), + "value": taint.Value, + }) + return nil } func (s *Scheduler) UntaintNode(nodeID, key, effect, source string) error { + key = strings.TrimSpace(key) + effectNorm := normalizeTaintEffect(effect) _, err := s.updateNode(nodeID, func(node *models.Node) { - effect = strings.TrimSpace(effect) - key = strings.TrimSpace(key) filtered := node.Taints[:0] - for _, taint := range node.Taints { - if taint.Key == key && (effect == "" || strings.EqualFold(taint.Effect, effect)) { + for _, t := range node.Taints { + if t.Key == key && (effectNorm == "" || strings.EqualFold(t.Effect, effectNorm)) { continue } - filtered = append(filtered, taint) + filtered = append(filtered, t) } node.Taints = filtered - node.StatusReason = fmt.Sprintf("taint %s:%s removed", key, effect) + node.StatusReason = "taint removed" node.StatusUpdatedBy = defaultString(source, "operator") node.StatusUpdatedAt = time.Now().UTC() }) - return err + if err != nil { + return err + } + s.emitEvent("NodeUntainted", "", nodeID, "node taint removed", map[string]interface{}{ + "source": defaultString(source, "operator"), + "key": key, + "effect": effectNorm, + }) + return nil } func (s *Scheduler) SetNodeLabel(nodeID, key, value, source string) error { + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) _, err := s.updateNode(nodeID, func(node *models.Node) { if node.Labels == nil { node.Labels = map[string]string{} } - node.Labels[strings.TrimSpace(key)] = strings.TrimSpace(value) - node.StatusReason = fmt.Sprintf("label %s set", strings.TrimSpace(key)) + node.Labels[key] = value + node.StatusReason = fmt.Sprintf("label %s set", key) node.StatusUpdatedBy = defaultString(source, "operator") node.StatusUpdatedAt = time.Now().UTC() }) - return err + if err != nil { + return err + } + s.emitEvent("NodeLabelSet", "", nodeID, fmt.Sprintf("label %s set", key), map[string]interface{}{ + "source": defaultString(source, "operator"), + "key": key, + "value": value, + }) + return nil } func (s *Scheduler) DeleteNodeLabel(nodeID, key, source string) error { + key = strings.TrimSpace(key) _, err := s.updateNode(nodeID, func(node *models.Node) { - delete(node.Labels, strings.TrimSpace(key)) - node.StatusReason = fmt.Sprintf("label %s deleted", strings.TrimSpace(key)) + delete(node.Labels, key) + node.StatusReason = fmt.Sprintf("label %s deleted", key) node.StatusUpdatedBy = defaultString(source, "operator") node.StatusUpdatedAt = time.Now().UTC() }) - return err + if err != nil { + return err + } + s.emitEvent("NodeLabelDeleted", "", nodeID, fmt.Sprintf("label %s deleted", key), map[string]interface{}{ + "source": defaultString(source, "operator"), + "key": key, + }) + return nil } +// updateNode applies mutate to the current node record and persists it. +// Kept as a thin wrapper over updateNodeCAS for the existing call sites +// (MarkNodeDraining, MarkNodeReady, TaintNode, UntaintNode, SetNodeLabel, +// DeleteNodeLabel) that don't need anything beyond "read, mutate, write +// safely." func (s *Scheduler) updateNode(nodeID string, mutate func(*models.Node)) (models.Node, error) { + return s.updateNodeCAS(nodeID, mutate) +} + +// maxCASConflictRetries bounds how many times updateNodeCAS will re-read +// and re-apply a mutation after losing a compare-and-swap race before +// giving up. Node records are small and writers are relatively few per +// node (heartbeat, reconciler failover, node monitor, operator actions), so +// a handful of retries is expected to be enough even under load; a caller +// that exhausts this is almost certainly hitting sustained concurrent +// writes to the same node and should surface that as an error rather than +// retry forever. +const maxCASConflictRetries = 5 + +// updateNodeCAS reads a node record, applies mutate, and writes it back +// using an etcd compare-and-swap on ModRevision, re-reading and re-applying +// the mutation if another writer updated the record in between (rather than +// silently overwriting whatever that writer just changed). This closes the +// lost-update window that a plain get-then-put has whenever more than one +// goroutine can touch the same node record around the same time — which is +// routine here: heartbeats, the parallelized reconciler's failover path, +// and the node monitor can all race on the same NotReady/Ready node. +func (s *Scheduler) updateNodeCAS(nodeID string, mutate func(*models.Node)) (models.Node, error) { if err := s.requireWritable(); err != nil { return models.Node{}, err } - resp, err := s.RetryableEtcdGet("/nodes/" + nodeID) - if err != nil { - return models.Node{}, fmt.Errorf("failed to get node %s from etcd: %w", nodeID, err) - } - if resp == nil || len(resp.Kvs) == 0 { - return models.Node{}, fmt.Errorf("node %s not found", nodeID) - } - var node models.Node - if err := json.Unmarshal(resp.Kvs[0].Value, &node); err != nil { - return models.Node{}, fmt.Errorf("failed to unmarshal node %s: %w", nodeID, err) - } - mutate(&node) - payload, err := json.Marshal(node) - if err != nil { - return models.Node{}, fmt.Errorf("failed to marshal node %s: %w", nodeID, err) - } - if err := s.RetryableEtcdPut("/nodes/"+nodeID, string(payload)); err != nil { - return models.Node{}, fmt.Errorf("failed to persist node %s: %w", nodeID, err) + key := "/nodes/" + nodeID + var lastConflictErr error + for attempt := 0; attempt < maxCASConflictRetries; attempt++ { + resp, err := s.RetryableEtcdGet(key) + if err != nil { + return models.Node{}, fmt.Errorf("failed to get node %s from etcd: %w", nodeID, err) + } + if resp == nil || len(resp.Kvs) == 0 { + return models.Node{}, fmt.Errorf("node %s not found", nodeID) + } + var node models.Node + if err := json.Unmarshal(resp.Kvs[0].Value, &node); err != nil { + return models.Node{}, fmt.Errorf("failed to unmarshal node %s: %w", nodeID, err) + } + modRevision := resp.Kvs[0].ModRevision + + mutate(&node) + payload, err := json.Marshal(node) + if err != nil { + return models.Node{}, fmt.Errorf("failed to marshal node %s: %w", nodeID, err) + } + + ok, err := s.RetryableEtcdCASPut(key, string(payload), modRevision) + if err != nil { + return models.Node{}, fmt.Errorf("failed to persist node %s: %w", nodeID, err) + } + if !ok { + lastConflictErr = fmt.Errorf("node %s changed concurrently (attempt %d)", nodeID, attempt+1) + nodeLogger.WithFields(logrus.Fields{ + "node_id": nodeID, + "attempt": attempt + 1, + }).Debug("node CAS conflict, retrying with fresh read") + continue + } + _ = s.RetryableEtcdPut(key+"/status", node.Status) + s.cacheNode(node) + return node, nil } - _ = s.RetryableEtcdPut("/nodes/"+nodeID+"/status", node.Status) - s.cacheNode(node) - return node, nil + return models.Node{}, fmt.Errorf("node %s: too many concurrent update conflicts: %w", nodeID, lastConflictErr) } func normalizeTaintEffect(effect string) string { @@ -213,6 +280,16 @@ func (s *Scheduler) MarkNodeWorkloadTypeUnsupported(nodeID, workloadType, reason if err := s.requireWritable(); err != nil { return err } + want := canonicalWorkloadType(workloadType) + if want == "" { + return nil + } + + // Pre-check against a snapshot read so we can skip the write (and log) + // entirely when this node's capability list already reflects the + // rejection. updateNodeCAS below re-derives the same decision against + // a fresh read, so a race here just costs one extra CAS attempt, never + // a missed update. resp, err := s.RetryableEtcdGet("/nodes/" + nodeID) if err != nil { return fmt.Errorf("failed to get node %s from etcd: %w", nodeID, err) @@ -220,52 +297,45 @@ func (s *Scheduler) MarkNodeWorkloadTypeUnsupported(nodeID, workloadType, reason if resp == nil || len(resp.Kvs) == 0 { return fmt.Errorf("node %s not found", nodeID) } - - var node models.Node - if err := json.Unmarshal(resp.Kvs[0].Value, &node); err != nil { + var snapshot models.Node + if err := json.Unmarshal(resp.Kvs[0].Value, &snapshot); err != nil { return fmt.Errorf("failed to unmarshal node %s: %w", nodeID, err) } - - want := canonicalWorkloadType(workloadType) - if want == "" { + if !workloadTypeCapabilityNeedsUpdate(snapshot, want) { return nil } - if len(node.SupportedWorkloadTypes) == 0 { - // Capability list absent on legacy node records. Only apply strict downgrade for VM. - if want == "vm" { + var updatedCapabilities []string + node, err := s.updateNodeCAS(nodeID, func(node *models.Node) { + if !workloadTypeCapabilityNeedsUpdate(*node, want) { + return + } + if len(node.SupportedWorkloadTypes) == 0 { + // Capability list absent on legacy node records. Only apply strict downgrade for VM. node.SupportedWorkloadTypes = []string{"container", "compose"} } else { - return nil - } - } else { - filtered := make([]string, 0, len(node.SupportedWorkloadTypes)) - changed := false - for _, t := range node.SupportedWorkloadTypes { - if canonicalWorkloadType(t) == want { - changed = true - continue + filtered := make([]string, 0, len(node.SupportedWorkloadTypes)) + for _, t := range node.SupportedWorkloadTypes { + if canonicalWorkloadType(t) == want { + continue + } + filtered = append(filtered, canonicalWorkloadType(t)) } - filtered = append(filtered, canonicalWorkloadType(t)) + node.SupportedWorkloadTypes = filtered } - if !changed { - return nil - } - node.SupportedWorkloadTypes = filtered - } - - node.StatusReason = fmt.Sprintf("runtime for %s unavailable on agent: %s", want, strings.TrimSpace(reason)) - node.StatusUpdatedBy = "reconciler" - node.StatusUpdatedAt = time.Now().UTC() - - payload, err := json.Marshal(node) + node.StatusReason = fmt.Sprintf("runtime for %s unavailable on agent: %s", want, strings.TrimSpace(reason)) + node.StatusUpdatedBy = "reconciler" + node.StatusUpdatedAt = time.Now().UTC() + updatedCapabilities = node.SupportedWorkloadTypes + }) if err != nil { - return fmt.Errorf("failed to marshal node %s: %w", nodeID, err) - } - if err := s.RetryableEtcdPut("/nodes/"+nodeID, string(payload)); err != nil { return fmt.Errorf("failed to persist workload type capability update for node %s: %w", nodeID, err) } - s.cacheNode(node) + if updatedCapabilities == nil { + // mutate ran (possibly via a CAS retry) but found nothing left to + // change on the freshest read. + return nil + } nodeLogger.WithFields(logrus.Fields{ "node_id": nodeID, "capability": node.SupportedWorkloadTypes, @@ -273,71 +343,68 @@ func (s *Scheduler) MarkNodeWorkloadTypeUnsupported(nodeID, workloadType, reason return nil } +// workloadTypeCapabilityNeedsUpdate reports whether node's capability list +// still needs to be updated to reflect that want is unsupported. +func workloadTypeCapabilityNeedsUpdate(node models.Node, want string) bool { + if len(node.SupportedWorkloadTypes) == 0 { + return want == "vm" + } + for _, t := range node.SupportedWorkloadTypes { + if canonicalWorkloadType(t) == want { + return true + } + } + return false +} + func (s *Scheduler) markNodeNotReady(nodeID, reason, source string) error { if err := s.requireWritable(); err != nil { return err } - resp, err := s.RetryableEtcdGet("/nodes/" + nodeID) - if err != nil { - return fmt.Errorf("failed to get node %s from etcd: %w", nodeID, err) - } - if resp == nil || len(resp.Kvs) == 0 { - return fmt.Errorf("node %s not found", nodeID) - } - var node models.Node - if err := json.Unmarshal(resp.Kvs[0].Value, &node); err != nil { - return fmt.Errorf("failed to unmarshal node %s: %w", nodeID, err) + // Cheap pre-check to avoid a needless write (and etcd revision bump) + // when the record already reflects this exact NotReady state. If this + // races with a concurrent writer, the worst case is one extra CAS + // write below — updateNodeCAS re-reads internally, so we never skip a + // write that was actually needed. + if resp, err := s.RetryableEtcdGet("/nodes/" + nodeID); err == nil && resp != nil && len(resp.Kvs) > 0 { + var current models.Node + if json.Unmarshal(resp.Kvs[0].Value, ¤t) == nil && strings.EqualFold(current.Status, "NotReady") { + incomingReason := strings.TrimSpace(reason) + if strings.TrimSpace(current.StatusReason) != "" && + strings.TrimSpace(current.StatusUpdatedBy) != "" && + !current.StatusUpdatedAt.IsZero() && + (incomingReason == "" || strings.EqualFold(strings.TrimSpace(current.StatusReason), incomingReason)) && + strings.EqualFold(strings.TrimSpace(current.StatusUpdatedBy), source) { + return nil + } + } } - if strings.EqualFold(node.Status, "NotReady") { - incomingReason := strings.TrimSpace(reason) - // Keep current record if metadata is complete and reason/source did not change. - if strings.TrimSpace(node.StatusReason) != "" && - strings.TrimSpace(node.StatusUpdatedBy) != "" && - !node.StatusUpdatedAt.IsZero() && - (incomingReason == "" || strings.EqualFold(strings.TrimSpace(node.StatusReason), incomingReason)) && - strings.EqualFold(strings.TrimSpace(node.StatusUpdatedBy), source) { - return nil - } - node.StatusReason = incomingReason + wasReady := false + _, err := s.updateNodeCAS(nodeID, func(node *models.Node) { + wasReady = !strings.EqualFold(node.Status, "NotReady") + node.Status = "NotReady" + node.StatusReason = strings.TrimSpace(reason) if node.StatusReason == "" { node.StatusReason = "node marked NotReady" } node.StatusUpdatedBy = source node.StatusUpdatedAt = time.Now().UTC() - payload, err := json.Marshal(node) - if err != nil { - return fmt.Errorf("failed to marshal node %s: %w", nodeID, err) - } - if err := s.RetryableEtcdPut("/nodes/"+nodeID, string(payload)); err != nil { - return fmt.Errorf("failed to persist NotReady metadata for node %s: %w", nodeID, err) - } - _ = s.RetryableEtcdPut("/nodes/"+nodeID+"/status", node.Status) - s.cacheNode(node) - return nil - } - node.Status = "NotReady" - node.StatusReason = strings.TrimSpace(reason) - if node.StatusReason == "" { - node.StatusReason = "node marked NotReady" - } - node.StatusUpdatedBy = source - node.StatusUpdatedAt = time.Now().UTC() - payload, err := json.Marshal(node) + }) if err != nil { - return fmt.Errorf("failed to marshal node %s: %w", nodeID, err) - } - if err := s.RetryableEtcdPut("/nodes/"+nodeID, string(payload)); err != nil { return fmt.Errorf("failed to persist NotReady for node %s: %w", nodeID, err) } - _ = s.RetryableEtcdPut("/nodes/"+nodeID+"/status", node.Status) - s.cacheNode(node) - s.emitEvent("NodeLost", "", nodeID, reason, nil) - nodeLogger.WithFields(logrus.Fields{ - "node_id": nodeID, - "source": source, - "reason": reason, - }).Warn("marked node NotReady") + if wasReady { + s.emitEvent("NodeLost", "", nodeID, reason, map[string]interface{}{ + "status": "NotReady", + "source": source, + }) + nodeLogger.WithFields(logrus.Fields{ + "node_id": nodeID, + "source": source, + "reason": reason, + }).Warn("marked node NotReady") + } return nil } From 2c9b5a4c0677cb6e12b48031095b981ca93d4fd7 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:48:35 +0330 Subject: [PATCH 12/31] Feat: Enhance workload monitoring by filtering owned workloads and improving concurrency --- persys-scheduler/internal/scheduler/monitor.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/persys-scheduler/internal/scheduler/monitor.go b/persys-scheduler/internal/scheduler/monitor.go index 75f9ac0..7ec44f5 100644 --- a/persys-scheduler/internal/scheduler/monitor.go +++ b/persys-scheduler/internal/scheduler/monitor.go @@ -7,6 +7,7 @@ import ( "time" "github.com/persys-dev/persys-cloud/persys-scheduler/internal/logging" + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" "github.com/sirupsen/logrus" ) @@ -90,16 +91,23 @@ func (m *Monitor) MonitorWorkloads(ctx context.Context, interval time.Duration) monitorLogger.WithError(err).Error("failed to get workloads for monitoring") continue } + owned := make([]models.Workload, 0, len(workloads)) for _, workload := range workloads { if workload.Status == "Deleted" { continue } + if !m.scheduler.ownsNode(workload.NodeID) { + continue + } + owned = append(owned, workload) + } + runBounded(owned, m.scheduler.backgroundLoopConcurrency(), func(workload models.Workload) { if err := m.syncWorkloadStatus(workload.ID); err != nil { monitorLogger.WithError(err).WithFields(logrus.Fields{ "workload_id": workload.ID, }).Warn("failed to sync workload") } - } + }) if err := m.scheduler.RefreshStateMetrics(); err != nil { monitorLogger.WithError(err).Warn("failed to refresh scheduler state metrics") } From a43a6997a32d7cf10df2538c02db052f5a5d1443 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:48:50 +0330 Subject: [PATCH 13/31] Feat: Enhance mode transition logging and add bounded concurrency utility function --- persys-scheduler/internal/scheduler/mode.go | 79 ++++++++++++++++++--- 1 file changed, 70 insertions(+), 9 deletions(-) diff --git a/persys-scheduler/internal/scheduler/mode.go b/persys-scheduler/internal/scheduler/mode.go index 1bf025e..cb4ae52 100644 --- a/persys-scheduler/internal/scheduler/mode.go +++ b/persys-scheduler/internal/scheduler/mode.go @@ -3,6 +3,7 @@ package scheduler import ( "context" "fmt" + "sync" "time" "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" @@ -60,8 +61,9 @@ func (s *Scheduler) requireWritable() error { func (s *Scheduler) enterDegraded(reason string) { s.modeMu.Lock() - defer s.modeMu.Unlock() - if s.mode == ModeDegraded { + from := s.mode + if from == ModeDegraded { + s.modeMu.Unlock() return } s.mode = ModeDegraded @@ -75,13 +77,24 @@ func (s *Scheduler) enterDegraded(reason string) { Workloads: workloads, Assignments: assignments, } - schedulerLogger.WithField("reason", reason).Warn("scheduler entered degraded mode") + changedAt := s.modeChangedAt + s.modeMu.Unlock() + + schedulerLogger.WithField("reason", reason).WithField("from", string(from)).WithField("to", string(ModeDegraded)).Warn("scheduler entered degraded mode") + s.emitEvent("SchedulerModeChanged", "", "", reason, map[string]interface{}{ + "from": string(from), + "to": string(ModeDegraded), + "reason": reason, + "changed_at": changedAt.UTC().Format(time.RFC3339), + "writable": false, + }) } func (s *Scheduler) enterRecovery(reason string) { s.modeMu.Lock() - defer s.modeMu.Unlock() - if s.mode == ModeRecovery { + from := s.mode + if from == ModeRecovery { + s.modeMu.Unlock() return } s.mode = ModeRecovery @@ -97,20 +110,41 @@ func (s *Scheduler) enterRecovery(reason string) { Assignments: assignments, } } - schedulerLogger.WithField("reason", reason).Warn("scheduler entered recovery mode") + changedAt := s.modeChangedAt + s.modeMu.Unlock() + + schedulerLogger.WithField("reason", reason).WithField("from", string(from)).WithField("to", string(ModeRecovery)).Warn("scheduler entered recovery mode") + s.emitEvent("SchedulerModeChanged", "", "", reason, map[string]interface{}{ + "from": string(from), + "to": string(ModeRecovery), + "reason": reason, + "changed_at": changedAt.UTC().Format(time.RFC3339), + "writable": false, + }) } func (s *Scheduler) enterNormal(reason string) { s.modeMu.Lock() - defer s.modeMu.Unlock() - if s.mode == ModeNormal { + from := s.mode + if from == ModeNormal { + s.modeMu.Unlock() return } s.mode = ModeNormal s.modeReasonText = reason s.modeChangedAt = time.Now().UTC() s.frozen = nil - schedulerLogger.WithField("reason", reason).Info("scheduler returned to normal mode") + changedAt := s.modeChangedAt + s.modeMu.Unlock() + + schedulerLogger.WithField("reason", reason).WithField("from", string(from)).WithField("to", string(ModeNormal)).Info("scheduler returned to normal mode") + s.emitEvent("SchedulerModeChanged", "", "", reason, map[string]interface{}{ + "from": string(from), + "to": string(ModeNormal), + "reason": reason, + "changed_at": changedAt.UTC().Format(time.RFC3339), + "writable": true, + }) } func (s *Scheduler) ModeSnapshot() (OperatingMode, string, time.Time) { @@ -217,6 +251,33 @@ func cacheSnapshot[T any](in map[string]T) []T { return out } +// runBounded calls fn once per item, with at most concurrency goroutines in +// flight at a time, and waits for all of them to finish before returning. +// Shared by MonitorNodes, MonitorWorkloads, and detectDriftOnce so each +// doesn't reimplement the same semaphore+WaitGroup boilerplate; all three +// were originally plain sequential for-loops making one synchronous +// network call per item; this gives them the same bounded-concurrency +// treatment the reconciler already got, without changing the fan-out shape +// of what each loop iteration does. +func runBounded[T any](items []T, concurrency int, fn func(T)) { + if concurrency <= 0 { + concurrency = 1 + } + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + for _, item := range items { + item := item + wg.Add(1) + sem <- struct{}{} + go func() { + defer wg.Done() + defer func() { <-sem }() + fn(item) + }() + } + wg.Wait() +} + func (s *Scheduler) withCacheLock(fn func()) { s.cacheMu.Lock() defer s.cacheMu.Unlock() From 6b33ba484aafbe446dcdf88895742b4abb8f7ea0 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:49:03 +0330 Subject: [PATCH 14/31] Feat: Implement leader election mechanism for scheduler with automatic failover --- persys-scheduler/internal/scheduler/leader.go | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/leader.go diff --git a/persys-scheduler/internal/scheduler/leader.go b/persys-scheduler/internal/scheduler/leader.go new file mode 100644 index 0000000..b6de8a0 --- /dev/null +++ b/persys-scheduler/internal/scheduler/leader.go @@ -0,0 +1,153 @@ +package scheduler + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/google/uuid" + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/logging" + "go.etcd.io/etcd/client/v3/concurrency" +) + +var leaderLogger = logging.C("scheduler.leader") + +// leaderElectionKey is the etcd key prefix campaigned on. In failover mode +// (the default), every scheduler replica pointed at the same etcd cluster +// contends directly on this key, so exactly one of them holds it at a +// time. In active-active mode, this is used as a prefix — see +// Scheduler.electionKey in sharding.go — with each shard campaigning on +// its own derived key instead. +const leaderElectionKey = "/persys/scheduler/leader" + +// leaderSessionTTL controls how quickly a dead leader's session (and thus +// its lease on the election key) expires, letting a standby take over. +// Shorter TTLs fail over faster but are more sensitive to transient GC +// pauses/network blips causing an unnecessary handover; 15s is a +// reasonable middle ground and matches common etcd lease-based election +// examples. +const leaderSessionTTL = 15 // seconds + +// instanceID identifies this scheduler process in the election (visible in +// etcd as the campaign value, useful for "who's the leader right now" +// debugging) and is generated once at construction time. +func newInstanceID() string { + host, err := os.Hostname() + if err != nil || host == "" { + host = "unknown-host" + } + return fmt.Sprintf("%s-%s", host, uuid.NewString()[:8]) +} + +// IsLeader reports whether this scheduler instance currently holds the +// leader election. Background singleton loops (reconciliation, drift +// detection, node/workload monitoring, the node-cache watch) only run while +// this is true; the gRPC API (RegisterNode, Heartbeat, ApplyWorkload, ...) +// runs on every replica regardless, since those paths write directly to +// etcd (with CAS where it matters) and are safe to serve from any replica. +func (s *Scheduler) IsLeader() bool { + return s.isLeader.Load() +} + +// RunWithLeaderElection blocks until ctx is cancelled, campaigning for +// leadership on leaderElectionKey and invoking runAsLeader (in the calling +// goroutine) for as long as this instance holds it. If leadership is lost — +// session/lease expiry from a GC pause, network partition, process being +// too slow to renew, or a clean loss on shutdown — runAsLeader's context is +// cancelled and this re-campaigns from scratch. Callers should make +// runAsLeader itself react to context cancellation the way the existing +// Start* background loops already do. +// +// This is what turns the scheduler from "exactly one process, full stop" +// into "exactly one *active* process, with automatic failover" — multiple +// replicas can run the same binary pointed at the same etcd cluster, and +// only the elected one drives reconciliation/monitoring/drift detection at +// any given moment, so a crashed or restarting leader doesn't pause +// cluster-wide reconciliation for longer than one election round. +func (s *Scheduler) RunWithLeaderElection(ctx context.Context, runAsLeader func(leaderCtx context.Context)) { + for { + if ctx.Err() != nil { + return + } + if err := s.runOneElectionTerm(ctx, runAsLeader); err != nil { + leaderLogger.WithError(err).Warn("leader election term ended with error, retrying") + } + s.isLeader.Store(false) + if ctx.Err() != nil { + return + } + select { + case <-ctx.Done(): + return + case <-time.After(2 * time.Second): + } + } +} + +// runOneElectionTerm campaigns once, and if elected, runs runAsLeader until +// either ctx is cancelled or leadership is lost (session expiry / observed +// change of leader). Returns when this instance is no longer leader (or +// never became leader due to an error), so the caller can decide whether to +// retry. +func (s *Scheduler) runOneElectionTerm(ctx context.Context, runAsLeader func(leaderCtx context.Context)) error { + session, err := concurrency.NewSession(s.etcdClient, concurrency.WithTTL(leaderSessionTTL), concurrency.WithContext(ctx)) + if err != nil { + return fmt.Errorf("failed to create election session: %w", err) + } + defer session.Close() + + election := concurrency.NewElection(session, s.electionKey()) + + leaderLogger.WithFields(map[string]interface{}{ + "instance_id": s.instanceID, + "election_key": s.electionKey(), + }).Info("campaigning for scheduler leadership") + if err := election.Campaign(ctx, s.instanceID); err != nil { + return fmt.Errorf("campaign failed: %w", err) + } + + s.isLeader.Store(true) + leaderLogger.WithField("instance_id", s.instanceID).Info("elected scheduler leader") + s.emitEvent("LeaderElected", "", "", "scheduler instance elected leader", map[string]interface{}{ + "instance_id": s.instanceID, + "election_key": s.electionKey(), + }) + + leaderCtx, cancelLeader := context.WithCancel(ctx) + defer cancelLeader() + + done := make(chan struct{}) + go func() { + defer close(done) + runAsLeader(leaderCtx) + }() + + // Give up leadership if: the caller's context is cancelled (normal + // shutdown), the etcd session expires (we failed to renew our lease in + // time — e.g. a long GC pause or network partition), or runAsLeader + // itself returns (shouldn't normally happen since it's built from + // blocking Start*-style loops, but don't hang forever if it does). + select { + case <-ctx.Done(): + case <-session.Done(): + leaderLogger.Warn("leader election session expired, relinquishing leadership") + case <-done: + leaderLogger.Warn("leader-only work returned unexpectedly, relinquishing leadership") + } + + cancelLeader() + <-done // wait for runAsLeader to actually stop before we resign/close the session + wasLeader := s.isLeader.Swap(false) + if wasLeader { + s.emitEvent("LeaderLost", "", "", "scheduler instance relinquished leadership", map[string]interface{}{ + "instance_id": s.instanceID, + }) + } + + resignCtx, resignCancel := context.WithTimeout(context.Background(), 5*time.Second) + defer resignCancel() + _ = election.Resign(resignCtx) + + return nil +} From aed89f83a4723b5aa1498daedeb601c817d0ab48 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:49:19 +0330 Subject: [PATCH 15/31] Feat: Add event handling and testing for known scheduler event types --- persys-scheduler/internal/scheduler/events.go | 103 ++++++++++++++++++ .../internal/scheduler/events_test.go | 27 +++++ 2 files changed, 130 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/events.go create mode 100644 persys-scheduler/internal/scheduler/events_test.go diff --git a/persys-scheduler/internal/scheduler/events.go b/persys-scheduler/internal/scheduler/events.go new file mode 100644 index 0000000..e76bd40 --- /dev/null +++ b/persys-scheduler/internal/scheduler/events.go @@ -0,0 +1,103 @@ +package scheduler + +import ( + "context" + "time" + + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/logging" + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" +) + +var eventsLogger = logging.C("scheduler.events") + +var knownSchedulerEventTypes = map[string]struct{}{ + // Topology + "NodeJoined": {}, + "NodeLost": {}, + "NodeLeft": {}, + // Workload lifecycle + "WorkloadScheduled": {}, + "WorkloadFailed": {}, + "DriftDetected": {}, + "RetryTriggered": {}, + "Rescheduled": {}, + "Relocated": {}, + "RescheduleSkipped": {}, + // Control-plane mode (etcd health / recovery) + "SchedulerModeChanged": {}, + // Leader election + "LeaderElected": {}, + "LeaderLost": {}, + // Operator node control + "NodeDraining": {}, + "NodeReady": {}, + "NodeTainted": {}, + "NodeUntainted": {}, + "NodeLabelSet": {}, + "NodeLabelDeleted": {}, +} + +func isKnownSchedulerEventType(eventType string) bool { + _, ok := knownSchedulerEventTypes[eventType] + return ok +} + +// eventWatchResyncBackoff mirrors nodeWatchResyncBackoff (node_watch.go) — +// how long WatchEvents waits before retrying after its Redis stream watch +// ends for any reason (connection loss, Redis restart, etc). +const eventWatchResyncBackoff = 2 * time.Second + +// WatchEvents streams cluster-wide scheduler events to onEvent as they're +// emitted (see Scheduler.emitEvent), starting from the current set of +// recent events (up to limit) and then following new ones live — with no +// gap and no duplicate delivery across that replay-to-live boundary, +// since readEventsFromStream returns the exact stream ID to resume from. +// Blocks until ctx is cancelled, onEvent returns an error (typically +// because the receiving gRPC stream's client disconnected), or an +// unrecoverable Redis error occurs. +// +// Because emitEvent writes to the single Redis Stream shared by every +// scheduler replica (see redis_store.go), watching via any one replica's +// Redis client sees every event emitted cluster-wide — this is what makes +// the events genuinely cluster-wide rather than per-replica, the same +// property the previous etcd-watch-based version had, without the etcd +// load. +// +// If Redis is unavailable when this is called, replay returns no history +// (readEventsFromStream degrades gracefully) and the live-watch loop +// below retries on eventWatchResyncBackoff until Redis comes back — a +// client connected during a Redis outage just sees the stream resume once +// it recovers, rather than an error. +func (s *Scheduler) WatchEvents(ctx context.Context, limit int64, onEvent func(models.SchedulerEvent) error) error { + recent, newestID := s.readEventsFromStream(limit) + // recent is newest-first; deliver oldest-first so a client sees a + // sensible chronological replay before switching to live events. + for i := len(recent) - 1; i >= 0; i-- { + if err := onEvent(recent[i]); err != nil { + return err + } + } + lastID := newestID + if lastID == "" { + lastID = "$" // nothing replayed; start from only new events + } + + for { + if ctx.Err() != nil { + return ctx.Err() + } + nextID, err := s.watchEventStream(ctx, lastID, onEvent) + lastID = nextID + if err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + eventsLogger.WithError(err).Debug("event stream watch ended, resyncing") + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(eventWatchResyncBackoff): + } + } +} diff --git a/persys-scheduler/internal/scheduler/events_test.go b/persys-scheduler/internal/scheduler/events_test.go new file mode 100644 index 0000000..b53167e --- /dev/null +++ b/persys-scheduler/internal/scheduler/events_test.go @@ -0,0 +1,27 @@ +package scheduler + +import "testing" + +func TestKnownSchedulerEventTypesIncludesClusterTroubleshootingSignals(t *testing.T) { + expected := []string{ + "NodeJoined", + "NodeLost", + "NodeLeft", + "WorkloadScheduled", + "WorkloadFailed", + "DriftDetected", + "RetryTriggered", + "Rescheduled", + "Relocated", + } + + for _, eventType := range expected { + if !isKnownSchedulerEventType(eventType) { + t.Fatalf("event type %q should be recognized as a scheduler event", eventType) + } + } + + if isKnownSchedulerEventType("RandomNoise") { + t.Fatal("unknown event types should not be accepted into the scheduler catalog") + } +} From 2d564e33e0f1fc823d9dbeb5f5ce586015ff8230 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:49:34 +0330 Subject: [PATCH 16/31] Feat: Implement RetryableEtcdCASPut for conditional key updates with retries --- persys-scheduler/internal/scheduler/etcd.go | 33 +++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/persys-scheduler/internal/scheduler/etcd.go b/persys-scheduler/internal/scheduler/etcd.go index b95401b..1716821 100644 --- a/persys-scheduler/internal/scheduler/etcd.go +++ b/persys-scheduler/internal/scheduler/etcd.go @@ -59,6 +59,39 @@ func (s *Scheduler) RetryableEtcdGet(key string, opts ...clientv3.OpOption) (*cl return nil, fmt.Errorf("failed to get key %s after %d attempts: %v", key, maxRetries+1, err) } +// RetryableEtcdCASPut writes value to key only if the key's current +// ModRevision still equals expectedModRevision — i.e. nothing else has +// written to it since the caller read it. ok=false (with err=nil) means the +// compare-and-swap lost the race; the caller should re-read and retry its +// read-modify-write rather than assume the write landed. +func (s *Scheduler) RetryableEtcdCASPut(key, value string, expectedModRevision int64) (ok bool, err error) { + if err := s.requireWritable(); err != nil { + return false, err + } + var lastErr error + for attempt := 0; attempt <= maxRetries; attempt++ { + ctx, cancel := context.WithTimeout(context.Background(), etcdTimeout) + txnResp, txnErr := s.etcdClient.Txn(ctx). + If(clientv3.Compare(clientv3.ModRevision(key), "=", expectedModRevision)). + Then(clientv3.OpPut(key, value)). + Commit() + cancel() + if txnErr == nil { + return txnResp.Succeeded, nil + } + lastErr = txnErr + etcdLogger.WithError(txnErr).WithFields(logrus.Fields{ + "attempt": attempt + 1, + "key": key, + }).Warn("etcd CAS put attempt failed") + if attempt < maxRetries { + time.Sleep(retryWaitTime) + } + } + s.enterDegraded(fmt.Sprintf("etcd CAS write failure key=%s: %v", key, lastErr)) + return false, fmt.Errorf("failed CAS put key %s after %d attempts: %v", key, maxRetries+1, lastErr) +} + // RetryableEtcdDelete performs a delete operation with retries. func (s *Scheduler) RetryableEtcdDelete(key string, opts ...clientv3.OpOption) error { if err := s.requireWritable(); err != nil { From 414233f62c84a5a907d6f0e54223a09ad28e2448 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:49:55 +0330 Subject: [PATCH 17/31] Feat: Implement drift event deduplication and enhance orphan workload handling --- persys-scheduler/internal/scheduler/drift.go | 152 +++++++++++++++++-- 1 file changed, 142 insertions(+), 10 deletions(-) diff --git a/persys-scheduler/internal/scheduler/drift.go b/persys-scheduler/internal/scheduler/drift.go index 4e02df6..10a89e3 100644 --- a/persys-scheduler/internal/scheduler/drift.go +++ b/persys-scheduler/internal/scheduler/drift.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" agentpb "github.com/persys-dev/persys-cloud/persys-scheduler/internal/agentpb" @@ -15,6 +16,48 @@ import ( var driftLogger = logging.C("scheduler.drift") +// recentlyEmittedDrift suppresses repeated DriftDetected events for the same +// (node, workload, type, action) within a short window. Without this, every +// drift cycle re-emits for the same residual orphan while the agent is still +// catching up on an async delete. +var ( + driftEventMu sync.Mutex + driftEventStamp = map[string]time.Time{} +) + +const driftEventDedupeWindow = 2 * time.Minute + +func driftEventKey(nodeID, workloadID, driftType, action string) string { + return strings.ToLower(strings.TrimSpace(nodeID)) + "|" + + strings.TrimSpace(workloadID) + "|" + + strings.ToLower(strings.TrimSpace(driftType)) + "|" + + strings.ToLower(strings.TrimSpace(action)) +} + +func shouldEmitDriftEvent(nodeID, workloadID, driftType, action string) bool { + key := driftEventKey(nodeID, workloadID, driftType, action) + now := time.Now() + driftEventMu.Lock() + defer driftEventMu.Unlock() + if last, ok := driftEventStamp[key]; ok && now.Sub(last) < driftEventDedupeWindow { + return false + } + driftEventStamp[key] = now + if len(driftEventStamp) > 4096 { + for k, t := range driftEventStamp { + if now.Sub(t) > driftEventDedupeWindow { + delete(driftEventStamp, k) + } + } + } + return true +} + +func isDesiredDeleted(sw models.Workload) bool { + d := strings.ToLower(strings.TrimSpace(sw.DesiredState)) + return d == "deleted" || d == "deleting" +} + func (s *Scheduler) driftDetectInterval() time.Duration { if s.cfg != nil && s.cfg.SchedulerDriftDetectInterval > 0 { return s.cfg.SchedulerDriftDetectInterval @@ -86,14 +129,29 @@ func (s *Scheduler) detectDriftOnce(ctx context.Context) { selected[endpoint] = node } + candidateNodes := make([]models.Node, 0, len(selected)) for _, node := range selected { + if !s.ownsNode(node.NodeID) { + continue + } + candidateNodes = append(candidateNodes, node) + } + + // byID (expectedAll) is read AND written from compareNodeDrift/ + // remediateOrphanOnAgent (the latter memoizes a freshly-looked-up + // workload into it) — now that nodes are probed concurrently below, + // that shared map needs a mutex; it didn't when this loop was + // sequential. + var byIDMu sync.Mutex + + runBounded(candidateNodes, s.backgroundLoopConcurrency(), func(node models.Node) { if ok, reason := driftProbeEligible(node); !ok { driftLogger.WithFields(logrus.Fields{ "node_id": node.NodeID, "endpoint": s.grpcAddressForNode(node), "reason": reason, }).Debug("drift detection: skipping node") - continue + return } agentList, err := s.listWorkloadsFromNode(ctx, node) if err != nil { @@ -101,10 +159,10 @@ func (s *Scheduler) detectDriftOnce(ctx context.Context) { "node_id": node.NodeID, "endpoint": s.grpcAddressForNode(node), }).Warn("drift detection: failed to list workloads from agent") - continue + return } - s.compareNodeDrift(node, byNode[node.NodeID], byID, agentList) - } + s.compareNodeDrift(node, byNode[node.NodeID], byID, &byIDMu, agentList) + }) } func driftProbeEligible(node models.Node) (bool, string) { @@ -121,7 +179,7 @@ func driftProbeEligible(node models.Node) (bool, string) { return true, "" } -func (s *Scheduler) compareNodeDrift(node models.Node, expected map[string]models.Workload, expectedAll map[string]models.Workload, actual []*agentpb.WorkloadStatus) { +func (s *Scheduler) compareNodeDrift(node models.Node, expected map[string]models.Workload, expectedAll map[string]models.Workload, byIDMu *sync.Mutex, actual []*agentpb.WorkloadStatus) { if expected == nil { expected = map[string]models.Workload{} } @@ -146,7 +204,7 @@ func (s *Scheduler) compareNodeDrift(node models.Node, expected map[string]model sw, ok := expected[id] if !ok { orphan++ - action, actionErr := s.remediateOrphanOnAgent(node, id, aw, expectedAll) + action, actionErr := s.remediateOrphanOnAgent(node, id, aw, expectedAll, byIDMu) resolved := actionErr == nil s.markDrift(models.DriftRecord{ NodeID: node.NodeID, @@ -174,6 +232,35 @@ func (s *Scheduler) compareNodeDrift(node models.Node, expected map[string]model continue } + // Workloads marked Deleted/Deleting must not be "aligned" to the agent's + // still-running state — that fights the delete path. Force residual + // cleanup on the agent instead. + if isDesiredDeleted(sw) { + orphan++ + actionErr := s.deleteOrphanFromNode(node, id) + s.markDrift(models.DriftRecord{ + NodeID: node.NodeID, + WorkloadID: id, + DriftType: "orphan_on_agent", + DetectedAt: time.Now().UTC(), + SchedulerStatus: sw.Status, + AgentStatus: mapActualStateToSchedulerStatus(aw.GetActualState()), + Action: "delete_deleted_residual_on_agent", + Resolved: actionErr == nil, + LastError: errString(actionErr), + }) + driftLogger.WithFields(logrus.Fields{ + "node_id": node.NodeID, + "workload_id": id, + "desired": sw.DesiredState, + "agent_state": mapActualStateToSchedulerStatus(aw.GetActualState()), + "drift_type": "orphan_on_agent", + "action": "delete_deleted_residual_on_agent", + "resolved": actionErr == nil, + }).Warn("drift detected: residual of deleted workload still on agent") + continue + } + expectedStatus := strings.ToLower(strings.TrimSpace(sw.Status)) actualStatus := strings.ToLower(strings.TrimSpace(mapActualStateToSchedulerStatus(aw.GetActualState()))) if expectedStatus != "" && actualStatus != "" && expectedStatus != actualStatus { @@ -197,6 +284,7 @@ func (s *Scheduler) compareNodeDrift(node models.Node, expected map[string]model "agent_state": mapActualStateToSchedulerStatus(aw.GetActualState()), "drift_type": "state_mismatch", }).Warn("drift detected") + continue } if strings.TrimSpace(sw.RevisionID) != "" && strings.TrimSpace(aw.GetRevisionId()) != "" && @@ -225,7 +313,10 @@ func (s *Scheduler) compareNodeDrift(node models.Node, expected map[string]model } for id, sw := range expected { - if canonical, ok := expectedAll[id]; ok { + byIDMu.Lock() + canonical, ok := expectedAll[id] + byIDMu.Unlock() + if ok { owner := strings.TrimSpace(canonical.NodeID) if owner != "" && !strings.EqualFold(owner, node.NodeID) { // Workload was re-bound during this cycle; avoid conflicting remediation. @@ -317,6 +408,19 @@ func (s *Scheduler) markDrift(record models.DriftRecord) { if s.isWritable() { s.writeDriftRecord(record) } + // Always log, but rate-limit the user-visible DriftDetected event so a + // residual orphan (e.g. agent async delete still in flight) does not spam + // the events stream every drift interval. + if !shouldEmitDriftEvent(record.NodeID, record.WorkloadID, record.DriftType, record.Action) { + driftLogger.WithFields(logrus.Fields{ + "node_id": record.NodeID, + "workload_id": record.WorkloadID, + "drift_type": record.DriftType, + "action": record.Action, + "resolved": record.Resolved, + }).Debug("drift event suppressed (dedupe window)") + return + } s.emitEvent("DriftDetected", record.WorkloadID, record.NodeID, record.DriftType, map[string]interface{}{ "scheduler_status": record.SchedulerStatus, "agent_status": record.AgentStatus, @@ -333,19 +437,23 @@ func errString(err error) string { return err.Error() } -func (s *Scheduler) remediateOrphanOnAgent(node models.Node, workloadID string, aw *agentpb.WorkloadStatus, expectedAll map[string]models.Workload) (string, error) { +func (s *Scheduler) remediateOrphanOnAgent(node models.Node, workloadID string, aw *agentpb.WorkloadStatus, expectedAll map[string]models.Workload, byIDMu *sync.Mutex) (string, error) { if !s.isWritable() { return "control_plane_frozen", errControlPlaneFrozen } + byIDMu.Lock() sw, known := expectedAll[workloadID] + byIDMu.Unlock() if !known { latest, err := s.GetWorkloadByID(workloadID) if err == nil { sw = latest known = true if expectedAll != nil { + byIDMu.Lock() expectedAll[workloadID] = latest + byIDMu.Unlock() } } else if !isWorkloadMissingError(err) { return "operator_investigation", fmt.Errorf("failed to re-check unknown workload %s before delete: %w", workloadID, err) @@ -404,11 +512,35 @@ func (s *Scheduler) remediateOrphanOnAgent(node models.Node, workloadID string, } func (s *Scheduler) deleteOrphanFromNode(node models.Node, workloadID string) error { - _, err := s.deleteWorkloadFromNode(context.Background(), node, workloadID) + ctx := context.Background() + _, err := s.deleteWorkloadFromNode(ctx, node, workloadID) if err != nil && !isWorkloadStatusNotFound(err) { return err } - return nil + + // Agent DeleteWorkload may return success for an *async* queue submit + // before the runtime instance is gone. Probe status so we do not mark + // the residual as resolved while ListWorkloads still reports it. + deadline := time.Now().Add(8 * time.Second) + for { + _, stErr := s.getWorkloadStatusFromNode(ctx, node, workloadID) + if stErr != nil && isWorkloadStatusNotFound(stErr) { + s.clearDriftRecord(node.NodeID, workloadID, "orphan_on_agent") + return nil + } + if stErr != nil && isNodeUnreachableError(stErr) { + // Node dropped mid-delete; treat as best-effort success so we + // do not spin forever offline. + return nil + } + if time.Now().After(deadline) { + if stErr == nil { + return fmt.Errorf("orphan %s still present on agent %s after delete (async delete pending or failed)", workloadID, node.NodeID) + } + return fmt.Errorf("orphan %s delete verification failed on agent %s: %v", workloadID, node.NodeID, stErr) + } + time.Sleep(400 * time.Millisecond) + } } func (s *Scheduler) adoptOrphanedWorkload(node models.Node, workload models.Workload, aw *agentpb.WorkloadStatus, reason string, expectedAll map[string]models.Workload) error { From 5177fa4dd179e870b991b26005b250d504dd2c40 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:50:10 +0330 Subject: [PATCH 18/31] Feat: Enhance CoreDNS registration to support instance-specific keys and add deregistration on shutdown --- persys-scheduler/internal/scheduler/dns.go | 46 ++++++++++++++++++++-- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/persys-scheduler/internal/scheduler/dns.go b/persys-scheduler/internal/scheduler/dns.go index fefff96..1b9a2de 100644 --- a/persys-scheduler/internal/scheduler/dns.go +++ b/persys-scheduler/internal/scheduler/dns.go @@ -38,14 +38,32 @@ func reverseDomain(domain string) string { return strings.Join(parts, "/") } -// RegisterSchedulerInCoreDNS registers the scheduler in CoreDNS for service discovery. +// RegisterSchedulerInCoreDNS registers this scheduler replica in CoreDNS for +// service discovery by persys-gateway. +// +// Each replica writes to its own instance-keyed child (mirroring the same +// pattern UpdateCoreDNS already uses for agent nodes above), rather than a +// single shared key. Previously every replica wrote to exactly one fixed +// key (/skydns/.../persys-scheduler), so with more than one scheduler +// running, whichever replica registered last silently clobbered the +// others — the gateway would only ever resolve one instance, and not +// necessarily a healthy one, no matter how many replicas were actually +// running. CoreDNS's etcd backend treats sibling keys under a shared +// prefix as multiple records for that name, so persys-gateway resolving +// persys-scheduler. now gets one answer per registered (i.e. +// running) replica. func (s *Scheduler) RegisterSchedulerInCoreDNS(ipAddress string, port int) error { if ipAddress == "" || port == 0 { return fmt.Errorf("invalid scheduler data: IPAddress and Port are required") } + instanceKey := s.instanceID + if strings.TrimSpace(instanceKey) == "" { + instanceKey = ipAddress + } + // Register SRV record for _persys-scheduler. - srvKey := fmt.Sprintf("/skydns/%s/_persys-scheduler", reverseDomain(s.domain)) + srvKey := fmt.Sprintf("/skydns/%s/_persys-scheduler/%s", reverseDomain(s.domain), instanceKey) srvRecord := struct { Host string `json:"host"` Port int `json:"port"` @@ -64,7 +82,7 @@ func (s *Scheduler) RegisterSchedulerInCoreDNS(ipAddress string, port int) error } // Also register A record for direct IP lookup - aKey := fmt.Sprintf("/skydns/%s/persys-scheduler", reverseDomain(s.domain)) + aKey := fmt.Sprintf("/skydns/%s/persys-scheduler/%s", reverseDomain(s.domain), instanceKey) aRecord := struct { Host string `json:"host"` TTL int `json:"ttl"` @@ -83,6 +101,28 @@ func (s *Scheduler) RegisterSchedulerInCoreDNS(ipAddress string, port int) error return nil } +// DeregisterSchedulerSelfFromCoreDNS removes this replica's own CoreDNS +// records. Called on clean shutdown so a stopped replica doesn't linger as +// a dead A/SRV record for persys-gateway to route to until the 300s TTL +// happens to expire on its own. Best-effort: errors are logged, not +// returned, since a failed deregistration during shutdown shouldn't block +// the rest of the shutdown sequence, and a stale record self-heals via TTL +// either way. +func (s *Scheduler) DeregisterSchedulerSelfFromCoreDNS() { + instanceKey := s.instanceID + if strings.TrimSpace(instanceKey) == "" { + return + } + aKey := fmt.Sprintf("/skydns/%s/persys-scheduler/%s", reverseDomain(s.domain), instanceKey) + srvKey := fmt.Sprintf("/skydns/%s/_persys-scheduler/%s", reverseDomain(s.domain), instanceKey) + if err := s.RetryableEtcdDelete(aKey); err != nil { + schedulerLogger.WithError(err).Debug("failed to deregister scheduler A record on shutdown") + } + if err := s.RetryableEtcdDelete(srvKey); err != nil { + schedulerLogger.WithError(err).Debug("failed to deregister scheduler SRV record on shutdown") + } +} + // RegisterSchedulerSelfInCoreDNS registers this scheduler instance into CoreDNS. // It resolves advertise IP/port from env with sane defaults. func (s *Scheduler) RegisterSchedulerSelfInCoreDNS(defaultPort int) error { From 106b2a5ab1e5e59d64512e11405edfad896abdcb Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:50:24 +0330 Subject: [PATCH 19/31] Feat: Implement disk management API with CreateDisk, ListDisks, GetDisk, and DeleteDisk functions --- persys-scheduler/internal/scheduler/disk.go | 329 ++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/disk.go diff --git a/persys-scheduler/internal/scheduler/disk.go b/persys-scheduler/internal/scheduler/disk.go new file mode 100644 index 0000000..586e25c --- /dev/null +++ b/persys-scheduler/internal/scheduler/disk.go @@ -0,0 +1,329 @@ +package scheduler + +import ( + "fmt" + "strings" + "time" + + "github.com/google/uuid" + "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" +) + +// CreateDiskRequest is the control-plane API for standalone disk inventory. +type CreateDiskRequest struct { + Name string `json:"name"` + Driver string `json:"driver"` // local | ceph-rbd | nfs + SizeGB int64 `json:"size_gb"` + FSType string `json:"fs_type,omitempty"` + AccessMode string `json:"access_mode,omitempty"` + RetainPolicy string `json:"retain_policy,omitempty"` + // NodeID optional: only for local pre-pin. Normally local disks get node_id + // from workload placement. + NodeID string `json:"node_id,omitempty"` + MountPath string `json:"mount_path,omitempty"` +} + +// DiskView is the API view of a managed volume record (standalone or derived). +type DiskView struct { + ID string `json:"id"` + Name string `json:"name"` + Driver string `json:"driver"` + SizeGB int64 `json:"size_gb"` + FSType string `json:"fs_type,omitempty"` + AccessMode string `json:"access_mode,omitempty"` + RetainPolicy string `json:"retain_policy,omitempty"` + Phase string `json:"phase"` + LastError string `json:"last_error,omitempty"` + NodeID string `json:"node_id,omitempty"` + Device string `json:"device,omitempty"` + Standalone bool `json:"standalone"` + MountPath string `json:"mount_path,omitempty"` + WorkloadRefs []string `json:"workload_refs,omitempty"` + AttachedNodes []string `json:"attached_nodes,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +func diskViewFromRecord(r models.ManagedVolumeRecord) DiskView { + return DiskView{ + ID: r.ID, + Name: r.Name, + Driver: r.Driver, + SizeGB: r.SizeGB, + FSType: r.FSType, + AccessMode: r.AccessMode, + RetainPolicy: r.RetainPolicy, + Phase: r.Phase, + LastError: r.LastError, + NodeID: r.NodeID, + Device: r.Device, + Standalone: r.Standalone, + MountPath: r.MountPath, + WorkloadRefs: r.WorkloadRefs, + AttachedNodes: r.AttachedNodes, + CreatedAt: r.CreatedAt, + UpdatedAt: r.UpdatedAt, + } +} + +func canonicalDiskDriver(d string) string { + d = strings.ToLower(strings.TrimSpace(d)) + switch d { + case "ceph_rbd", "rbd", "ceph-rbd": + return "ceph-rbd" + case "local", "nfs": + return d + default: + return d + } +} + +// CreateDisk registers a standalone disk in etcd. +// +// ceph-rbd / nfs: phase Available (image/export provisioned on first agent attach). +// local: phase Pending until bound to the workload placement node. +func (s *Scheduler) CreateDisk(req CreateDiskRequest) (*DiskView, error) { + name := strings.TrimSpace(req.Name) + if name == "" { + return nil, fmt.Errorf("name is required") + } + driver := canonicalDiskDriver(req.Driver) + if driver == "" { + return nil, fmt.Errorf("driver is required (local|ceph-rbd|nfs)") + } + if req.SizeGB <= 0 { + req.SizeGB = 1 + } + retain := strings.TrimSpace(req.RetainPolicy) + if retain == "" { + retain = "Delete" + } + fs := strings.TrimSpace(req.FSType) + if fs == "" && driver != "local" { + fs = "ext4" + } + access := strings.TrimSpace(req.AccessMode) + if access == "" { + access = "ReadWriteOnce" + } + mount := strings.TrimSpace(req.MountPath) + if mount == "" { + mount = "/data" + } + + id := "disk-" + uuid.NewString() + phase := "Available" + if driver == "local" { + phase = "Pending" + } + + device := "" + if driver == "ceph-rbd" { + device = fmt.Sprintf("rbd:%s", name) + } + + now := time.Now().UTC() + rec := models.ManagedVolumeRecord{ + ID: id, + Name: name, + Driver: driver, + SizeGB: req.SizeGB, + AccessMode: access, + FSType: fs, + RetainPolicy: retain, + Phase: phase, + NodeID: strings.TrimSpace(req.NodeID), + Device: device, + Standalone: true, + MountPath: mount, + CreatedAt: now, + UpdatedAt: now, + } + if err := s.saveManagedVolumeRecord(rec); err != nil { + return nil, err + } + view := diskViewFromRecord(rec) + return &view, nil +} + +// ListDisks returns all managed volume records. +func (s *Scheduler) ListDisks() ([]DiskView, error) { + recs, err := s.listManagedVolumeRecords() + if err != nil { + return nil, err + } + out := make([]DiskView, 0, len(recs)) + for _, r := range recs { + out = append(out, diskViewFromRecord(r)) + } + return out, nil +} + +// GetDisk returns one disk by id. +func (s *Scheduler) GetDisk(id string) (*DiskView, error) { + rec, ok, err := s.getManagedVolumeRecord(strings.TrimSpace(id)) + if err != nil { + return nil, err + } + if !ok || rec == nil { + return nil, fmt.Errorf("disk %q not found", id) + } + v := diskViewFromRecord(*rec) + return &v, nil +} + +// DeleteDisk removes inventory. Refuses if still referenced unless force. +func (s *Scheduler) DeleteDisk(id string, force bool) error { + rec, ok, err := s.getManagedVolumeRecord(strings.TrimSpace(id)) + if err != nil { + return err + } + if !ok || rec == nil { + return nil + } + if !force && len(rec.WorkloadRefs) > 0 { + return fmt.Errorf("disk %q is bound to workloads %v; detach or pass force", id, rec.WorkloadRefs) + } + return s.deleteManagedVolumeRecord(rec.ID) +} + +// ExpandDiskRefsToManagedVolumes resolves disk ids into ManagedVolumeSpec for the agent. +// Local disks: stamp NodeID from assignedNode when empty. +func (s *Scheduler) ExpandDiskRefsToManagedVolumes(diskIDs []string, assignedNode string, workloadID string) ([]models.ManagedVolumeSpec, error) { + out := make([]models.ManagedVolumeSpec, 0, len(diskIDs)) + for _, id := range diskIDs { + id = strings.TrimSpace(id) + if id == "" { + continue + } + rec, ok, err := s.getManagedVolumeRecord(id) + if err != nil { + return nil, err + } + if !ok || rec == nil { + return nil, fmt.Errorf("disk %q not found", id) + } + refs := append([]string{}, rec.WorkloadRefs...) + found := false + for _, r := range refs { + if r == workloadID { + found = true + break + } + } + if !found && workloadID != "" { + refs = append(refs, workloadID) + } + rec.WorkloadRefs = refs + if rec.Driver == "local" && strings.TrimSpace(rec.NodeID) == "" && assignedNode != "" { + rec.NodeID = assignedNode + rec.Phase = "Provisioned" + } + if assignedNode != "" { + nfound := false + for _, n := range rec.AttachedNodes { + if n == assignedNode { + nfound = true + break + } + } + if !nfound { + rec.AttachedNodes = append(rec.AttachedNodes, assignedNode) + } + if rec.Phase == "Available" || rec.Phase == "Pending" || rec.Phase == "Provisioned" { + rec.Phase = "Attached" + } + } + if err := s.saveManagedVolumeRecord(*rec); err != nil { + return nil, err + } + + mount := rec.MountPath + if mount == "" { + mount = "/data" + } + out = append(out, models.ManagedVolumeSpec{ + Name: rec.Name, + Driver: rec.Driver, + SizeGB: rec.SizeGB, + AccessMode: rec.AccessMode, + FSType: rec.FSType, + MountPath: mount, + RetainPolicy: rec.RetainPolicy, + }) + } + return out, nil +} + +// AttachDiskIDsToWorkload reads persys.disk.ids (or disk_ids) from metadata +// and appends expanded managed volume specs onto the workload. +func (s *Scheduler) AttachDiskIDsToWorkload(workload *models.Workload) error { + if workload == nil { + return nil + } + ids := parseDiskIDsFromWorkload(*workload) + if len(ids) == 0 { + return nil + } + specs, err := s.ExpandDiskRefsToManagedVolumes(ids, strings.TrimSpace(workload.NodeID), workload.ID) + if err != nil { + return err + } + if workload.VM != nil { + workload.VM.ManagedVolumes = append(workload.VM.ManagedVolumes, specs...) + return nil + } + workload.ManagedVolumes = append(workload.ManagedVolumes, specs...) + return nil +} + +func parseDiskIDsFromWorkload(w models.Workload) []string { + raw := metaString(w.Metadata, "persys.disk.ids", "disk_ids") + if raw == "" { + raw = metaStringFromStringMap(w.Labels, "persys.disk.ids", "disk_ids") + } + if raw == "" && w.VM != nil { + raw = metaStringFromStringMap(w.VM.Metadata, "persys.disk.ids", "disk_ids") + } + if raw == "" { + return nil + } + parts := strings.FieldsFunc(raw, func(r rune) bool { + return r == ',' || r == ' ' || r == ';' + }) + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out +} + +func metaString(m map[string]interface{}, keys ...string) string { + if m == nil { + return "" + } + for _, k := range keys { + if v, ok := m[k]; ok && v != nil { + s := strings.TrimSpace(fmt.Sprint(v)) + if s != "" && s != "" { + return s + } + } + } + return "" +} + +func metaStringFromStringMap(m map[string]string, keys ...string) string { + if m == nil { + return "" + } + for _, k := range keys { + if s := strings.TrimSpace(m[k]); s != "" { + return s + } + } + return "" +} From 5a57727c44581c94faf4a232c27f62a6ab82ef0c Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:50:42 +0330 Subject: [PATCH 20/31] Feat: Implement bucket management API with CreateBucket, ListBuckets, GetBucket, DeleteBucket, and GetBucketAccess functions --- persys-scheduler/internal/scheduler/bucket.go | 1001 +++++++++++++++++ 1 file changed, 1001 insertions(+) create mode 100644 persys-scheduler/internal/scheduler/bucket.go diff --git a/persys-scheduler/internal/scheduler/bucket.go b/persys-scheduler/internal/scheduler/bucket.go new file mode 100644 index 0000000..74c23a6 --- /dev/null +++ b/persys-scheduler/internal/scheduler/bucket.go @@ -0,0 +1,1001 @@ +package scheduler + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "sync" + "time" + + pb "github.com/persys-dev/persys-cloud/pkg/vaultmanager/vaultmanagerv1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +// Object storage is RGW-authoritative. The scheduler only proxies S3/RGW +// (and optionally reads/writes bucket access material in Vault). No etcd. + +var bucketNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$`) + +// BucketView is the API-facing projection derived from RGW. +type BucketView struct { + ID string + Name string + Region string + Owner string + Endpoint string + Versioning bool + ObjectCount int64 + SizeBytes int64 + Phase string + LastError string + CreatedAt time.Time + UpdatedAt time.Time +} + +// BucketAccess is S3 client configuration for end users. +type BucketAccess struct { + Endpoint string + Region string + Bucket string + AccessKey string + SecretKey string + VaultPath string + S3URL string +} + +// ObjectInfo is one object listing entry. +type ObjectInfo struct { + Key string + SizeBytes int64 + ETag string + LastModified string + StorageClass string +} + +// CreateBucketRequest is the control-plane input. +type CreateBucketRequest struct { + Name string + Region string + Versioning bool +} + +type rgwEnv struct { + endpoint string + region string + // Admin / control-plane credentials only (create/delete/list on RGW). + // User-facing access keys always come from Vault. + accessKey string + secretKey string +} + +func (s *Scheduler) rgwEnv() (rgwEnv, error) { + endpoint := strings.TrimSpace(os.Getenv("PERSYS_RGW_ENDPOINT")) + if endpoint == "" { + endpoint = strings.TrimSpace(os.Getenv("PERSYS_S3_ENDPOINT")) + } + region := strings.TrimSpace(os.Getenv("PERSYS_RGW_REGION")) + if region == "" { + region = "default" + } + ak := strings.TrimSpace(os.Getenv("PERSYS_RGW_ACCESS_KEY")) + if ak == "" { + ak = strings.TrimSpace(os.Getenv("PERSYS_S3_ACCESS_KEY")) + } + sk := strings.TrimSpace(os.Getenv("PERSYS_RGW_SECRET_KEY")) + if sk == "" { + sk = strings.TrimSpace(os.Getenv("PERSYS_S3_SECRET_KEY")) + } + if endpoint == "" || ak == "" || sk == "" { + return rgwEnv{}, fmt.Errorf("RGW not configured: set PERSYS_RGW_ENDPOINT, PERSYS_RGW_ACCESS_KEY, PERSYS_RGW_SECRET_KEY") + } + return rgwEnv{ + endpoint: endpoint, + region: region, + accessKey: ak, + secretKey: sk, + }, nil +} + +func (s *Scheduler) rgwClient() (*rgwClient, rgwEnv, error) { + env, err := s.rgwEnv() + if err != nil { + return nil, env, err + } + return &rgwClient{ + endpoint: env.endpoint, + region: env.region, + accessKey: env.accessKey, + secretKey: env.secretKey, + }, env, nil +} + +func vaultPathForBucket(name string) string { + base := strings.TrimSpace(os.Getenv("PERSYS_RGW_VAULT_PATH_PREFIX")) + if base == "" { + base = "secret/data/persys/rgw/buckets" + } + base = strings.Trim(base, "/") + return base + "/" + name +} + +// CreateBucket creates a bucket on RGW and stores user credentials in Vault (required). +func (s *Scheduler) CreateBucket(req CreateBucketRequest) (*BucketView, *BucketAccess, error) { + name := strings.ToLower(strings.TrimSpace(req.Name)) + if !bucketNameRE.MatchString(name) { + return nil, nil, fmt.Errorf("invalid bucket name %q (S3 rules: 3-63 chars, lowercase, digits, dots, hyphens)", name) + } + client, env, err := s.rgwClient() + if err != nil { + return nil, nil, err + } + if req.Region != "" { + env.region = strings.TrimSpace(req.Region) + client.region = env.region + } + + exists, err := client.headBucket(name) + if err != nil { + return nil, nil, err + } + if exists { + return nil, nil, fmt.Errorf("bucket %q already exists", name) + } + if err := client.createBucket(name); err != nil { + return nil, nil, fmt.Errorf("rgw create bucket: %w", err) + } + if req.Versioning { + _ = client.putBucketVersioning(name, true) + } + + userAK, err := randomAccessKey() + if err != nil { + return nil, nil, fmt.Errorf("generate access key: %w", err) + } + userSK, err := randomSecretKey() + if err != nil { + return nil, nil, fmt.Errorf("generate secret key: %w", err) + } + access := &BucketAccess{ + Endpoint: env.endpoint, + Region: env.region, + Bucket: name, + AccessKey: userAK, + SecretKey: userSK, + VaultPath: vaultPathForBucket(name), + S3URL: "s3://" + name, + } + // User credentials live only in Vault (not etcd, not env). + if err := s.vaultPutBucketAccess(name, access); err != nil { + // Best-effort rollback of empty bucket so we do not leave orphan RGW state + // without recoverable credentials. + _ = client.deleteBucket(name) + return nil, nil, fmt.Errorf("vault store bucket access: %w", err) + } + // Best-effort: register key with RGW admin ops so S3 auth accepts it. + if err := client.ensureUserKey(name, userAK, userSK); err != nil { + // Vault already has the material; surface warning via LastError on view. + // Do not fail create — operators can fix RGW user mapping separately. + _ = err + } + + now := time.Now().UTC() + view := &BucketView{ + ID: name, + Name: name, + Region: env.region, + Endpoint: env.endpoint, + Versioning: req.Versioning, + Phase: "Available", + CreatedAt: now, + UpdatedAt: now, + } + return view, access, nil +} + +// ListBuckets lists buckets from RGW (no local inventory). +func (s *Scheduler) ListBuckets() ([]BucketView, error) { + client, env, err := s.rgwClient() + if err != nil { + return nil, err + } + names, err := client.listBuckets() + if err != nil { + return nil, err + } + out := make([]BucketView, 0, len(names)) + for _, n := range names { + out = append(out, BucketView{ + ID: n.Name, + Name: n.Name, + Region: env.region, + Endpoint: env.endpoint, + Phase: "Available", + CreatedAt: n.CreationDate, + UpdatedAt: n.CreationDate, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out, nil +} + +// GetBucket returns one bucket if it exists on RGW. +func (s *Scheduler) GetBucket(idOrName string) (*BucketView, error) { + name := strings.TrimSpace(idOrName) + if name == "" { + return nil, fmt.Errorf("bucket name is required") + } + client, env, err := s.rgwClient() + if err != nil { + return nil, err + } + ok, err := client.headBucket(name) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("bucket %q not found", name) + } + return &BucketView{ + ID: name, + Name: name, + Region: env.region, + Endpoint: env.endpoint, + Phase: "Available", + }, nil +} + +// DeleteBucket removes the bucket on RGW. +func (s *Scheduler) DeleteBucket(idOrName string, force bool) error { + name := strings.TrimSpace(idOrName) + if name == "" { + return fmt.Errorf("bucket name is required") + } + client, _, err := s.rgwClient() + if err != nil { + return err + } + if err := client.deleteBucket(name); err != nil { + if force { + _ = s.vaultDeleteBucketAccess(name) + return nil + } + return fmt.Errorf("rgw delete bucket: %w", err) + } + _ = s.vaultDeleteBucketAccess(name) + return nil +} + +// GetBucketAccess returns S3 credentials from Vault only. +func (s *Scheduler) GetBucketAccess(idOrName string) (*BucketAccess, error) { + name := strings.TrimSpace(idOrName) + if name == "" { + return nil, fmt.Errorf("bucket name is required") + } + client, env, err := s.rgwClient() + if err != nil { + return nil, err + } + ok, err := client.headBucket(name) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("bucket %q not found", name) + } + + access, err := s.vaultGetBucketAccess(name) + if err != nil { + return nil, fmt.Errorf("bucket access not in Vault (path %s): %w", vaultPathForBucket(name), err) + } + // Fill endpoint/region from live RGW config when Vault blob is partial. + if access.Endpoint == "" { + access.Endpoint = env.endpoint + } + if access.Region == "" { + access.Region = env.region + } + if access.Bucket == "" { + access.Bucket = name + } + if access.S3URL == "" { + access.S3URL = "s3://" + name + } + access.VaultPath = vaultPathForBucket(name) + return access, nil +} + +// ListBucketObjects lists objects via RGW. +func (s *Scheduler) ListBucketObjects(idOrName, prefix, continuation string, maxKeys int32) ([]ObjectInfo, string, bool, error) { + name := strings.TrimSpace(idOrName) + if name == "" { + return nil, "", false, fmt.Errorf("bucket name is required") + } + client, _, err := s.rgwClient() + if err != nil { + return nil, "", false, err + } + ok, err := client.headBucket(name) + if err != nil { + return nil, "", false, err + } + if !ok { + return nil, "", false, fmt.Errorf("bucket %q not found", name) + } + if maxKeys <= 0 { + maxKeys = 100 + } + if maxKeys > 1000 { + maxKeys = 1000 + } + return client.listObjects(name, prefix, continuation, int(maxKeys)) +} + +// vaultCredCache avoids hammering vault-manager + Vault login on every bucket op. +var ( + vaultTokMu sync.Mutex + vaultTokCached string + vaultTokExpiry time.Time +) + +func vaultAddr() (string, error) { + addr := strings.TrimSpace(os.Getenv("PERSYS_VAULT_ADDR")) + if addr == "" { + return "", fmt.Errorf("PERSYS_VAULT_ADDR is required for object-storage credentials") + } + return strings.TrimRight(addr, "/"), nil +} + +func vaultManagerAddr() string { + a := strings.TrimSpace(os.Getenv("PERSYS_VAULT_MANAGER_ADDR")) + if a == "" { + a = "vault-manager:50069" + } + return a +} + +func vaultServiceName() string { + n := strings.TrimSpace(os.Getenv("PERSYS_VAULT_SERVICE_NAME")) + if n == "" { + n = "persys-scheduler" + } + return n +} + +// fetchAppRoleFromVaultManager calls vault-manager GetServiceCredentials (same +// path certmanager uses for PKI). No manual role_id/secret_id env required. +func fetchAppRoleFromVaultManager(ctx context.Context) (roleID, secretID string, err error) { + addr := vaultManagerAddr() + conn, err := grpc.DialContext(ctx, addr, grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + return "", "", fmt.Errorf("dial vault-manager %s: %w", addr, err) + } + defer conn.Close() + + client := pb.NewVaultManagerServiceClient(conn) + resp, err := client.GetServiceCredentials(ctx, &pb.GetServiceCredentialsRequest{ + ServiceName: vaultServiceName(), + }) + if err != nil { + return "", "", fmt.Errorf("vault-manager GetServiceCredentials(%s): %w", vaultServiceName(), err) + } + if resp.GetRoleId() == "" || resp.GetSecretId() == "" { + return "", "", fmt.Errorf("vault-manager returned empty AppRole credentials for %s", vaultServiceName()) + } + return resp.GetRoleId(), resp.GetSecretId(), nil +} + +func vaultAppRoleLogin(addr, roleID, secretID string) (token string, leaseTTL time.Duration, err error) { + mount := strings.TrimSpace(os.Getenv("PERSYS_VAULT_APPROLE_MOUNT")) + if mount == "" { + mount = "auth/approle" + } + body, _ := json.Marshal(map[string]string{ + "role_id": roleID, + "secret_id": secretID, + }) + req, err := http.NewRequest(http.MethodPost, addr+"/v1/"+strings.Trim(mount, "/")+"/login", bytes.NewReader(body)) + if err != nil { + return "", 0, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + if err != nil { + return "", 0, err + } + defer resp.Body.Close() + b, _ := io.ReadAll(io.LimitReader(resp.Body, 8192)) + if resp.StatusCode >= 300 { + return "", 0, fmt.Errorf("vault approle login: %d %s", resp.StatusCode, strings.TrimSpace(string(b))) + } + var parsed struct { + Auth struct { + ClientToken string `json:"client_token"` + LeaseDuration int `json:"lease_duration"` + } `json:"auth"` + } + if err := json.Unmarshal(b, &parsed); err != nil { + return "", 0, err + } + if parsed.Auth.ClientToken == "" { + return "", 0, fmt.Errorf("vault approle login: empty client_token") + } + ttl := time.Duration(parsed.Auth.LeaseDuration) * time.Second + if ttl <= 0 { + ttl = time.Hour + } + return parsed.Auth.ClientToken, ttl, nil +} + +// vaultToken resolves a Vault client token. +// Preference order matches the rest of Persys: +// 1. Cached token (still valid) +// 2. PERSYS_VAULT_TOKEN (dev/break-glass only) +// 3. vault-manager → AppRole role_id/secret_id → Vault login +// (production path; same as certmanager) +func vaultToken() (string, error) { + vaultTokMu.Lock() + defer vaultTokMu.Unlock() + + if vaultTokCached != "" && time.Now().Before(vaultTokExpiry) { + return vaultTokCached, nil + } + + // Explicit token still allowed for local/dev. + if tok := strings.TrimSpace(os.Getenv("PERSYS_VAULT_TOKEN")); tok != "" { + vaultTokCached = tok + vaultTokExpiry = time.Now().Add(30 * time.Minute) + return tok, nil + } + + addr, err := vaultAddr() + if err != nil { + return "", err + } + + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + roleID, secretID, err := fetchAppRoleFromVaultManager(ctx) + if err != nil { + // Last resort: static env AppRole (legacy); prefer vault-manager. + roleID = strings.TrimSpace(os.Getenv("PERSYS_VAULT_APPROLE_ROLE_ID")) + secretID = strings.TrimSpace(os.Getenv("PERSYS_VAULT_APPROLE_SECRET_ID")) + if roleID == "" || secretID == "" { + return "", fmt.Errorf("vault credentials: %w", err) + } + } + + tok, ttl, err := vaultAppRoleLogin(addr, roleID, secretID) + if err != nil { + return "", err + } + // Refresh a bit before lease end. + refresh := ttl * 80 / 100 + if refresh < time.Minute { + refresh = ttl - 10*time.Second + } + if refresh < 30*time.Second { + refresh = 30 * time.Second + } + vaultTokCached = tok + vaultTokExpiry = time.Now().Add(refresh) + return tok, nil +} + +func (s *Scheduler) vaultPutBucketAccess(name string, access *BucketAccess) error { + if access == nil { + return fmt.Errorf("access is nil") + } + addr, err := vaultAddr() + if err != nil { + return err + } + token, err := vaultToken() + if err != nil { + return err + } + path := vaultPathForBucket(name) + body, _ := json.Marshal(map[string]any{ + "data": map[string]string{ + "endpoint": access.Endpoint, + "region": access.Region, + "bucket": access.Bucket, + "access_key": access.AccessKey, + "secret_key": access.SecretKey, + "s3_url": access.S3URL, + }, + }) + req, err := http.NewRequest(http.MethodPost, addr+"/v1/"+path, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("X-Vault-Token", token) + req.Header.Set("Content-Type", "application/json") + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return fmt.Errorf("vault put %s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(b))) + } + return nil +} + +func (s *Scheduler) vaultGetBucketAccess(name string) (*BucketAccess, error) { + addr, err := vaultAddr() + if err != nil { + return nil, err + } + token, err := vaultToken() + if err != nil { + return nil, err + } + path := vaultPathForBucket(name) + req, err := http.NewRequest(http.MethodGet, addr+"/v1/"+path, nil) + if err != nil { + return nil, err + } + req.Header.Set("X-Vault-Token", token) + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return nil, fmt.Errorf("vault get %s: %d %s", path, resp.StatusCode, strings.TrimSpace(string(b))) + } + var parsed struct { + Data struct { + Data map[string]string `json:"data"` + } `json:"data"` + } + if err := json.NewDecoder(resp.Body).Decode(&parsed); err != nil { + return nil, err + } + d := parsed.Data.Data + if d["access_key"] == "" || d["secret_key"] == "" { + return nil, fmt.Errorf("vault secret missing access_key/secret_key") + } + return &BucketAccess{ + Endpoint: d["endpoint"], + Region: d["region"], + Bucket: d["bucket"], + AccessKey: d["access_key"], + SecretKey: d["secret_key"], + VaultPath: path, + S3URL: d["s3_url"], + }, nil +} + +func (s *Scheduler) vaultDeleteBucketAccess(name string) error { + addr, err := vaultAddr() + if err != nil { + return err + } + token, err := vaultToken() + if err != nil { + return err + } + path := vaultPathForBucket(name) + // KV v2 delete metadata path uses secret/metadata/... when data path is secret/data/... + metaPath := path + if strings.Contains(path, "/data/") { + metaPath = strings.Replace(path, "/data/", "/metadata/", 1) + } + req, err := http.NewRequest(http.MethodDelete, addr+"/v1/"+metaPath, nil) + if err != nil { + return err + } + req.Header.Set("X-Vault-Token", token) + resp, err := (&http.Client{Timeout: 15 * time.Second}).Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 && resp.StatusCode != http.StatusNotFound { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 2048)) + return fmt.Errorf("vault delete %s: %d %s", metaPath, resp.StatusCode, strings.TrimSpace(string(b))) + } + return nil +} + +func randomAccessKey() (string, error) { + // 20 chars alphanumeric, similar shape to AWS access keys + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + b := make([]byte, 20) + if _, err := rand.Read(b); err != nil { + return "", err + } + for i := range b { + b[i] = alphabet[int(b[i])%len(alphabet)] + } + return "PERS" + string(b), nil +} + +func randomSecretKey() (string, error) { + b := make([]byte, 30) + if _, err := rand.Read(b); err != nil { + return "", err + } + return hex.EncodeToString(b), nil // 60 hex chars +} + +type rgwClient struct { + endpoint string + region string + accessKey string + secretKey string + http *http.Client +} + +func (c *rgwClient) client() *http.Client { + if c.http != nil { + return c.http + } + return &http.Client{Timeout: 30 * time.Second} +} + +func (c *rgwClient) baseURL() string { + return strings.TrimRight(c.endpoint, "/") +} + +type rgwBucketName struct { + Name string + CreationDate time.Time +} + +func (c *rgwClient) listBuckets() ([]rgwBucketName, error) { + req, err := http.NewRequest(http.MethodGet, c.baseURL()+"/", nil) + if err != nil { + return nil, err + } + if err := c.sign(req, nil); err != nil { + return nil, err + } + resp, err := c.client().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, err + } + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("list buckets status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + var parsed struct { + Buckets struct { + Bucket []struct { + Name string `xml:"Name"` + CreationDate string `xml:"CreationDate"` + } `xml:"Bucket"` + } `xml:"Buckets"` + } + if err := xml.Unmarshal(body, &parsed); err != nil { + return nil, fmt.Errorf("parse list buckets: %w", err) + } + out := make([]rgwBucketName, 0, len(parsed.Buckets.Bucket)) + for _, b := range parsed.Buckets.Bucket { + t, _ := time.Parse(time.RFC3339, b.CreationDate) + out = append(out, rgwBucketName{Name: b.Name, CreationDate: t}) + } + return out, nil +} + +func (c *rgwClient) headBucket(name string) (bool, error) { + req, err := http.NewRequest(http.MethodHead, c.baseURL()+"/"+url.PathEscape(name), nil) + if err != nil { + return false, err + } + if err := c.sign(req, nil); err != nil { + return false, err + } + resp, err := c.client().Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusNoContent, http.StatusMovedPermanently: + return true, nil + case http.StatusNotFound: + return false, nil + case http.StatusForbidden: + list, err := c.listBuckets() + if err != nil { + return false, err + } + for _, b := range list { + if b.Name == name { + return true, nil + } + } + return false, nil + default: + return false, fmt.Errorf("head bucket status %d", resp.StatusCode) + } +} + +func (c *rgwClient) createBucket(name string) error { + req, err := http.NewRequest(http.MethodPut, c.baseURL()+"/"+url.PathEscape(name), nil) + if err != nil { + return err + } + if err := c.sign(req, nil); err != nil { + return err + } + resp, err := c.client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode >= 300 && resp.StatusCode != http.StatusConflict { + return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + return nil +} + +func (c *rgwClient) deleteBucket(name string) error { + req, err := http.NewRequest(http.MethodDelete, c.baseURL()+"/"+url.PathEscape(name), nil) + if err != nil { + return err + } + if err := c.sign(req, nil); err != nil { + return err + } + resp, err := c.client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode >= 300 && resp.StatusCode != http.StatusNotFound { + return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + return nil +} + +func (c *rgwClient) putBucketVersioning(name string, enabled bool) error { + status := "Suspended" + if enabled { + status = "Enabled" + } + payload := []byte(fmt.Sprintf(`%s`, status)) + req, err := http.NewRequest(http.MethodPut, c.baseURL()+"/"+url.PathEscape(name)+"?versioning", bytes.NewReader(payload)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/xml") + if err := c.sign(req, payload); err != nil { + return err + } + resp, err := c.client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode >= 300 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + return fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + return nil +} + +type listBucketResult struct { + XMLName xml.Name `xml:"ListBucketResult"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + Contents []struct { + Key string `xml:"Key"` + LastModified string `xml:"LastModified"` + ETag string `xml:"ETag"` + Size int64 `xml:"Size"` + StorageClass string `xml:"StorageClass"` + } `xml:"Contents"` +} + +func (c *rgwClient) listObjects(bucket, prefix, continuation string, maxKeys int) ([]ObjectInfo, string, bool, error) { + q := url.Values{} + q.Set("list-type", "2") + q.Set("max-keys", fmt.Sprintf("%d", maxKeys)) + if prefix != "" { + q.Set("prefix", prefix) + } + if continuation != "" { + q.Set("continuation-token", continuation) + } + u := c.baseURL() + "/" + url.PathEscape(bucket) + "?" + q.Encode() + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + return nil, "", false, err + } + if err := c.sign(req, nil); err != nil { + return nil, "", false, err + } + resp, err := c.client().Do(req) + if err != nil { + return nil, "", false, err + } + defer resp.Body.Close() + body, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) + if err != nil { + return nil, "", false, err + } + if resp.StatusCode >= 300 { + return nil, "", false, fmt.Errorf("status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + var parsed listBucketResult + if err := xml.Unmarshal(body, &parsed); err != nil { + return nil, "", false, fmt.Errorf("parse list response: %w", err) + } + out := make([]ObjectInfo, 0, len(parsed.Contents)) + for _, o := range parsed.Contents { + out = append(out, ObjectInfo{ + Key: o.Key, + SizeBytes: o.Size, + ETag: strings.Trim(o.ETag, `"`), + LastModified: o.LastModified, + StorageClass: o.StorageClass, + }) + } + return out, parsed.NextContinuationToken, parsed.IsTruncated, nil +} + + +// ensureUserKey best-effort registers an S3 key via RGW Admin Ops API so the +// Vault-issued credentials can authenticate. Requires the admin user to have +// admin caps. Path prefix from PERSYS_RGW_ADMIN_PATH (default "admin"). +func (c *rgwClient) ensureUserKey(bucket, accessKey, secretKey string) error { + admin := strings.Trim(strings.TrimSpace(os.Getenv("PERSYS_RGW_ADMIN_PATH")), "/") + if admin == "" { + admin = "admin" + } + uid := "persys-bkt-" + bucket + q := url.Values{} + q.Set("uid", uid) + q.Set("display-name", "Persys bucket "+bucket) + q.Set("access-key", accessKey) + q.Set("secret-key", secretKey) + u := c.baseURL() + "/" + admin + "/user?" + q.Encode() + req, err := http.NewRequest(http.MethodPut, u, nil) + if err != nil { + return err + } + if err := c.sign(req, nil); err != nil { + return err + } + resp, err := c.client().Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode >= 300 && resp.StatusCode != http.StatusConflict { + q2 := url.Values{} + q2.Set("uid", uid) + q2.Set("access-key", accessKey) + q2.Set("secret-key", secretKey) + u2 := c.baseURL() + "/" + admin + "/user?key&" + q2.Encode() + req2, err := http.NewRequest(http.MethodPut, u2, nil) + if err != nil { + return fmt.Errorf("admin user create: %d %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + if err := c.sign(req2, nil); err != nil { + return err + } + resp2, err := c.client().Do(req2) + if err != nil { + return err + } + defer resp2.Body.Close() + if resp2.StatusCode >= 300 { + b2, _ := io.ReadAll(io.LimitReader(resp2.Body, 2048)) + return fmt.Errorf("admin user/key: %d %s / key %d %s", resp.StatusCode, strings.TrimSpace(string(body)), resp2.StatusCode, strings.TrimSpace(string(b2))) + } + } + return nil +} + +func (c *rgwClient) sign(req *http.Request, payload []byte) error { + now := time.Now().UTC() + amzDate := now.Format("20060102T150405Z") + dateStamp := now.Format("20060102") + region := c.region + if region == "" { + region = "default" + } + service := "s3" + if payload == nil { + payload = []byte{} + } + payloadHash := sha256Hex(payload) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + req.Header.Set("X-Amz-Date", amzDate) + if req.Header.Get("Host") == "" { + req.Header.Set("Host", req.URL.Host) + } + canonicalHdrs, signedHeaders := canonicalHeaders(req) + canonicalRequest := strings.Join([]string{ + req.Method, + req.URL.EscapedPath(), + req.URL.RawQuery, + canonicalHdrs, + signedHeaders, + payloadHash, + }, "\n") + credentialScope := dateStamp + "/" + region + "/" + service + "/aws4_request" + stringToSign := strings.Join([]string{ + "AWS4-HMAC-SHA256", + amzDate, + credentialScope, + sha256Hex([]byte(canonicalRequest)), + }, "\n") + signingKey := sigV4Key(c.secretKey, dateStamp, region, service) + signature := hex.EncodeToString(hmacSHA256(signingKey, []byte(stringToSign))) + req.Header.Set("Authorization", fmt.Sprintf( + "AWS4-HMAC-SHA256 Credential=%s/%s, SignedHeaders=%s, Signature=%s", + c.accessKey, credentialScope, signedHeaders, signature, + )) + return nil +} + +func canonicalHeaders(req *http.Request) (string, string) { + headerMap := map[string]string{"host": req.URL.Host} + keys := []string{"host"} + for k, vals := range req.Header { + lk := strings.ToLower(k) + if lk == "host" { + continue + } + if strings.HasPrefix(lk, "x-amz-") || lk == "content-type" || lk == "content-md5" { + headerMap[lk] = strings.TrimSpace(strings.Join(vals, ",")) + keys = append(keys, lk) + } + } + sort.Strings(keys) + var b strings.Builder + for _, k := range keys { + b.WriteString(k) + b.WriteString(":") + b.WriteString(headerMap[k]) + b.WriteString("\n") + } + return b.String(), strings.Join(keys, ";") +} + +func sha256Hex(b []byte) string { + sum := sha256.Sum256(b) + return hex.EncodeToString(sum[:]) +} + +func hmacSHA256(key, data []byte) []byte { + m := hmac.New(sha256.New, key) + _, _ = m.Write(data) + return m.Sum(nil) +} + +func sigV4Key(secret, dateStamp, region, service string) []byte { + kDate := hmacSHA256([]byte("AWS4"+secret), []byte(dateStamp)) + kRegion := hmacSHA256(kDate, []byte(region)) + kService := hmacSHA256(kRegion, []byte(service)) + return hmacSHA256(kService, []byte("aws4_request")) +} \ No newline at end of file From 8854ade53276ba25b76d3439db33045c5be3915f Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:50:56 +0330 Subject: [PATCH 21/31] Feat: Optimize agent connection management and enhance TLS handling with certmanager integration --- .../internal/scheduler/agent_grpc.go | 364 +++++++++++++----- 1 file changed, 275 insertions(+), 89 deletions(-) diff --git a/persys-scheduler/internal/scheduler/agent_grpc.go b/persys-scheduler/internal/scheduler/agent_grpc.go index 8fcf3b2..ab1cf36 100644 --- a/persys-scheduler/internal/scheduler/agent_grpc.go +++ b/persys-scheduler/internal/scheduler/agent_grpc.go @@ -14,13 +14,16 @@ import ( "strings" "time" + "github.com/persys-dev/persys-cloud/pkg/certmanager" agentpb "github.com/persys-dev/persys-cloud/persys-scheduler/internal/agentpb" metricspkg "github.com/persys-dev/persys-cloud/persys-scheduler/internal/metrics" "github.com/persys-dev/persys-cloud/persys-scheduler/internal/models" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" "google.golang.org/grpc/status" ) @@ -94,43 +97,147 @@ func (s *Scheduler) loadClientTLSConfig() (*tls.Config, error) { return nil, fmt.Errorf("invalid CA PEM in %s", caPath) } - tlsCfg := &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: caPool} + // RootCAs from disk now; GetClientCertificate re-reads the keypair on every + // handshake so certmanager rotations / ForceRotate are visible without + // recreating the grpc.ClientConn's static tls.Config snapshot... except + // pooled conns still need invalidate+redial after rotate (see dial/RPC paths). + tlsCfg := &tls.Config{ + MinVersion: tls.VersionTLS12, + RootCAs: caPool, + } if certPath != "" && keyPath != "" { - cert, err := tls.LoadX509KeyPair(certPath, keyPath) - if err != nil { + // Validate material exists up front. + if _, err := tls.LoadX509KeyPair(certPath, keyPath); err != nil { return nil, fmt.Errorf("load client key pair: %w", err) } - tlsCfg.Certificates = []tls.Certificate{cert} + tlsCfg.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, err + } + return &cert, nil + } } return tlsCfg, nil } -func (s *Scheduler) newAgentClient(node models.Node) (agentpb.AgentServiceClient, *grpc.ClientConn, error) { +// agentConnEntry is a pooled connection to a single agent, keyed by node ID. +// Keeping connections alive across calls avoids paying a fresh TCP+TLS +// handshake on every single RPC, which is the dominant cost at fleet sizes +// in the thousands of nodes. +type agentConnEntry struct { + conn *grpc.ClientConn + addr string +} + +// getAgentClient returns a client bound to a pooled, long-lived connection +// for the given node, dialing (or re-dialing, if the node's endpoint changed +// or the previous connection is no longer usable) as needed. +func (s *Scheduler) getAgentClient(node models.Node) (agentpb.AgentServiceClient, error) { addr := s.grpcAddressForNode(node) if strings.TrimSpace(addr) == "" { - return nil, nil, fmt.Errorf("node %s has invalid grpc address", node.NodeID) + return nil, fmt.Errorf("node %s has invalid grpc address", node.NodeID) + } + + s.agentConnMu.Lock() + if entry, ok := s.agentConns[node.NodeID]; ok && entry.addr == addr { + if entry.conn.GetState() != connectivity.Shutdown { + s.agentConnMu.Unlock() + return agentpb.NewAgentServiceClient(entry.conn), nil + } + } + s.agentConnMu.Unlock() + + // Dial outside the lock so a slow/unreachable node can't stall callers + // that need a connection to a different node. + conn, err := s.dialAgentConn(addr) + if err != nil { + return nil, fmt.Errorf("dial agent %s (%s): %w", node.NodeID, addr, err) } - var dialOpts []grpc.DialOption - if s.schedulerAgentTLSEnabled() { - tlsCfg, err := s.loadClientTLSConfig() - if err != nil { - return nil, nil, err + s.agentConnMu.Lock() + defer s.agentConnMu.Unlock() + if existing, ok := s.agentConns[node.NodeID]; ok { + if existing.addr == addr && existing.conn.GetState() != connectivity.Shutdown { + // Another goroutine already established a usable connection + // while we were dialing; keep it and drop ours. + _ = conn.Close() + return agentpb.NewAgentServiceClient(existing.conn), nil } - dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))) - } else { - dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) + _ = existing.conn.Close() } - dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) - dialOpts = append(dialOpts, grpc.WithBlock()) + if s.agentConns == nil { + s.agentConns = make(map[string]*agentConnEntry) + } + s.agentConns[node.NodeID] = &agentConnEntry{conn: conn, addr: addr} + return agentpb.NewAgentServiceClient(conn), nil +} +func (s *Scheduler) dialAgentConn(addr string) (*grpc.ClientConn, error) { ctx, cancel := context.WithTimeout(context.Background(), s.rpcTimeout()) defer cancel() - conn, err := grpc.DialContext(ctx, addr, dialOpts...) - if err != nil { - return nil, nil, fmt.Errorf("dial agent %s (%s): %w", node.NodeID, addr, err) + + doDial := func(ctx context.Context) (*grpc.ClientConn, error) { + var dialOpts []grpc.DialOption + if s.schedulerAgentTLSEnabled() { + tlsCfg, err := s.loadClientTLSConfig() + if err != nil { + return nil, err + } + dialOpts = append(dialOpts, grpc.WithTransportCredentials(credentials.NewTLS(tlsCfg))) + } else { + dialOpts = append(dialOpts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + dialOpts = append(dialOpts, grpc.WithStatsHandler(otelgrpc.NewClientHandler())) + // Keepalive pings detect a dead agent connection (e.g. after a network + // partition or agent restart) so a broken pooled connection doesn't sit + // around silently failing every RPC until something explicitly redials. + dialOpts = append(dialOpts, grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 30 * time.Second, + Timeout: 10 * time.Second, + PermitWithoutStream: true, + })) + dialOpts = append(dialOpts, grpc.WithBlock()) + return grpc.DialContext(ctx, addr, dialOpts...) + } + + // When certmanager is wired, retry dial on cert-related TLS errors with ForceRotate. + if s.certMgr != nil && s.schedulerAgentTLSEnabled() { + var conn *grpc.ClientConn + err := certmanager.WithCertRetry(ctx, s.certMgr, 3, func(ctx context.Context) error { + c, err := doDial(ctx) + if err != nil { + return err + } + conn = c + return nil + }) + return conn, err + } + return doDial(ctx) +} + +// closeAgentConns closes every pooled agent connection. Called on scheduler +// shutdown. +func (s *Scheduler) closeAgentConns() { + s.agentConnMu.Lock() + defer s.agentConnMu.Unlock() + for id, entry := range s.agentConns { + _ = entry.conn.Close() + delete(s.agentConns, id) + } +} + +// invalidateAgentConn drops a pooled connection so the next call re-dials. +// Used when an RPC fails in a way that suggests the connection itself is +// bad (as opposed to an application-level error from a healthy connection). +func (s *Scheduler) invalidateAgentConn(nodeID string) { + s.agentConnMu.Lock() + defer s.agentConnMu.Unlock() + if entry, ok := s.agentConns[nodeID]; ok { + _ = entry.conn.Close() + delete(s.agentConns, nodeID) } - return agentpb.NewAgentServiceClient(conn), conn, nil } func (s *Scheduler) schedulerAgentTLSEnabled() bool { @@ -305,18 +412,49 @@ func (s *Scheduler) buildApplyWorkloadRequest(workload models.Workload) (*agentp Env: workload.EnvVars, }} - case "vm": + case "vm", "microvm": + // Agent proto enum has no MICROVM value; share WORKLOAD_TYPE_VM + spec.vm. + // Stamp firecracker so the agent selects MicroVMRuntime instead of KVM. req.Type = agentpb.WorkloadType_WORKLOAD_TYPE_VM if workload.VM == nil { - return nil, fmt.Errorf("vm spec is required for vm workloads") + return nil, fmt.Errorf("vm spec is required for %s workloads", workload.Type) + } + isMicroVM := strings.EqualFold(strings.TrimSpace(workload.Type), "microvm") + meta := map[string]string{} + for k, v := range workload.VM.Metadata { + meta[k] = v + } + if img := strings.TrimSpace(workload.VM.OsImage); img != "" { + meta["os_image"] = img + meta["persys.vm.os_image"] = img + } + if workload.VM.DiskGB > 0 { + meta["disk_gb"] = fmt.Sprintf("%d", workload.VM.DiskGB) + meta["persys.vm.disk_gb"] = fmt.Sprintf("%d", workload.VM.DiskGB) + } + if rt := strings.TrimSpace(workload.VM.Runtime); rt != "" { + meta["persys.vm.runtime"] = rt + meta["runtime"] = rt + } + if isMicroVM { + meta["persys.vm.runtime"] = "firecracker" + meta["runtime"] = "firecracker" + meta["persys.workload.type"] = "microvm" + } + runtimeName := strings.TrimSpace(workload.VM.Runtime) + if isMicroVM { + runtimeName = "firecracker" } vmSpec := &agentpb.VMSpec{ Name: workload.VM.Name, Vcpus: workload.VM.VCPUs, MemoryMb: workload.VM.MemoryMB, CloudInit: workload.VM.CloudInit, - Metadata: workload.VM.Metadata, + Metadata: meta, ManagedVolumes: toAgentManagedVolumes(workload.VM.ManagedVolumes), + OsImage: strings.TrimSpace(workload.VM.OsImage), + DiskGb: workload.VM.DiskGB, + Runtime: runtimeName, } if workload.VM.CloudInitConfig != nil { vmSpec.CloudInitConfig = &agentpb.CloudInitConfig{ @@ -325,25 +463,58 @@ func (s *Scheduler) buildApplyWorkloadRequest(workload models.Workload) (*agentp NetworkConfig: workload.VM.CloudInitConfig.NetworkConfig, VendorData: workload.VM.CloudInitConfig.VendorData, } + // Pass login overrides through metadata until proto gains fields. + if u := strings.TrimSpace(workload.VM.CloudInitConfig.Username); u != "" { + meta["persys.vm.username"] = u + } + if k := strings.TrimSpace(workload.VM.CloudInitConfig.SSHPublicKey); k != "" { + meta["persys.vm.ssh_public_key"] = k + } + if p := strings.TrimSpace(workload.VM.CloudInitConfig.Password); p != "" { + meta["persys.vm.password"] = p + } + vmSpec.Metadata = meta } - for _, disk := range workload.VM.Disks { - disk = normalizeVMDiskForAgent(workload.ID, disk) - vmSpec.Disks = append(vmSpec.Disks, &agentpb.DiskConfig{ - Path: disk.Path, - Device: disk.Device, - Format: disk.Format, - SizeGb: disk.SizeGB, - Type: disk.Type, - Boot: disk.Boot, - }) + // When OsImage is set and disks are empty, do NOT synthesize a blank + // path here — the agent creates the overlay from os_image. + if len(workload.VM.Disks) == 0 && strings.TrimSpace(workload.VM.OsImage) != "" { + // leave Disks empty intentionally + } else { + for _, disk := range workload.VM.Disks { + disk = normalizeVMDiskForAgent(workload.ID, disk) + // If this is a boot disk and OsImage is set, do not force a path + // that collides with the base image; agent will overlay. + vmSpec.Disks = append(vmSpec.Disks, &agentpb.DiskConfig{ + Path: disk.Path, + Device: disk.Device, + Format: disk.Format, + SizeGb: disk.SizeGB, + Type: disk.Type, + Boot: disk.Boot, + }) + } } for _, network := range workload.VM.Networks { + netName := network.Network + if netName == "" && network.Bridge != "" { + netName = network.Bridge + } + if netName == "" && network.HostDevName != "" { + netName = network.HostDevName + } vmSpec.Networks = append(vmSpec.Networks, &agentpb.NetworkConfig{ - Network: network.Network, - MacAddress: network.MAC, - IpAddress: network.IPAddress, + Network: netName, + MacAddress: network.MAC, + IpAddress: network.IPAddress, + HostDevName: network.HostDevName, + Model: network.Model, + Bridge: network.Bridge, }) + if network.HostDevName != "" { + meta[fmt.Sprintf("persys.vm.host_dev.%s", netName)] = network.HostDevName + } } + vmSpec.Metadata = meta req.Spec.Spec = &agentpb.WorkloadSpec_Vm{Vm: vmSpec} default: @@ -397,18 +568,37 @@ func toAgentManagedVolumes(in []models.ManagedVolumeSpec) []*agentpb.ManagedVolu return out } + +// callAgentRPC runs fn against a pooled agent client. On cert-related TLS +// failures it invalidates the pooled connection, ForceRotates (if certMgr is +// set), and retries once with a fresh dial. +func (s *Scheduler) callAgentRPC(ctx context.Context, node models.Node, fn func(agentpb.AgentServiceClient) error) error { + client, err := s.getAgentClient(node) + if err != nil { + return err + } + err = fn(client) + if err == nil || !certmanager.IsCertRelatedTLSError(err) { + return err + } + + s.invalidateAgentConn(node.NodeID) + if s.certMgr != nil { + _ = s.certMgr.ForceRotate(ctx) + } + client, err = s.getAgentClient(node) + if err != nil { + return err + } + return fn(client) +} + func (s *Scheduler) applyWorkloadOnNode(ctx context.Context, node models.Node, workload models.Workload) (resp *agentpb.ApplyWorkloadResponse, err error) { start := time.Now() defer func() { metricspkg.ObserveAgentRPC("ApplyWorkload", err, time.Since(start)) }() - client, conn, err := s.newAgentClient(node) - if err != nil { - return nil, err - } - defer conn.Close() - req, err := s.buildApplyWorkloadRequest(workload) if err != nil { return nil, err @@ -419,11 +609,13 @@ func (s *Scheduler) applyWorkloadOnNode(ctx context.Context, node models.Node, w } ctx, cancel := context.WithTimeout(ctx, s.rpcTimeout()) defer cancel() - resp, err = client.ApplyWorkload(ctx, req) - if err != nil { - return nil, err - } - return resp, nil + + err = s.callAgentRPC(ctx, node, func(client agentpb.AgentServiceClient) error { + var rpcErr error + resp, rpcErr = client.ApplyWorkload(ctx, req) + return rpcErr + }) + return resp, err } func (s *Scheduler) getWorkloadStatusFromNode(ctx context.Context, node models.Node, workloadID string) (resp *agentpb.WorkloadStatus, err error) { @@ -432,23 +624,21 @@ func (s *Scheduler) getWorkloadStatusFromNode(ctx context.Context, node models.N metricspkg.ObserveAgentRPC("GetWorkloadStatus", err, time.Since(start)) }() - client, conn, err := s.newAgentClient(node) - if err != nil { - return nil, err - } - defer conn.Close() - if ctx == nil { ctx = context.Background() } ctx, cancel := context.WithTimeout(ctx, s.rpcTimeout()) defer cancel() - statusResp, err := client.GetWorkloadStatus(ctx, &agentpb.GetWorkloadStatusRequest{Id: workloadID}) - if err != nil { - return nil, err - } - resp = statusResp.GetStatus() - return resp, nil + + err = s.callAgentRPC(ctx, node, func(client agentpb.AgentServiceClient) error { + statusResp, rpcErr := client.GetWorkloadStatus(ctx, &agentpb.GetWorkloadStatusRequest{Id: workloadID}) + if rpcErr != nil { + return rpcErr + } + resp = statusResp.GetStatus() + return nil + }) + return resp, err } func (s *Scheduler) listWorkloadsFromNode(ctx context.Context, node models.Node) (workloads []*agentpb.WorkloadStatus, err error) { @@ -457,22 +647,21 @@ func (s *Scheduler) listWorkloadsFromNode(ctx context.Context, node models.Node) metricspkg.ObserveAgentRPC("ListWorkloads", err, time.Since(start)) }() - client, conn, err := s.newAgentClient(node) - if err != nil { - return nil, err - } - defer conn.Close() - if ctx == nil { ctx = context.Background() } ctx, cancel := context.WithTimeout(ctx, s.rpcTimeout()) defer cancel() - resp, err := client.ListWorkloads(ctx, &agentpb.ListWorkloadsRequest{Type: agentpb.WorkloadType_WORKLOAD_TYPE_UNSPECIFIED}) - if err != nil { - return nil, err - } - return resp.GetWorkloads(), nil + + err = s.callAgentRPC(ctx, node, func(client agentpb.AgentServiceClient) error { + resp, rpcErr := client.ListWorkloads(ctx, &agentpb.ListWorkloadsRequest{Type: agentpb.WorkloadType_WORKLOAD_TYPE_UNSPECIFIED}) + if rpcErr != nil { + return rpcErr + } + workloads = resp.GetWorkloads() + return nil + }) + return workloads, err } func (s *Scheduler) getWorkloadActionsFromNode(ctx context.Context, node models.Node, workloadID string, limit int32) (actions []*agentpb.AgentAction, err error) { @@ -481,23 +670,21 @@ func (s *Scheduler) getWorkloadActionsFromNode(ctx context.Context, node models. metricspkg.ObserveAgentRPC("ListActions", err, time.Since(start)) }() - client, conn, err := s.newAgentClient(node) - if err != nil { - return nil, err - } - defer conn.Close() - if ctx == nil { ctx = context.Background() } ctx, cancel := context.WithTimeout(ctx, s.rpcTimeout()) defer cancel() - resp, err := client.ListActions(ctx, &agentpb.ListActionsRequest{WorkloadId: workloadID, NewestFirst: true, Limit: limit}) - if err != nil { - return nil, err - } - actions = resp.GetActions() - return actions, nil + + err = s.callAgentRPC(ctx, node, func(client agentpb.AgentServiceClient) error { + resp, rpcErr := client.ListActions(ctx, &agentpb.ListActionsRequest{WorkloadId: workloadID, NewestFirst: true, Limit: limit}) + if rpcErr != nil { + return rpcErr + } + actions = resp.GetActions() + return nil + }) + return actions, err } func (s *Scheduler) deleteWorkloadFromNode(ctx context.Context, node models.Node, workloadID string) (resp *agentpb.DeleteWorkloadResponse, err error) { @@ -506,18 +693,17 @@ func (s *Scheduler) deleteWorkloadFromNode(ctx context.Context, node models.Node metricspkg.ObserveAgentRPC("DeleteWorkload", err, time.Since(start)) }() - client, conn, err := s.newAgentClient(node) - if err != nil { - return nil, err - } - defer conn.Close() - if ctx == nil { ctx = context.Background() } ctx, cancel := context.WithTimeout(ctx, s.rpcTimeout()) defer cancel() - resp, err = client.DeleteWorkload(ctx, &agentpb.DeleteWorkloadRequest{Id: workloadID}) + + err = s.callAgentRPC(ctx, node, func(client agentpb.AgentServiceClient) error { + var rpcErr error + resp, rpcErr = client.DeleteWorkload(ctx, &agentpb.DeleteWorkloadRequest{Id: workloadID}) + return rpcErr + }) return resp, err } From 126778f85de954f1c525599e74ffad77a684e6c6 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:51:14 +0330 Subject: [PATCH 22/31] Feat: Enhance VMSpec and disk management structures with additional fields for improved configuration --- persys-scheduler/internal/models/models.go | 45 +++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/persys-scheduler/internal/models/models.go b/persys-scheduler/internal/models/models.go index 3f1fc74..fa25ed5 100644 --- a/persys-scheduler/internal/models/models.go +++ b/persys-scheduler/internal/models/models.go @@ -167,31 +167,41 @@ type DriftRecord struct { } // VMSpec defines VM-specific fields for scheduler API and persistence. +// Happy path: OsImage + DiskGB + VCPUs + MemoryMB. Agent creates a writable +// overlay from OsImage and generates cloud-init credentials. type VMSpec struct { Name string `json:"name,omitempty"` VCPUs int32 `json:"vcpus,omitempty"` MemoryMB int64 `json:"memoryMb,omitempty"` + OsImage string `json:"osImage,omitempty"` + DiskGB int64 `json:"diskGb,omitempty"` Disks []VMDiskConfig `json:"disks,omitempty"` Networks []VMNetworkConfig `json:"networks,omitempty"` CloudInit string `json:"cloudInit,omitempty"` Metadata map[string]string `json:"metadata,omitempty"` CloudInitConfig *CloudInitConfig `json:"cloudInitConfig,omitempty"` ManagedVolumes []ManagedVolumeSpec `json:"managedVolumes,omitempty"` + Runtime string `json:"runtime,omitempty"` // libvirt|firecracker } type VMDiskConfig struct { - Path string `json:"path,omitempty"` - Device string `json:"device,omitempty"` - Format string `json:"format,omitempty"` - SizeGB int64 `json:"sizeGb,omitempty"` - Type string `json:"type,omitempty"` - Boot bool `json:"boot,omitempty"` + Path string `json:"path,omitempty"` + Device string `json:"device,omitempty"` + Format string `json:"format,omitempty"` + SizeGB int64 `json:"sizeGb,omitempty"` + Type string `json:"type,omitempty"` + Boot bool `json:"boot,omitempty"` + BackingFile string `json:"backingFile,omitempty"` + Storage string `json:"storage,omitempty"` // local|nfs|ceph-rbd } type VMNetworkConfig struct { - Network string `json:"network,omitempty"` - MAC string `json:"macAddress,omitempty"` - IPAddress string `json:"ipAddress,omitempty"` + Network string `json:"network,omitempty"` + MAC string `json:"macAddress,omitempty"` + IPAddress string `json:"ipAddress,omitempty"` + HostDevName string `json:"hostDevName,omitempty"` // Firecracker TAP + Model string `json:"model,omitempty"` + Bridge string `json:"bridge,omitempty"` } type CloudInitConfig struct { @@ -199,6 +209,9 @@ type CloudInitConfig struct { MetaData string `json:"metaData,omitempty"` NetworkConfig string `json:"networkConfig,omitempty"` VendorData string `json:"vendorData,omitempty"` + Username string `json:"username,omitempty"` + SSHPublicKey string `json:"sshPublicKey,omitempty"` + Password string `json:"password,omitempty"` } type ManagedVolumeSpec struct { @@ -233,7 +246,8 @@ type WorkloadReason struct { Retryable bool `json:"retryable,omitempty"` } -// ManagedVolumeRecord is the control-plane source of truth for a managed volume. +// ManagedVolumeRecord is the control-plane source of truth for a managed volume +// (workload-attached or standalone disk inventory). type ManagedVolumeRecord struct { ID string `json:"id"` Name string `json:"name"` @@ -242,12 +256,17 @@ type ManagedVolumeRecord struct { AccessMode string `json:"accessMode,omitempty"` FSType string `json:"fsType,omitempty"` RetainPolicy string `json:"retainPolicy,omitempty"` - Phase string `json:"phase,omitempty"` // Provisioning|Provisioned|Attached|Released|Retained|Deleting|Deleted|Error + Phase string `json:"phase,omitempty"` // Pending|Available|Provisioning|Provisioned|Attached|Released|Retained|Deleting|Deleted|Error LastError string `json:"lastError,omitempty"` WorkloadRefs []string `json:"workloadRefs,omitempty"` AttachedNodes []string `json:"attachedNodes,omitempty"` - CreatedAt time.Time `json:"createdAt,omitempty"` - UpdatedAt time.Time `json:"updatedAt,omitempty"` + // Standalone disk inventory fields (CreateDisk API). + NodeID string `json:"nodeId,omitempty"` // local pin; set from workload placement node + Device string `json:"device,omitempty"` // e.g. rbd:pool/name once known + Standalone bool `json:"standalone,omitempty"` // created via CreateDisk, not only via workload + MountPath string `json:"mountPath,omitempty"` // default container mount when bound + CreatedAt time.Time `json:"createdAt,omitempty"` + UpdatedAt time.Time `json:"updatedAt,omitempty"` } // VolumeAttachmentRecord is the control-plane source of truth for node/workload attachment. From df31bb2259dcaa016972b153e896e973b3f6c1b7 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:51:33 +0330 Subject: [PATCH 23/31] Feat: Add ListEvents and WatchEvents methods for cluster-wide scheduler event management --- persys-scheduler/internal/grpcapi/service.go | 493 +++++++++++++++++-- 1 file changed, 463 insertions(+), 30 deletions(-) diff --git a/persys-scheduler/internal/grpcapi/service.go b/persys-scheduler/internal/grpcapi/service.go index ec1152c..857478d 100644 --- a/persys-scheduler/internal/grpcapi/service.go +++ b/persys-scheduler/internal/grpcapi/service.go @@ -240,6 +240,11 @@ func (s *Service) ApplyWorkload(ctx context.Context, in *controlv1.ApplyWorkload } annotateRPC(ctx, attribute.String("scheduler.workload_type", strings.TrimSpace(workload.Type))) + // Expand standalone disk inventory refs (metadata persys.disk.ids) into managed_volumes. + if err := s.sched.AttachDiskIDsToWorkload(&workload); err != nil { + return &controlv1.ApplyWorkloadResponse{Success: false, FailureReason: controlv1.FailureReason_INVALID_SPEC, ErrorMessage: err.Error()}, nil + } + var persisted models.Workload if _, err := s.sched.GetWorkloadByID(workload.ID); err == nil { updated, err := s.sched.UpdateWorkloadSpec(workload.ID, workload) @@ -629,6 +634,118 @@ func (s *Service) GetClusterSummary(ctx context.Context, _ *controlv1.GetCluster return resp, nil } +// ListEvents returns recent cluster-wide scheduler events, most-recent +// storage order (etcd prefix scan order — callers that need strict +// chronological ordering should sort client-side on the returned +// timestamps, since etcd's own key order is by event ID, not time). +func (s *Service) ListEvents(ctx context.Context, in *controlv1.ListEventsRequest) (*controlv1.ListEventsResponse, error) { + limit := int64(0) + if in != nil { + limit = in.GetLimit() + annotateRPC(ctx, + attribute.String("scheduler.event_type", strings.TrimSpace(in.GetType())), + attribute.String("scheduler.workload_id", strings.TrimSpace(in.GetWorkloadId())), + attribute.String("scheduler.node_id", strings.TrimSpace(in.GetNodeId())), + ) + } + + events, err := s.sched.ListSchedulerEvents(limit) + if err != nil { + rpcErr := status.Error(codes.Internal, err.Error()) + recordRPCError(ctx, rpcErr) + return nil, rpcErr + } + + out := make([]*controlv1.SchedulerEventView, 0, len(events)) + for _, event := range events { + if !eventMatchesFilter(event, in) { + continue + } + out = append(out, schedulerEventToView(event)) + } + return &controlv1.ListEventsResponse{Events: out}, nil +} + +// WatchEvents streams cluster-wide scheduler events to the caller as they +// happen (see Scheduler.WatchEvents), replaying recent history first. +// Blocks until the client disconnects or the stream's context is +// cancelled (e.g. server shutdown). +func (s *Service) WatchEvents(in *controlv1.WatchEventsRequest, stream controlv1.AgentControl_WatchEventsServer) error { + if in != nil { + annotateRPC(stream.Context(), + attribute.String("scheduler.event_type", strings.TrimSpace(in.GetType())), + attribute.String("scheduler.workload_id", strings.TrimSpace(in.GetWorkloadId())), + attribute.String("scheduler.node_id", strings.TrimSpace(in.GetNodeId())), + ) + } + + err := s.sched.WatchEvents(stream.Context(), defaultWatchEventsReplayLimit, func(event models.SchedulerEvent) error { + if !eventMatchesFilter(event, in) { + return nil + } + return stream.Send(schedulerEventToView(event)) + }) + if err != nil && stream.Context().Err() == nil { + rpcErr := status.Error(codes.Internal, err.Error()) + recordRPCError(stream.Context(), rpcErr) + return rpcErr + } + return nil +} + +// defaultWatchEventsReplayLimit bounds how much history WatchEvents +// replays to a newly-connected client before switching to live events. +const defaultWatchEventsReplayLimit = 100 + +// eventFilter is satisfied by both ListEventsRequest and WatchEventsRequest +// (both carry the same three optional filter fields), letting +// eventMatchesFilter serve both RPC handlers. +type eventFilter interface { + GetType() string + GetWorkloadId() string + GetNodeId() string +} + +func eventMatchesFilter(event models.SchedulerEvent, filter eventFilter) bool { + if filter == nil { + return true + } + if t := strings.TrimSpace(filter.GetType()); t != "" && !strings.EqualFold(event.Type, t) { + return false + } + if w := strings.TrimSpace(filter.GetWorkloadId()); w != "" && event.WorkloadID != w { + return false + } + if n := strings.TrimSpace(filter.GetNodeId()); n != "" && event.NodeID != n { + return false + } + return true +} + +// schedulerEventToView converts the internal event model to its proto +// view. Details values are stringified (fmt.Sprintf("%v", ...)) rather +// than mapped to google.protobuf.Struct — event details are +// informational/display-oriented, not structured data a client needs to +// round-trip losslessly, so the simpler map shape was +// chosen deliberately (see the field comment in control.proto). +func schedulerEventToView(event models.SchedulerEvent) *controlv1.SchedulerEventView { + view := &controlv1.SchedulerEventView{ + Id: event.ID, + Type: event.Type, + WorkloadId: event.WorkloadID, + NodeId: event.NodeID, + Reason: event.Reason, + Timestamp: timestamppb.New(event.Timestamp), + } + if len(event.Details) > 0 { + view.Details = make(map[string]string, len(event.Details)) + for k, v := range event.Details { + view.Details[k] = fmt.Sprintf("%v", v) + } + } + return view +} + func (s *Service) ControlStream(stream controlv1.AgentControl_ControlStreamServer) error { err := status.Error(codes.Unimplemented, "ControlStream is not implemented yet") recordRPCError(stream.Context(), err) @@ -663,7 +780,7 @@ func taintsToProto(taints []models.NodeTaint) []*controlv1.NodeTaint { } func workloadToView(workload models.Workload) *controlv1.WorkloadView { - return &controlv1.WorkloadView{ + view := &controlv1.WorkloadView{ WorkloadId: workload.ID, Type: workload.Type, DesiredState: workload.DesiredState, @@ -678,6 +795,10 @@ func workloadToView(workload models.Workload) *controlv1.WorkloadView { Reason: reasonToProto(workload.StatusInfo.Reason, workload.StatusInfo.LastUpdated), Usage: usageToProto(workload.Usage, workload.ID, workload.Type), } + // Structured access fields land after `make proto` regenerates + // WorkloadView (PrimaryIp, LoginUser, LoginPassword, LoginMethod). + // Until then Status carries the same data for persysctl + dashboard. + return view } func workloadStatusForView(workload models.Workload) string { @@ -685,16 +806,48 @@ func workloadStatusForView(workload models.Workload) string { if status == "" { status = "Unknown" } - if !strings.EqualFold(strings.TrimSpace(workload.Type), "vm") { + // Surface guest access on VM / microVM (and any type that reported access meta). + parts := guestAccessStatusParts(workload) + if len(parts) == 0 { return status } - if ip, ok := workload.Metadata["vm.primary_ip"]; ok { - ipStr := strings.TrimSpace(fmt.Sprintf("%v", ip)) - if ipStr != "" { - return fmt.Sprintf("%s (ip=%s)", status, ipStr) - } + return fmt.Sprintf("%s (%s)", status, strings.Join(parts, " ")) +} + +// guestAccessStatusParts builds the parenthetical access summary used by +// persysctl / dashboard (same channel as vm.primary_ip today). +func guestAccessStatusParts(workload models.Workload) []string { + if workload.Metadata == nil { + return nil } - return status + var parts []string + if ip, ok := metadataString(workload.Metadata, "vm.primary_ip"); ok { + parts = append(parts, "ip="+ip) + } + if user, ok := metadataString(workload.Metadata, "vm.login_user"); ok { + parts = append(parts, "user="+user) + } + method, _ := metadataString(workload.Metadata, "vm.login_method") + if pass, ok := metadataString(workload.Metadata, "vm.login_password"); ok { + parts = append(parts, "password="+pass) + } else if method == "ssh_key" { + parts = append(parts, "login=ssh_key") + } + return parts +} + +// workloadAccessFields returns structured access info for WorkloadView once +// control.proto fields primary_ip / login_* are generated (make proto). +// Until then, callers rely on guestAccessStatusParts embedded in Status. +func workloadAccessFields(workload models.Workload) (ip, user, password, method string) { + if workload.Metadata == nil { + return "", "", "", "" + } + ip, _ = metadataString(workload.Metadata, "vm.primary_ip") + user, _ = metadataString(workload.Metadata, "vm.login_user") + password, _ = metadataString(workload.Metadata, "vm.login_password") + method, _ = metadataString(workload.Metadata, "vm.login_method") + return ip, user, password, method } func workloadFailureReasonForView(workload models.Workload) string { @@ -995,7 +1148,8 @@ func controlApplyToModel(in *controlv1.ApplyWorkloadRequest) (models.Workload, e } else { w.ComposeYAML = cp.GetInlineYaml() } - case "vm": + case "vm", "microvm": + // microvm uses the same WorkloadSpec.vm oneof as KVM VMs; keep Type=microvm for placement. if embeddedVM, ok := parseEmbeddedVMSpec(w.Metadata); ok { w.VM = embeddedVM delete(w.Metadata, "persys.vm_spec_b64") @@ -1006,10 +1160,16 @@ func controlApplyToModel(in *controlv1.ApplyWorkloadRequest) (models.Workload, e return models.Workload{}, fmt.Errorf("vm spec required") } w.VM = &models.VMSpec{ - VCPUs: vm.GetVcpus(), - MemoryMB: vm.GetMemoryMb(), + VCPUs: vm.GetVcpus(), + MemoryMB: vm.GetMemoryMb(), + OsImage: strings.TrimSpace(vm.GetOsImage()), CloudInit: "", } + // Prefer explicit disk size from first disk entry; else leave 0 and + // let the agent default DiskGB when synthesizing from os_image. + if disks := vm.GetDisks(); len(disks) > 0 && disks[0].GetSizeGb() > 0 { + w.VM.DiskGB = disks[0].GetSizeGb() + } if ci := vm.GetCloudInit(); ci != nil { w.VM.CloudInitConfig = &models.CloudInitConfig{ UserData: ci.GetUserData(), @@ -1029,10 +1189,30 @@ func controlApplyToModel(in *controlv1.ApplyWorkloadRequest) (models.Workload, e SizeGB: d.GetSizeGb(), Device: device, Format: "qcow2", + Boot: device == "vda", + Storage: "local", }) } + // When only os_image was provided (common dashboard path), leave Disks + // empty so the agent synthesizes a root overlay from OsImage. + if w.VM.OsImage != "" && len(w.VM.Disks) == 1 && w.VM.Disks[0].Path == "" && w.VM.Disks[0].SizeGB > 0 { + // Keep single size hint via DiskGB; agent owns path placement. + w.VM.DiskGB = w.VM.Disks[0].SizeGB + w.VM.Disks = nil + } for _, n := range vm.GetNetworks() { - w.VM.Networks = append(w.VM.Networks, models.VMNetworkConfig{Network: n.GetBridge(), IPAddress: n.GetStaticIp()}) + netName := strings.TrimSpace(n.GetBridge()) + if netName == "" { + netName = "default" + } + w.VM.Networks = append(w.VM.Networks, models.VMNetworkConfig{ + Network: netName, + Bridge: strings.TrimSpace(n.GetBridge()), + IPAddress: n.GetStaticIp(), + }) + } + if len(w.VM.Networks) == 0 { + w.VM.Networks = []models.VMNetworkConfig{{Network: "default"}} } default: return models.Workload{}, fmt.Errorf("unsupported workload type %q", in.GetSpec().GetType()) @@ -1059,18 +1239,25 @@ func parseEmbeddedVMSpec(metadata map[string]interface{}) (*models.VMSpec, bool) Name string `json:"name"` VCPUs int32 `json:"vcpus"` MemoryMB int64 `json:"memory_mb"` + OsImage string `json:"os_image"` + DiskGB int64 `json:"disk_gb"` Disks []struct { - Path string `json:"path"` - Device string `json:"device"` - Format string `json:"format"` - SizeGB int64 `json:"size_gb"` - Type string `json:"type"` - Boot bool `json:"boot"` + Path string `json:"path"` + Device string `json:"device"` + Format string `json:"format"` + SizeGB int64 `json:"size_gb"` + Type string `json:"type"` + Boot bool `json:"boot"` + BackingFile string `json:"backing_file"` + Storage string `json:"storage"` } `json:"disks"` Networks []struct { - Network string `json:"network"` - MAC string `json:"mac_address"` - IPAddress string `json:"ip_address"` + Network string `json:"network"` + MAC string `json:"mac_address"` + IPAddress string `json:"ip_address"` + HostDevName string `json:"host_dev_name"` + Model string `json:"model"` + Bridge string `json:"bridge"` } `json:"networks"` CloudInit string `json:"cloud_init"` Metadata map[string]string `json:"metadata"` @@ -1079,6 +1266,9 @@ func parseEmbeddedVMSpec(metadata map[string]interface{}) (*models.VMSpec, bool) MetaData string `json:"meta_data"` NetworkConfig string `json:"network_config"` VendorData string `json:"vendor_data"` + Username string `json:"username"` + SSHPublicKey string `json:"ssh_public_key"` + Password string `json:"password"` } `json:"cloud_init_config"` ManagedVolumes []struct { Name string `json:"name"` @@ -1099,6 +1289,8 @@ func parseEmbeddedVMSpec(metadata map[string]interface{}) (*models.VMSpec, bool) Name: spec.Name, VCPUs: spec.VCPUs, MemoryMB: spec.MemoryMB, + OsImage: strings.TrimSpace(spec.OsImage), + DiskGB: spec.DiskGB, CloudInit: spec.CloudInit, Metadata: spec.Metadata, } @@ -1108,23 +1300,31 @@ func parseEmbeddedVMSpec(metadata map[string]interface{}) (*models.VMSpec, bool) MetaData: spec.CloudInitConfig.MetaData, NetworkConfig: spec.CloudInitConfig.NetworkConfig, VendorData: spec.CloudInitConfig.VendorData, + Username: spec.CloudInitConfig.Username, + SSHPublicKey: spec.CloudInitConfig.SSHPublicKey, + Password: spec.CloudInitConfig.Password, } } for _, d := range spec.Disks { out.Disks = append(out.Disks, models.VMDiskConfig{ - Path: d.Path, - Device: d.Device, - Format: d.Format, - SizeGB: d.SizeGB, - Type: d.Type, - Boot: d.Boot, + Path: d.Path, + Device: d.Device, + Format: d.Format, + SizeGB: d.SizeGB, + Type: d.Type, + Boot: d.Boot, + BackingFile: d.BackingFile, + Storage: d.Storage, }) } for _, n := range spec.Networks { out.Networks = append(out.Networks, models.VMNetworkConfig{ - Network: n.Network, - MAC: n.MAC, - IPAddress: n.IPAddress, + Network: n.Network, + MAC: n.MAC, + IPAddress: n.IPAddress, + HostDevName: n.HostDevName, + Model: n.Model, + Bridge: n.Bridge, }) } for _, mv := range spec.ManagedVolumes { @@ -1227,3 +1427,236 @@ func normalizeSupportedStorageDrivers(drivers []string) []string { } return out } + +// --- Standalone disks (mTLS gRPC AgentControl) --- + +func (s *Service) CreateDisk(ctx context.Context, in *controlv1.CreateDiskRequest) (*controlv1.CreateDiskResponse, error) { + if in == nil { + return nil, status.Error(codes.InvalidArgument, "request is required") + } + if !s.sched.IsWritable() { + return nil, status.Error(codes.Unavailable, "scheduler degraded/recovery mode; control plane frozen") + } + view, err := s.sched.CreateDisk(scheduler.CreateDiskRequest{ + Name: in.GetName(), + Driver: in.GetDriver(), + SizeGB: in.GetSizeGb(), + FSType: in.GetFsType(), + AccessMode: in.GetAccessMode(), + RetainPolicy: in.GetRetainPolicy(), + NodeID: in.GetNodeId(), + MountPath: in.GetMountPath(), + }) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return &controlv1.CreateDiskResponse{Disk: diskViewToProto(view)}, nil +} + +func (s *Service) ListDisks(ctx context.Context, in *controlv1.ListDisksRequest) (*controlv1.ListDisksResponse, error) { + list, err := s.sched.ListDisks() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + out := make([]*controlv1.DiskView, 0, len(list)) + for i := range list { + out = append(out, diskViewToProto(&list[i])) + } + return &controlv1.ListDisksResponse{Disks: out}, nil +} + +func (s *Service) GetDisk(ctx context.Context, in *controlv1.GetDiskRequest) (*controlv1.GetDiskResponse, error) { + if in == nil || strings.TrimSpace(in.GetDiskId()) == "" { + return nil, status.Error(codes.InvalidArgument, "disk_id is required") + } + view, err := s.sched.GetDisk(in.GetDiskId()) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + return &controlv1.GetDiskResponse{Disk: diskViewToProto(view)}, nil +} + +func (s *Service) DeleteDisk(ctx context.Context, in *controlv1.DeleteDiskRequest) (*controlv1.DeleteDiskResponse, error) { + if in == nil || strings.TrimSpace(in.GetDiskId()) == "" { + return nil, status.Error(codes.InvalidArgument, "disk_id is required") + } + if !s.sched.IsWritable() { + return &controlv1.DeleteDiskResponse{Success: false, ErrorMessage: "scheduler degraded/recovery mode; control plane frozen"}, nil + } + if err := s.sched.DeleteDisk(in.GetDiskId(), in.GetForce()); err != nil { + return &controlv1.DeleteDiskResponse{Success: false, ErrorMessage: err.Error()}, nil + } + return &controlv1.DeleteDiskResponse{Success: true}, nil +} + +func diskViewToProto(v *scheduler.DiskView) *controlv1.DiskView { + if v == nil { + return nil + } + out := &controlv1.DiskView{ + Id: v.ID, + Name: v.Name, + Driver: v.Driver, + SizeGb: v.SizeGB, + FsType: v.FSType, + AccessMode: v.AccessMode, + RetainPolicy: v.RetainPolicy, + Phase: v.Phase, + LastError: v.LastError, + NodeId: v.NodeID, + Device: v.Device, + Standalone: v.Standalone, + MountPath: v.MountPath, + WorkloadRefs: v.WorkloadRefs, + AttachedNodes: v.AttachedNodes, + } + if !v.CreatedAt.IsZero() { + out.CreatedAt = timestamppb.New(v.CreatedAt) + } + if !v.UpdatedAt.IsZero() { + out.UpdatedAt = timestamppb.New(v.UpdatedAt) + } + return out +} + + +// --- Object storage (Ceph RGW / S3) --- + +func (s *Service) CreateBucket(ctx context.Context, in *controlv1.CreateBucketRequest) (*controlv1.CreateBucketResponse, error) { + if in == nil { + return nil, status.Error(codes.InvalidArgument, "request is required") + } + if !s.sched.IsWritable() { + return nil, status.Error(codes.Unavailable, "scheduler degraded/recovery mode; control plane frozen") + } + view, access, err := s.sched.CreateBucket(scheduler.CreateBucketRequest{ + Name: in.GetName(), + Region: in.GetRegion(), + Versioning: in.GetVersioning(), + }) + if err != nil { + return nil, status.Error(codes.InvalidArgument, err.Error()) + } + return &controlv1.CreateBucketResponse{ + Bucket: bucketViewToProto(view), + Access: bucketAccessToProto(access), + }, nil +} + +func (s *Service) ListBuckets(ctx context.Context, in *controlv1.ListBucketsRequest) (*controlv1.ListBucketsResponse, error) { + list, err := s.sched.ListBuckets() + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + out := make([]*controlv1.BucketView, 0, len(list)) + for i := range list { + out = append(out, bucketViewToProto(&list[i])) + } + return &controlv1.ListBucketsResponse{Buckets: out}, nil +} + +func (s *Service) GetBucket(ctx context.Context, in *controlv1.GetBucketRequest) (*controlv1.GetBucketResponse, error) { + if in == nil || strings.TrimSpace(in.GetBucketId()) == "" { + return nil, status.Error(codes.InvalidArgument, "bucket_id is required") + } + view, err := s.sched.GetBucket(in.GetBucketId()) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + return &controlv1.GetBucketResponse{Bucket: bucketViewToProto(view)}, nil +} + +func (s *Service) DeleteBucket(ctx context.Context, in *controlv1.DeleteBucketRequest) (*controlv1.DeleteBucketResponse, error) { + if in == nil || strings.TrimSpace(in.GetBucketId()) == "" { + return nil, status.Error(codes.InvalidArgument, "bucket_id is required") + } + if !s.sched.IsWritable() { + return &controlv1.DeleteBucketResponse{Success: false, ErrorMessage: "scheduler degraded/recovery mode; control plane frozen"}, nil + } + if err := s.sched.DeleteBucket(in.GetBucketId(), in.GetForce()); err != nil { + return &controlv1.DeleteBucketResponse{Success: false, ErrorMessage: err.Error()}, nil + } + return &controlv1.DeleteBucketResponse{Success: true}, nil +} + +func (s *Service) GetBucketAccess(ctx context.Context, in *controlv1.GetBucketAccessRequest) (*controlv1.GetBucketAccessResponse, error) { + if in == nil || strings.TrimSpace(in.GetBucketId()) == "" { + return nil, status.Error(codes.InvalidArgument, "bucket_id is required") + } + access, err := s.sched.GetBucketAccess(in.GetBucketId()) + if err != nil { + return nil, status.Error(codes.NotFound, err.Error()) + } + return &controlv1.GetBucketAccessResponse{Access: bucketAccessToProto(access)}, nil +} + +func (s *Service) ListBucketObjects(ctx context.Context, in *controlv1.ListBucketObjectsRequest) (*controlv1.ListBucketObjectsResponse, error) { + if in == nil || strings.TrimSpace(in.GetBucketId()) == "" { + return nil, status.Error(codes.InvalidArgument, "bucket_id is required") + } + objs, next, truncated, err := s.sched.ListBucketObjects( + in.GetBucketId(), + in.GetPrefix(), + in.GetContinuationToken(), + in.GetMaxKeys(), + ) + if err != nil { + return nil, status.Error(codes.Internal, err.Error()) + } + out := make([]*controlv1.ObjectInfo, 0, len(objs)) + for _, o := range objs { + out = append(out, &controlv1.ObjectInfo{ + Key: o.Key, + SizeBytes: o.SizeBytes, + Etag: o.ETag, + LastModified: o.LastModified, + StorageClass: o.StorageClass, + }) + } + return &controlv1.ListBucketObjectsResponse{ + Objects: out, + NextContinuationToken: next, + IsTruncated: truncated, + Prefix: in.GetPrefix(), + }, nil +} + +func bucketViewToProto(v *scheduler.BucketView) *controlv1.BucketView { + if v == nil { + return nil + } + out := &controlv1.BucketView{ + Id: v.ID, + Name: v.Name, + Region: v.Region, + Owner: v.Owner, + Endpoint: v.Endpoint, + Versioning: v.Versioning, + ObjectCount: v.ObjectCount, + SizeBytes: v.SizeBytes, + Phase: v.Phase, + LastError: v.LastError, + } + if !v.CreatedAt.IsZero() { + out.CreatedAt = timestamppb.New(v.CreatedAt) + } + if !v.UpdatedAt.IsZero() { + out.UpdatedAt = timestamppb.New(v.UpdatedAt) + } + return out +} + +func bucketAccessToProto(a *scheduler.BucketAccess) *controlv1.BucketAccess { + if a == nil { + return nil + } + return &controlv1.BucketAccess{ + Endpoint: a.Endpoint, + Region: a.Region, + Bucket: a.Bucket, + AccessKey: a.AccessKey, + SecretKey: a.SecretKey, + VaultPath: a.VaultPath, + S3Url: a.S3URL, + } +} \ No newline at end of file From 8c411ba3bcd5d8653f202ab19dc116c835cc74ac Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:52:03 +0330 Subject: [PATCH 24/31] Chore: Update gRPC protobuf implementations --- .../internal/controlv1/control.pb.go | 2688 ++++++++++++++--- .../internal/controlv1/control_grpc.pb.go | 485 ++- 2 files changed, 2831 insertions(+), 342 deletions(-) diff --git a/persys-scheduler/internal/controlv1/control.pb.go b/persys-scheduler/internal/controlv1/control.pb.go index a2d0fc4..583bcbf 100644 --- a/persys-scheduler/internal/controlv1/control.pb.go +++ b/persys-scheduler/internal/controlv1/control.pb.go @@ -3483,6 +3483,279 @@ func (x *ListWorkloadsRequest) GetStatus() string { return "" } +type SchedulerEventView struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` // e.g. "NodeLost", "WorkloadScheduled", "DriftDetected" + WorkloadId string `protobuf:"bytes,3,opt,name=workload_id,json=workloadId,proto3" json:"workload_id,omitempty"` // optional, empty if not workload-scoped + NodeId string `protobuf:"bytes,4,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // optional, empty if not node-scoped + Reason string `protobuf:"bytes,5,opt,name=reason,proto3" json:"reason,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // Free-form auxiliary data. Values are stringified on the way out + // (models.SchedulerEvent.Details is map[string]interface{} on the Go + // side) — this is a deliberate simplification over a + // google.protobuf.Struct, since event details are informational/ + // display-oriented, not structured data a client needs to + // round-trip losslessly. + Details map[string]string `protobuf:"bytes,7,rep,name=details,proto3" json:"details,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SchedulerEventView) Reset() { + *x = SchedulerEventView{} + mi := &file_control_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SchedulerEventView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SchedulerEventView) ProtoMessage() {} + +func (x *SchedulerEventView) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[49] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SchedulerEventView.ProtoReflect.Descriptor instead. +func (*SchedulerEventView) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{49} +} + +func (x *SchedulerEventView) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SchedulerEventView) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *SchedulerEventView) GetWorkloadId() string { + if x != nil { + return x.WorkloadId + } + return "" +} + +func (x *SchedulerEventView) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + +func (x *SchedulerEventView) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +func (x *SchedulerEventView) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp + } + return nil +} + +func (x *SchedulerEventView) GetDetails() map[string]string { + if x != nil { + return x.Details + } + return nil +} + +type ListEventsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Limit int64 `protobuf:"varint,1,opt,name=limit,proto3" json:"limit,omitempty"` // 0 means server default + Type string `protobuf:"bytes,2,opt,name=type,proto3" json:"type,omitempty"` // optional filter + WorkloadId string `protobuf:"bytes,3,opt,name=workload_id,json=workloadId,proto3" json:"workload_id,omitempty"` // optional filter + NodeId string `protobuf:"bytes,4,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // optional filter + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListEventsRequest) Reset() { + *x = ListEventsRequest{} + mi := &file_control_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListEventsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListEventsRequest) ProtoMessage() {} + +func (x *ListEventsRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[50] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListEventsRequest.ProtoReflect.Descriptor instead. +func (*ListEventsRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{50} +} + +func (x *ListEventsRequest) GetLimit() int64 { + if x != nil { + return x.Limit + } + return 0 +} + +func (x *ListEventsRequest) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *ListEventsRequest) GetWorkloadId() string { + if x != nil { + return x.WorkloadId + } + return "" +} + +func (x *ListEventsRequest) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + +type ListEventsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Events []*SchedulerEventView `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListEventsResponse) Reset() { + *x = ListEventsResponse{} + mi := &file_control_proto_msgTypes[51] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListEventsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListEventsResponse) ProtoMessage() {} + +func (x *ListEventsResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[51] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListEventsResponse.ProtoReflect.Descriptor instead. +func (*ListEventsResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{51} +} + +func (x *ListEventsResponse) GetEvents() []*SchedulerEventView { + if x != nil { + return x.Events + } + return nil +} + +type WatchEventsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Same optional filters as ListEventsRequest. The stream first replays + // recent matching events (server-side default limit), then continues + // with new matching events as they're emitted. + Type string `protobuf:"bytes,1,opt,name=type,proto3" json:"type,omitempty"` + WorkloadId string `protobuf:"bytes,2,opt,name=workload_id,json=workloadId,proto3" json:"workload_id,omitempty"` + NodeId string `protobuf:"bytes,3,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WatchEventsRequest) Reset() { + *x = WatchEventsRequest{} + mi := &file_control_proto_msgTypes[52] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WatchEventsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WatchEventsRequest) ProtoMessage() {} + +func (x *WatchEventsRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[52] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WatchEventsRequest.ProtoReflect.Descriptor instead. +func (*WatchEventsRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{52} +} + +func (x *WatchEventsRequest) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *WatchEventsRequest) GetWorkloadId() string { + if x != nil { + return x.WorkloadId + } + return "" +} + +func (x *WatchEventsRequest) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + type GetWorkloadRequest struct { state protoimpl.MessageState `protogen:"open.v1"` WorkloadId string `protobuf:"bytes,1,opt,name=workload_id,json=workloadId,proto3" json:"workload_id,omitempty"` @@ -3492,7 +3765,7 @@ type GetWorkloadRequest struct { func (x *GetWorkloadRequest) Reset() { *x = GetWorkloadRequest{} - mi := &file_control_proto_msgTypes[49] + mi := &file_control_proto_msgTypes[53] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3504,7 +3777,7 @@ func (x *GetWorkloadRequest) String() string { func (*GetWorkloadRequest) ProtoMessage() {} func (x *GetWorkloadRequest) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[49] + mi := &file_control_proto_msgTypes[53] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3517,7 +3790,7 @@ func (x *GetWorkloadRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadRequest.ProtoReflect.Descriptor instead. func (*GetWorkloadRequest) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{49} + return file_control_proto_rawDescGZIP(), []int{53} } func (x *GetWorkloadRequest) GetWorkloadId() string { @@ -3536,7 +3809,7 @@ type ListWorkloadsResponse struct { func (x *ListWorkloadsResponse) Reset() { *x = ListWorkloadsResponse{} - mi := &file_control_proto_msgTypes[50] + mi := &file_control_proto_msgTypes[54] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3548,7 +3821,7 @@ func (x *ListWorkloadsResponse) String() string { func (*ListWorkloadsResponse) ProtoMessage() {} func (x *ListWorkloadsResponse) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[50] + mi := &file_control_proto_msgTypes[54] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3561,7 +3834,7 @@ func (x *ListWorkloadsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListWorkloadsResponse.ProtoReflect.Descriptor instead. func (*ListWorkloadsResponse) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{50} + return file_control_proto_rawDescGZIP(), []int{54} } func (x *ListWorkloadsResponse) GetWorkloads() []*WorkloadView { @@ -3580,7 +3853,7 @@ type GetWorkloadResponse struct { func (x *GetWorkloadResponse) Reset() { *x = GetWorkloadResponse{} - mi := &file_control_proto_msgTypes[51] + mi := &file_control_proto_msgTypes[55] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -3592,7 +3865,7 @@ func (x *GetWorkloadResponse) String() string { func (*GetWorkloadResponse) ProtoMessage() {} func (x *GetWorkloadResponse) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[51] + mi := &file_control_proto_msgTypes[55] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3605,7 +3878,7 @@ func (x *GetWorkloadResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetWorkloadResponse.ProtoReflect.Descriptor instead. func (*GetWorkloadResponse) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{51} + return file_control_proto_rawDescGZIP(), []int{55} } func (x *GetWorkloadResponse) GetWorkload() *WorkloadView { @@ -3637,19 +3910,1619 @@ type WorkloadView struct { func (x *WorkloadView) Reset() { *x = WorkloadView{} - mi := &file_control_proto_msgTypes[52] + mi := &file_control_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkloadView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkloadView) ProtoMessage() {} + +func (x *WorkloadView) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkloadView.ProtoReflect.Descriptor instead. +func (*WorkloadView) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{56} +} + +func (x *WorkloadView) GetWorkloadId() string { + if x != nil { + return x.WorkloadId + } + return "" +} + +func (x *WorkloadView) GetType() string { + if x != nil { + return x.Type + } + return "" +} + +func (x *WorkloadView) GetDesiredState() string { + if x != nil { + return x.DesiredState + } + return "" +} + +func (x *WorkloadView) GetStatus() string { + if x != nil { + return x.Status + } + return "" +} + +func (x *WorkloadView) GetAssignedNodeId() string { + if x != nil { + return x.AssignedNodeId + } + return "" +} + +func (x *WorkloadView) GetRevisionId() string { + if x != nil { + return x.RevisionId + } + return "" +} + +func (x *WorkloadView) GetRetryAttempts() int32 { + if x != nil { + return x.RetryAttempts + } + return 0 +} + +func (x *WorkloadView) GetRetryMaxAttempts() int32 { + if x != nil { + return x.RetryMaxAttempts + } + return 0 +} + +func (x *WorkloadView) GetRetryNextAt() *timestamppb.Timestamp { + if x != nil { + return x.RetryNextAt + } + return nil +} + +func (x *WorkloadView) GetFailureReason() string { + if x != nil { + return x.FailureReason + } + return "" +} + +func (x *WorkloadView) GetLastUpdated() *timestamppb.Timestamp { + if x != nil { + return x.LastUpdated + } + return nil +} + +func (x *WorkloadView) GetReason() *ReasonDetail { + if x != nil { + return x.Reason + } + return nil +} + +func (x *WorkloadView) GetUsage() *WorkloadUsageSnapshot { + if x != nil { + return x.Usage + } + return nil +} + +func (x *WorkloadView) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +type GetClusterSummaryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetClusterSummaryRequest) Reset() { + *x = GetClusterSummaryRequest{} + mi := &file_control_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetClusterSummaryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetClusterSummaryRequest) ProtoMessage() {} + +func (x *GetClusterSummaryRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetClusterSummaryRequest.ProtoReflect.Descriptor instead. +func (*GetClusterSummaryRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{57} +} + +type GetClusterSummaryResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + TotalNodes int32 `protobuf:"varint,1,opt,name=total_nodes,json=totalNodes,proto3" json:"total_nodes,omitempty"` + ReadyNodes int32 `protobuf:"varint,2,opt,name=ready_nodes,json=readyNodes,proto3" json:"ready_nodes,omitempty"` + NotReadyNodes int32 `protobuf:"varint,3,opt,name=not_ready_nodes,json=notReadyNodes,proto3" json:"not_ready_nodes,omitempty"` + TotalWorkloads int32 `protobuf:"varint,4,opt,name=total_workloads,json=totalWorkloads,proto3" json:"total_workloads,omitempty"` + RunningWorkloads int32 `protobuf:"varint,5,opt,name=running_workloads,json=runningWorkloads,proto3" json:"running_workloads,omitempty"` + PendingWorkloads int32 `protobuf:"varint,6,opt,name=pending_workloads,json=pendingWorkloads,proto3" json:"pending_workloads,omitempty"` + FailedWorkloads int32 `protobuf:"varint,7,opt,name=failed_workloads,json=failedWorkloads,proto3" json:"failed_workloads,omitempty"` + DeletedWorkloads int32 `protobuf:"varint,8,opt,name=deleted_workloads,json=deletedWorkloads,proto3" json:"deleted_workloads,omitempty"` + GeneratedAt *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=generated_at,json=generatedAt,proto3" json:"generated_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetClusterSummaryResponse) Reset() { + *x = GetClusterSummaryResponse{} + mi := &file_control_proto_msgTypes[58] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetClusterSummaryResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetClusterSummaryResponse) ProtoMessage() {} + +func (x *GetClusterSummaryResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[58] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetClusterSummaryResponse.ProtoReflect.Descriptor instead. +func (*GetClusterSummaryResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{58} +} + +func (x *GetClusterSummaryResponse) GetTotalNodes() int32 { + if x != nil { + return x.TotalNodes + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetReadyNodes() int32 { + if x != nil { + return x.ReadyNodes + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetNotReadyNodes() int32 { + if x != nil { + return x.NotReadyNodes + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetTotalWorkloads() int32 { + if x != nil { + return x.TotalWorkloads + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetRunningWorkloads() int32 { + if x != nil { + return x.RunningWorkloads + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetPendingWorkloads() int32 { + if x != nil { + return x.PendingWorkloads + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetFailedWorkloads() int32 { + if x != nil { + return x.FailedWorkloads + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetDeletedWorkloads() int32 { + if x != nil { + return x.DeletedWorkloads + } + return 0 +} + +func (x *GetClusterSummaryResponse) GetGeneratedAt() *timestamppb.Timestamp { + if x != nil { + return x.GeneratedAt + } + return nil +} + +type ControlMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Message: + // + // *ControlMessage_Register + // *ControlMessage_Heartbeat + // *ControlMessage_Apply + // *ControlMessage_Delete + Message isControlMessage_Message `protobuf_oneof:"message"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ControlMessage) Reset() { + *x = ControlMessage{} + mi := &file_control_proto_msgTypes[59] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ControlMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ControlMessage) ProtoMessage() {} + +func (x *ControlMessage) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[59] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ControlMessage.ProtoReflect.Descriptor instead. +func (*ControlMessage) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{59} +} + +func (x *ControlMessage) GetMessage() isControlMessage_Message { + if x != nil { + return x.Message + } + return nil +} + +func (x *ControlMessage) GetRegister() *RegisterNodeRequest { + if x != nil { + if x, ok := x.Message.(*ControlMessage_Register); ok { + return x.Register + } + } + return nil +} + +func (x *ControlMessage) GetHeartbeat() *HeartbeatRequest { + if x != nil { + if x, ok := x.Message.(*ControlMessage_Heartbeat); ok { + return x.Heartbeat + } + } + return nil +} + +func (x *ControlMessage) GetApply() *ApplyWorkloadRequest { + if x != nil { + if x, ok := x.Message.(*ControlMessage_Apply); ok { + return x.Apply + } + } + return nil +} + +func (x *ControlMessage) GetDelete() *DeleteWorkloadRequest { + if x != nil { + if x, ok := x.Message.(*ControlMessage_Delete); ok { + return x.Delete + } + } + return nil +} + +type isControlMessage_Message interface { + isControlMessage_Message() +} + +type ControlMessage_Register struct { + Register *RegisterNodeRequest `protobuf:"bytes,1,opt,name=register,proto3,oneof"` +} + +type ControlMessage_Heartbeat struct { + Heartbeat *HeartbeatRequest `protobuf:"bytes,2,opt,name=heartbeat,proto3,oneof"` +} + +type ControlMessage_Apply struct { + Apply *ApplyWorkloadRequest `protobuf:"bytes,3,opt,name=apply,proto3,oneof"` +} + +type ControlMessage_Delete struct { + Delete *DeleteWorkloadRequest `protobuf:"bytes,4,opt,name=delete,proto3,oneof"` +} + +func (*ControlMessage_Register) isControlMessage_Message() {} + +func (*ControlMessage_Heartbeat) isControlMessage_Message() {} + +func (*ControlMessage_Apply) isControlMessage_Message() {} + +func (*ControlMessage_Delete) isControlMessage_Message() {} + +type CreateDiskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Driver string `protobuf:"bytes,2,opt,name=driver,proto3" json:"driver,omitempty"` // local | ceph-rbd | nfs + SizeGb int64 `protobuf:"varint,3,opt,name=size_gb,json=sizeGb,proto3" json:"size_gb,omitempty"` + FsType string `protobuf:"bytes,4,opt,name=fs_type,json=fsType,proto3" json:"fs_type,omitempty"` + AccessMode string `protobuf:"bytes,5,opt,name=access_mode,json=accessMode,proto3" json:"access_mode,omitempty"` + RetainPolicy string `protobuf:"bytes,6,opt,name=retain_policy,json=retainPolicy,proto3" json:"retain_policy,omitempty"` // Delete | Retain + NodeId string `protobuf:"bytes,7,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` // optional pre-pin for local + MountPath string `protobuf:"bytes,8,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateDiskRequest) Reset() { + *x = CreateDiskRequest{} + mi := &file_control_proto_msgTypes[60] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateDiskRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateDiskRequest) ProtoMessage() {} + +func (x *CreateDiskRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[60] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateDiskRequest.ProtoReflect.Descriptor instead. +func (*CreateDiskRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{60} +} + +func (x *CreateDiskRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CreateDiskRequest) GetDriver() string { + if x != nil { + return x.Driver + } + return "" +} + +func (x *CreateDiskRequest) GetSizeGb() int64 { + if x != nil { + return x.SizeGb + } + return 0 +} + +func (x *CreateDiskRequest) GetFsType() string { + if x != nil { + return x.FsType + } + return "" +} + +func (x *CreateDiskRequest) GetAccessMode() string { + if x != nil { + return x.AccessMode + } + return "" +} + +func (x *CreateDiskRequest) GetRetainPolicy() string { + if x != nil { + return x.RetainPolicy + } + return "" +} + +func (x *CreateDiskRequest) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + +func (x *CreateDiskRequest) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + +type CreateDiskResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Disk *DiskView `protobuf:"bytes,1,opt,name=disk,proto3" json:"disk,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateDiskResponse) Reset() { + *x = CreateDiskResponse{} + mi := &file_control_proto_msgTypes[61] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateDiskResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateDiskResponse) ProtoMessage() {} + +func (x *CreateDiskResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[61] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateDiskResponse.ProtoReflect.Descriptor instead. +func (*CreateDiskResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{61} +} + +func (x *CreateDiskResponse) GetDisk() *DiskView { + if x != nil { + return x.Disk + } + return nil +} + +type ListDisksRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListDisksRequest) Reset() { + *x = ListDisksRequest{} + mi := &file_control_proto_msgTypes[62] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListDisksRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListDisksRequest) ProtoMessage() {} + +func (x *ListDisksRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[62] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListDisksRequest.ProtoReflect.Descriptor instead. +func (*ListDisksRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{62} +} + +type ListDisksResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Disks []*DiskView `protobuf:"bytes,1,rep,name=disks,proto3" json:"disks,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListDisksResponse) Reset() { + *x = ListDisksResponse{} + mi := &file_control_proto_msgTypes[63] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListDisksResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListDisksResponse) ProtoMessage() {} + +func (x *ListDisksResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[63] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListDisksResponse.ProtoReflect.Descriptor instead. +func (*ListDisksResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{63} +} + +func (x *ListDisksResponse) GetDisks() []*DiskView { + if x != nil { + return x.Disks + } + return nil +} + +type GetDiskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + DiskId string `protobuf:"bytes,1,opt,name=disk_id,json=diskId,proto3" json:"disk_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetDiskRequest) Reset() { + *x = GetDiskRequest{} + mi := &file_control_proto_msgTypes[64] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetDiskRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetDiskRequest) ProtoMessage() {} + +func (x *GetDiskRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[64] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetDiskRequest.ProtoReflect.Descriptor instead. +func (*GetDiskRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{64} +} + +func (x *GetDiskRequest) GetDiskId() string { + if x != nil { + return x.DiskId + } + return "" +} + +type GetDiskResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Disk *DiskView `protobuf:"bytes,1,opt,name=disk,proto3" json:"disk,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetDiskResponse) Reset() { + *x = GetDiskResponse{} + mi := &file_control_proto_msgTypes[65] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetDiskResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetDiskResponse) ProtoMessage() {} + +func (x *GetDiskResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[65] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetDiskResponse.ProtoReflect.Descriptor instead. +func (*GetDiskResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{65} +} + +func (x *GetDiskResponse) GetDisk() *DiskView { + if x != nil { + return x.Disk + } + return nil +} + +type DeleteDiskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + DiskId string `protobuf:"bytes,1,opt,name=disk_id,json=diskId,proto3" json:"disk_id,omitempty"` + Force bool `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteDiskRequest) Reset() { + *x = DeleteDiskRequest{} + mi := &file_control_proto_msgTypes[66] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteDiskRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteDiskRequest) ProtoMessage() {} + +func (x *DeleteDiskRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[66] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteDiskRequest.ProtoReflect.Descriptor instead. +func (*DeleteDiskRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{66} +} + +func (x *DeleteDiskRequest) GetDiskId() string { + if x != nil { + return x.DiskId + } + return "" +} + +func (x *DeleteDiskRequest) GetForce() bool { + if x != nil { + return x.Force + } + return false +} + +type DeleteDiskResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteDiskResponse) Reset() { + *x = DeleteDiskResponse{} + mi := &file_control_proto_msgTypes[67] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteDiskResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteDiskResponse) ProtoMessage() {} + +func (x *DeleteDiskResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[67] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteDiskResponse.ProtoReflect.Descriptor instead. +func (*DeleteDiskResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{67} +} + +func (x *DeleteDiskResponse) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +func (x *DeleteDiskResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +type DiskView struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Driver string `protobuf:"bytes,3,opt,name=driver,proto3" json:"driver,omitempty"` + SizeGb int64 `protobuf:"varint,4,opt,name=size_gb,json=sizeGb,proto3" json:"size_gb,omitempty"` + FsType string `protobuf:"bytes,5,opt,name=fs_type,json=fsType,proto3" json:"fs_type,omitempty"` + AccessMode string `protobuf:"bytes,6,opt,name=access_mode,json=accessMode,proto3" json:"access_mode,omitempty"` + RetainPolicy string `protobuf:"bytes,7,opt,name=retain_policy,json=retainPolicy,proto3" json:"retain_policy,omitempty"` + Phase string `protobuf:"bytes,8,opt,name=phase,proto3" json:"phase,omitempty"` + LastError string `protobuf:"bytes,9,opt,name=last_error,json=lastError,proto3" json:"last_error,omitempty"` + NodeId string `protobuf:"bytes,10,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` + Device string `protobuf:"bytes,11,opt,name=device,proto3" json:"device,omitempty"` + Standalone bool `protobuf:"varint,12,opt,name=standalone,proto3" json:"standalone,omitempty"` + MountPath string `protobuf:"bytes,13,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + WorkloadRefs []string `protobuf:"bytes,14,rep,name=workload_refs,json=workloadRefs,proto3" json:"workload_refs,omitempty"` + AttachedNodes []string `protobuf:"bytes,15,rep,name=attached_nodes,json=attachedNodes,proto3" json:"attached_nodes,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,16,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,17,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DiskView) Reset() { + *x = DiskView{} + mi := &file_control_proto_msgTypes[68] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DiskView) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DiskView) ProtoMessage() {} + +func (x *DiskView) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[68] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DiskView.ProtoReflect.Descriptor instead. +func (*DiskView) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{68} +} + +func (x *DiskView) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *DiskView) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *DiskView) GetDriver() string { + if x != nil { + return x.Driver + } + return "" +} + +func (x *DiskView) GetSizeGb() int64 { + if x != nil { + return x.SizeGb + } + return 0 +} + +func (x *DiskView) GetFsType() string { + if x != nil { + return x.FsType + } + return "" +} + +func (x *DiskView) GetAccessMode() string { + if x != nil { + return x.AccessMode + } + return "" +} + +func (x *DiskView) GetRetainPolicy() string { + if x != nil { + return x.RetainPolicy + } + return "" +} + +func (x *DiskView) GetPhase() string { + if x != nil { + return x.Phase + } + return "" +} + +func (x *DiskView) GetLastError() string { + if x != nil { + return x.LastError + } + return "" +} + +func (x *DiskView) GetNodeId() string { + if x != nil { + return x.NodeId + } + return "" +} + +func (x *DiskView) GetDevice() string { + if x != nil { + return x.Device + } + return "" +} + +func (x *DiskView) GetStandalone() bool { + if x != nil { + return x.Standalone + } + return false +} + +func (x *DiskView) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + +func (x *DiskView) GetWorkloadRefs() []string { + if x != nil { + return x.WorkloadRefs + } + return nil +} + +func (x *DiskView) GetAttachedNodes() []string { + if x != nil { + return x.AttachedNodes + } + return nil +} + +func (x *DiskView) GetCreatedAt() *timestamppb.Timestamp { + if x != nil { + return x.CreatedAt + } + return nil +} + +func (x *DiskView) GetUpdatedAt() *timestamppb.Timestamp { + if x != nil { + return x.UpdatedAt + } + return nil +} + +type CreateBucketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` + Versioning bool `protobuf:"varint,3,opt,name=versioning,proto3" json:"versioning,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBucketRequest) Reset() { + *x = CreateBucketRequest{} + mi := &file_control_proto_msgTypes[69] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBucketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBucketRequest) ProtoMessage() {} + +func (x *CreateBucketRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[69] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBucketRequest.ProtoReflect.Descriptor instead. +func (*CreateBucketRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{69} +} + +func (x *CreateBucketRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CreateBucketRequest) GetRegion() string { + if x != nil { + return x.Region + } + return "" +} + +func (x *CreateBucketRequest) GetVersioning() bool { + if x != nil { + return x.Versioning + } + return false +} + +type CreateBucketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bucket *BucketView `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + Access *BucketAccess `protobuf:"bytes,2,opt,name=access,proto3" json:"access,omitempty"` // credentials returned once on create + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateBucketResponse) Reset() { + *x = CreateBucketResponse{} + mi := &file_control_proto_msgTypes[70] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateBucketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateBucketResponse) ProtoMessage() {} + +func (x *CreateBucketResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[70] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateBucketResponse.ProtoReflect.Descriptor instead. +func (*CreateBucketResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{70} +} + +func (x *CreateBucketResponse) GetBucket() *BucketView { + if x != nil { + return x.Bucket + } + return nil +} + +func (x *CreateBucketResponse) GetAccess() *BucketAccess { + if x != nil { + return x.Access + } + return nil +} + +type ListBucketsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketsRequest) Reset() { + *x = ListBucketsRequest{} + mi := &file_control_proto_msgTypes[71] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketsRequest) ProtoMessage() {} + +func (x *ListBucketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[71] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketsRequest.ProtoReflect.Descriptor instead. +func (*ListBucketsRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{71} +} + +type ListBucketsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Buckets []*BucketView `protobuf:"bytes,1,rep,name=buckets,proto3" json:"buckets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketsResponse) Reset() { + *x = ListBucketsResponse{} + mi := &file_control_proto_msgTypes[72] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketsResponse) ProtoMessage() {} + +func (x *ListBucketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[72] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketsResponse.ProtoReflect.Descriptor instead. +func (*ListBucketsResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{72} +} + +func (x *ListBucketsResponse) GetBuckets() []*BucketView { + if x != nil { + return x.Buckets + } + return nil +} + +type GetBucketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` // id or name + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketRequest) Reset() { + *x = GetBucketRequest{} + mi := &file_control_proto_msgTypes[73] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketRequest) ProtoMessage() {} + +func (x *GetBucketRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[73] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketRequest.ProtoReflect.Descriptor instead. +func (*GetBucketRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{73} +} + +func (x *GetBucketRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +type GetBucketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Bucket *BucketView `protobuf:"bytes,1,opt,name=bucket,proto3" json:"bucket,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketResponse) Reset() { + *x = GetBucketResponse{} + mi := &file_control_proto_msgTypes[74] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketResponse) ProtoMessage() {} + +func (x *GetBucketResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[74] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketResponse.ProtoReflect.Descriptor instead. +func (*GetBucketResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{74} +} + +func (x *GetBucketResponse) GetBucket() *BucketView { + if x != nil { + return x.Bucket + } + return nil +} + +type DeleteBucketRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` + Force bool `protobuf:"varint,2,opt,name=force,proto3" json:"force,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBucketRequest) Reset() { + *x = DeleteBucketRequest{} + mi := &file_control_proto_msgTypes[75] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBucketRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBucketRequest) ProtoMessage() {} + +func (x *DeleteBucketRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[75] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBucketRequest.ProtoReflect.Descriptor instead. +func (*DeleteBucketRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{75} +} + +func (x *DeleteBucketRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +func (x *DeleteBucketRequest) GetForce() bool { + if x != nil { + return x.Force + } + return false +} + +type DeleteBucketResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Success bool `protobuf:"varint,1,opt,name=success,proto3" json:"success,omitempty"` + ErrorMessage string `protobuf:"bytes,2,opt,name=error_message,json=errorMessage,proto3" json:"error_message,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteBucketResponse) Reset() { + *x = DeleteBucketResponse{} + mi := &file_control_proto_msgTypes[76] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteBucketResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteBucketResponse) ProtoMessage() {} + +func (x *DeleteBucketResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[76] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteBucketResponse.ProtoReflect.Descriptor instead. +func (*DeleteBucketResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{76} +} + +func (x *DeleteBucketResponse) GetSuccess() bool { + if x != nil { + return x.Success + } + return false +} + +func (x *DeleteBucketResponse) GetErrorMessage() string { + if x != nil { + return x.ErrorMessage + } + return "" +} + +type GetBucketAccessRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketAccessRequest) Reset() { + *x = GetBucketAccessRequest{} + mi := &file_control_proto_msgTypes[77] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketAccessRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketAccessRequest) ProtoMessage() {} + +func (x *GetBucketAccessRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[77] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketAccessRequest.ProtoReflect.Descriptor instead. +func (*GetBucketAccessRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{77} +} + +func (x *GetBucketAccessRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +type GetBucketAccessResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Access *BucketAccess `protobuf:"bytes,1,opt,name=access,proto3" json:"access,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBucketAccessResponse) Reset() { + *x = GetBucketAccessResponse{} + mi := &file_control_proto_msgTypes[78] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBucketAccessResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBucketAccessResponse) ProtoMessage() {} + +func (x *GetBucketAccessResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[78] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBucketAccessResponse.ProtoReflect.Descriptor instead. +func (*GetBucketAccessResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{78} +} + +func (x *GetBucketAccessResponse) GetAccess() *BucketAccess { + if x != nil { + return x.Access + } + return nil +} + +type ListBucketObjectsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + BucketId string `protobuf:"bytes,1,opt,name=bucket_id,json=bucketId,proto3" json:"bucket_id,omitempty"` + Prefix string `protobuf:"bytes,2,opt,name=prefix,proto3" json:"prefix,omitempty"` + ContinuationToken string `protobuf:"bytes,3,opt,name=continuation_token,json=continuationToken,proto3" json:"continuation_token,omitempty"` + MaxKeys int32 `protobuf:"varint,4,opt,name=max_keys,json=maxKeys,proto3" json:"max_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketObjectsRequest) Reset() { + *x = ListBucketObjectsRequest{} + mi := &file_control_proto_msgTypes[79] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketObjectsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketObjectsRequest) ProtoMessage() {} + +func (x *ListBucketObjectsRequest) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[79] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketObjectsRequest.ProtoReflect.Descriptor instead. +func (*ListBucketObjectsRequest) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{79} +} + +func (x *ListBucketObjectsRequest) GetBucketId() string { + if x != nil { + return x.BucketId + } + return "" +} + +func (x *ListBucketObjectsRequest) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +func (x *ListBucketObjectsRequest) GetContinuationToken() string { + if x != nil { + return x.ContinuationToken + } + return "" +} + +func (x *ListBucketObjectsRequest) GetMaxKeys() int32 { + if x != nil { + return x.MaxKeys + } + return 0 +} + +type ListBucketObjectsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Objects []*ObjectInfo `protobuf:"bytes,1,rep,name=objects,proto3" json:"objects,omitempty"` + NextContinuationToken string `protobuf:"bytes,2,opt,name=next_continuation_token,json=nextContinuationToken,proto3" json:"next_continuation_token,omitempty"` + IsTruncated bool `protobuf:"varint,3,opt,name=is_truncated,json=isTruncated,proto3" json:"is_truncated,omitempty"` + Prefix string `protobuf:"bytes,4,opt,name=prefix,proto3" json:"prefix,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListBucketObjectsResponse) Reset() { + *x = ListBucketObjectsResponse{} + mi := &file_control_proto_msgTypes[80] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListBucketObjectsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListBucketObjectsResponse) ProtoMessage() {} + +func (x *ListBucketObjectsResponse) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[80] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListBucketObjectsResponse.ProtoReflect.Descriptor instead. +func (*ListBucketObjectsResponse) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{80} +} + +func (x *ListBucketObjectsResponse) GetObjects() []*ObjectInfo { + if x != nil { + return x.Objects + } + return nil +} + +func (x *ListBucketObjectsResponse) GetNextContinuationToken() string { + if x != nil { + return x.NextContinuationToken + } + return "" +} + +func (x *ListBucketObjectsResponse) GetIsTruncated() bool { + if x != nil { + return x.IsTruncated + } + return false +} + +func (x *ListBucketObjectsResponse) GetPrefix() string { + if x != nil { + return x.Prefix + } + return "" +} + +type BucketView struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Region string `protobuf:"bytes,3,opt,name=region,proto3" json:"region,omitempty"` + Owner string `protobuf:"bytes,4,opt,name=owner,proto3" json:"owner,omitempty"` + Endpoint string `protobuf:"bytes,5,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + Versioning bool `protobuf:"varint,6,opt,name=versioning,proto3" json:"versioning,omitempty"` + ObjectCount int64 `protobuf:"varint,7,opt,name=object_count,json=objectCount,proto3" json:"object_count,omitempty"` + SizeBytes int64 `protobuf:"varint,8,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + Phase string `protobuf:"bytes,9,opt,name=phase,proto3" json:"phase,omitempty"` + LastError string `protobuf:"bytes,10,opt,name=last_error,json=lastError,proto3" json:"last_error,omitempty"` + CreatedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=created_at,json=createdAt,proto3" json:"created_at,omitempty"` + UpdatedAt *timestamppb.Timestamp `protobuf:"bytes,12,opt,name=updated_at,json=updatedAt,proto3" json:"updated_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BucketView) Reset() { + *x = BucketView{} + mi := &file_control_proto_msgTypes[81] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *WorkloadView) String() string { +func (x *BucketView) String() string { return protoimpl.X.MessageStringOf(x) } -func (*WorkloadView) ProtoMessage() {} +func (*BucketView) ProtoMessage() {} -func (x *WorkloadView) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[52] +func (x *BucketView) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[81] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3660,175 +5533,123 @@ func (x *WorkloadView) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use WorkloadView.ProtoReflect.Descriptor instead. -func (*WorkloadView) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{52} +// Deprecated: Use BucketView.ProtoReflect.Descriptor instead. +func (*BucketView) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{81} } -func (x *WorkloadView) GetWorkloadId() string { +func (x *BucketView) GetId() string { if x != nil { - return x.WorkloadId + return x.Id } return "" } -func (x *WorkloadView) GetType() string { +func (x *BucketView) GetName() string { if x != nil { - return x.Type + return x.Name } return "" } -func (x *WorkloadView) GetDesiredState() string { +func (x *BucketView) GetRegion() string { if x != nil { - return x.DesiredState + return x.Region } return "" } -func (x *WorkloadView) GetStatus() string { +func (x *BucketView) GetOwner() string { if x != nil { - return x.Status + return x.Owner } return "" } -func (x *WorkloadView) GetAssignedNodeId() string { +func (x *BucketView) GetEndpoint() string { if x != nil { - return x.AssignedNodeId + return x.Endpoint } return "" } -func (x *WorkloadView) GetRevisionId() string { +func (x *BucketView) GetVersioning() bool { if x != nil { - return x.RevisionId + return x.Versioning } - return "" + return false } -func (x *WorkloadView) GetRetryAttempts() int32 { +func (x *BucketView) GetObjectCount() int64 { if x != nil { - return x.RetryAttempts + return x.ObjectCount } return 0 } -func (x *WorkloadView) GetRetryMaxAttempts() int32 { +func (x *BucketView) GetSizeBytes() int64 { if x != nil { - return x.RetryMaxAttempts + return x.SizeBytes } return 0 } -func (x *WorkloadView) GetRetryNextAt() *timestamppb.Timestamp { - if x != nil { - return x.RetryNextAt - } - return nil -} - -func (x *WorkloadView) GetFailureReason() string { +func (x *BucketView) GetPhase() string { if x != nil { - return x.FailureReason + return x.Phase } return "" } -func (x *WorkloadView) GetLastUpdated() *timestamppb.Timestamp { - if x != nil { - return x.LastUpdated - } - return nil -} - -func (x *WorkloadView) GetReason() *ReasonDetail { +func (x *BucketView) GetLastError() string { if x != nil { - return x.Reason + return x.LastError } - return nil + return "" } -func (x *WorkloadView) GetUsage() *WorkloadUsageSnapshot { +func (x *BucketView) GetCreatedAt() *timestamppb.Timestamp { if x != nil { - return x.Usage + return x.CreatedAt } return nil } -func (x *WorkloadView) GetCreatedAt() *timestamppb.Timestamp { +func (x *BucketView) GetUpdatedAt() *timestamppb.Timestamp { if x != nil { - return x.CreatedAt + return x.UpdatedAt } return nil } -type GetClusterSummaryRequest struct { +type BucketAccess struct { state protoimpl.MessageState `protogen:"open.v1"` + Endpoint string `protobuf:"bytes,1,opt,name=endpoint,proto3" json:"endpoint,omitempty"` + Region string `protobuf:"bytes,2,opt,name=region,proto3" json:"region,omitempty"` + Bucket string `protobuf:"bytes,3,opt,name=bucket,proto3" json:"bucket,omitempty"` + AccessKey string `protobuf:"bytes,4,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` + SecretKey string `protobuf:"bytes,5,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"` + VaultPath string `protobuf:"bytes,6,opt,name=vault_path,json=vaultPath,proto3" json:"vault_path,omitempty"` // when secrets live in Vault + S3Url string `protobuf:"bytes,7,opt,name=s3_url,json=s3Url,proto3" json:"s3_url,omitempty"` // e.g. s3://bucket unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *GetClusterSummaryRequest) Reset() { - *x = GetClusterSummaryRequest{} - mi := &file_control_proto_msgTypes[53] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) -} - -func (x *GetClusterSummaryRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*GetClusterSummaryRequest) ProtoMessage() {} - -func (x *GetClusterSummaryRequest) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[53] - if x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use GetClusterSummaryRequest.ProtoReflect.Descriptor instead. -func (*GetClusterSummaryRequest) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{53} -} - -type GetClusterSummaryResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - TotalNodes int32 `protobuf:"varint,1,opt,name=total_nodes,json=totalNodes,proto3" json:"total_nodes,omitempty"` - ReadyNodes int32 `protobuf:"varint,2,opt,name=ready_nodes,json=readyNodes,proto3" json:"ready_nodes,omitempty"` - NotReadyNodes int32 `protobuf:"varint,3,opt,name=not_ready_nodes,json=notReadyNodes,proto3" json:"not_ready_nodes,omitempty"` - TotalWorkloads int32 `protobuf:"varint,4,opt,name=total_workloads,json=totalWorkloads,proto3" json:"total_workloads,omitempty"` - RunningWorkloads int32 `protobuf:"varint,5,opt,name=running_workloads,json=runningWorkloads,proto3" json:"running_workloads,omitempty"` - PendingWorkloads int32 `protobuf:"varint,6,opt,name=pending_workloads,json=pendingWorkloads,proto3" json:"pending_workloads,omitempty"` - FailedWorkloads int32 `protobuf:"varint,7,opt,name=failed_workloads,json=failedWorkloads,proto3" json:"failed_workloads,omitempty"` - DeletedWorkloads int32 `protobuf:"varint,8,opt,name=deleted_workloads,json=deletedWorkloads,proto3" json:"deleted_workloads,omitempty"` - GeneratedAt *timestamppb.Timestamp `protobuf:"bytes,9,opt,name=generated_at,json=generatedAt,proto3" json:"generated_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache -} - -func (x *GetClusterSummaryResponse) Reset() { - *x = GetClusterSummaryResponse{} - mi := &file_control_proto_msgTypes[54] +func (x *BucketAccess) Reset() { + *x = BucketAccess{} + mi := &file_control_proto_msgTypes[82] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *GetClusterSummaryResponse) String() string { +func (x *BucketAccess) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetClusterSummaryResponse) ProtoMessage() {} +func (*BucketAccess) ProtoMessage() {} -func (x *GetClusterSummaryResponse) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[54] +func (x *BucketAccess) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[82] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3839,102 +5660,86 @@ func (x *GetClusterSummaryResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetClusterSummaryResponse.ProtoReflect.Descriptor instead. -func (*GetClusterSummaryResponse) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{54} -} - -func (x *GetClusterSummaryResponse) GetTotalNodes() int32 { - if x != nil { - return x.TotalNodes - } - return 0 -} - -func (x *GetClusterSummaryResponse) GetReadyNodes() int32 { - if x != nil { - return x.ReadyNodes - } - return 0 +// Deprecated: Use BucketAccess.ProtoReflect.Descriptor instead. +func (*BucketAccess) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{82} } -func (x *GetClusterSummaryResponse) GetNotReadyNodes() int32 { +func (x *BucketAccess) GetEndpoint() string { if x != nil { - return x.NotReadyNodes + return x.Endpoint } - return 0 + return "" } -func (x *GetClusterSummaryResponse) GetTotalWorkloads() int32 { +func (x *BucketAccess) GetRegion() string { if x != nil { - return x.TotalWorkloads + return x.Region } - return 0 + return "" } -func (x *GetClusterSummaryResponse) GetRunningWorkloads() int32 { +func (x *BucketAccess) GetBucket() string { if x != nil { - return x.RunningWorkloads + return x.Bucket } - return 0 + return "" } -func (x *GetClusterSummaryResponse) GetPendingWorkloads() int32 { +func (x *BucketAccess) GetAccessKey() string { if x != nil { - return x.PendingWorkloads + return x.AccessKey } - return 0 + return "" } -func (x *GetClusterSummaryResponse) GetFailedWorkloads() int32 { +func (x *BucketAccess) GetSecretKey() string { if x != nil { - return x.FailedWorkloads + return x.SecretKey } - return 0 + return "" } -func (x *GetClusterSummaryResponse) GetDeletedWorkloads() int32 { +func (x *BucketAccess) GetVaultPath() string { if x != nil { - return x.DeletedWorkloads + return x.VaultPath } - return 0 + return "" } -func (x *GetClusterSummaryResponse) GetGeneratedAt() *timestamppb.Timestamp { +func (x *BucketAccess) GetS3Url() string { if x != nil { - return x.GeneratedAt + return x.S3Url } - return nil + return "" } -type ControlMessage struct { - state protoimpl.MessageState `protogen:"open.v1"` - // Types that are valid to be assigned to Message: - // - // *ControlMessage_Register - // *ControlMessage_Heartbeat - // *ControlMessage_Apply - // *ControlMessage_Delete - Message isControlMessage_Message `protobuf_oneof:"message"` +type ObjectInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + SizeBytes int64 `protobuf:"varint,2,opt,name=size_bytes,json=sizeBytes,proto3" json:"size_bytes,omitempty"` + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + LastModified string `protobuf:"bytes,4,opt,name=last_modified,json=lastModified,proto3" json:"last_modified,omitempty"` + StorageClass string `protobuf:"bytes,5,opt,name=storage_class,json=storageClass,proto3" json:"storage_class,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *ControlMessage) Reset() { - *x = ControlMessage{} - mi := &file_control_proto_msgTypes[55] +func (x *ObjectInfo) Reset() { + *x = ObjectInfo{} + mi := &file_control_proto_msgTypes[83] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *ControlMessage) String() string { +func (x *ObjectInfo) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ControlMessage) ProtoMessage() {} +func (*ObjectInfo) ProtoMessage() {} -func (x *ControlMessage) ProtoReflect() protoreflect.Message { - mi := &file_control_proto_msgTypes[55] +func (x *ObjectInfo) ProtoReflect() protoreflect.Message { + mi := &file_control_proto_msgTypes[83] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -3945,82 +5750,46 @@ func (x *ControlMessage) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ControlMessage.ProtoReflect.Descriptor instead. -func (*ControlMessage) Descriptor() ([]byte, []int) { - return file_control_proto_rawDescGZIP(), []int{55} +// Deprecated: Use ObjectInfo.ProtoReflect.Descriptor instead. +func (*ObjectInfo) Descriptor() ([]byte, []int) { + return file_control_proto_rawDescGZIP(), []int{83} } -func (x *ControlMessage) GetMessage() isControlMessage_Message { +func (x *ObjectInfo) GetKey() string { if x != nil { - return x.Message + return x.Key } - return nil + return "" } -func (x *ControlMessage) GetRegister() *RegisterNodeRequest { +func (x *ObjectInfo) GetSizeBytes() int64 { if x != nil { - if x, ok := x.Message.(*ControlMessage_Register); ok { - return x.Register - } + return x.SizeBytes } - return nil + return 0 } -func (x *ControlMessage) GetHeartbeat() *HeartbeatRequest { +func (x *ObjectInfo) GetEtag() string { if x != nil { - if x, ok := x.Message.(*ControlMessage_Heartbeat); ok { - return x.Heartbeat - } + return x.Etag } - return nil + return "" } -func (x *ControlMessage) GetApply() *ApplyWorkloadRequest { +func (x *ObjectInfo) GetLastModified() string { if x != nil { - if x, ok := x.Message.(*ControlMessage_Apply); ok { - return x.Apply - } + return x.LastModified } - return nil + return "" } -func (x *ControlMessage) GetDelete() *DeleteWorkloadRequest { +func (x *ObjectInfo) GetStorageClass() string { if x != nil { - if x, ok := x.Message.(*ControlMessage_Delete); ok { - return x.Delete - } + return x.StorageClass } - return nil -} - -type isControlMessage_Message interface { - isControlMessage_Message() -} - -type ControlMessage_Register struct { - Register *RegisterNodeRequest `protobuf:"bytes,1,opt,name=register,proto3,oneof"` -} - -type ControlMessage_Heartbeat struct { - Heartbeat *HeartbeatRequest `protobuf:"bytes,2,opt,name=heartbeat,proto3,oneof"` -} - -type ControlMessage_Apply struct { - Apply *ApplyWorkloadRequest `protobuf:"bytes,3,opt,name=apply,proto3,oneof"` -} - -type ControlMessage_Delete struct { - Delete *DeleteWorkloadRequest `protobuf:"bytes,4,opt,name=delete,proto3,oneof"` + return "" } -func (*ControlMessage_Register) isControlMessage_Message() {} - -func (*ControlMessage_Heartbeat) isControlMessage_Message() {} - -func (*ControlMessage_Apply) isControlMessage_Message() {} - -func (*ControlMessage_Delete) isControlMessage_Message() {} - var File_control_proto protoreflect.FileDescriptor const file_control_proto_rawDesc = "" + @@ -4315,7 +6084,32 @@ const file_control_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"G\n" + "\x14ListWorkloadsRequest\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12\x16\n" + - "\x06status\x18\x02 \x01(\tR\x06status\"5\n" + + "\x06status\x18\x02 \x01(\tR\x06status\"\xce\x02\n" + + "\x12SchedulerEventView\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04type\x18\x02 \x01(\tR\x04type\x12\x1f\n" + + "\vworkload_id\x18\x03 \x01(\tR\n" + + "workloadId\x12\x17\n" + + "\anode_id\x18\x04 \x01(\tR\x06nodeId\x12\x16\n" + + "\x06reason\x18\x05 \x01(\tR\x06reason\x128\n" + + "\ttimestamp\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12L\n" + + "\adetails\x18\a \x03(\v22.persys.control.v1.SchedulerEventView.DetailsEntryR\adetails\x1a:\n" + + "\fDetailsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"w\n" + + "\x11ListEventsRequest\x12\x14\n" + + "\x05limit\x18\x01 \x01(\x03R\x05limit\x12\x12\n" + + "\x04type\x18\x02 \x01(\tR\x04type\x12\x1f\n" + + "\vworkload_id\x18\x03 \x01(\tR\n" + + "workloadId\x12\x17\n" + + "\anode_id\x18\x04 \x01(\tR\x06nodeId\"S\n" + + "\x12ListEventsResponse\x12=\n" + + "\x06events\x18\x01 \x03(\v2%.persys.control.v1.SchedulerEventViewR\x06events\"b\n" + + "\x12WatchEventsRequest\x12\x12\n" + + "\x04type\x18\x01 \x01(\tR\x04type\x12\x1f\n" + + "\vworkload_id\x18\x02 \x01(\tR\n" + + "workloadId\x12\x17\n" + + "\anode_id\x18\x03 \x01(\tR\x06nodeId\"5\n" + "\x12GetWorkloadRequest\x12\x1f\n" + "\vworkload_id\x18\x01 \x01(\tR\n" + "workloadId\"V\n" + @@ -4360,7 +6154,135 @@ const file_control_proto_rawDesc = "" + "\theartbeat\x18\x02 \x01(\v2#.persys.control.v1.HeartbeatRequestH\x00R\theartbeat\x12?\n" + "\x05apply\x18\x03 \x01(\v2'.persys.control.v1.ApplyWorkloadRequestH\x00R\x05apply\x12B\n" + "\x06delete\x18\x04 \x01(\v2(.persys.control.v1.DeleteWorkloadRequestH\x00R\x06deleteB\t\n" + - "\amessage*\xda\x01\n" + + "\amessage\"\xef\x01\n" + + "\x11CreateDiskRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x06driver\x18\x02 \x01(\tR\x06driver\x12\x17\n" + + "\asize_gb\x18\x03 \x01(\x03R\x06sizeGb\x12\x17\n" + + "\afs_type\x18\x04 \x01(\tR\x06fsType\x12\x1f\n" + + "\vaccess_mode\x18\x05 \x01(\tR\n" + + "accessMode\x12#\n" + + "\rretain_policy\x18\x06 \x01(\tR\fretainPolicy\x12\x17\n" + + "\anode_id\x18\a \x01(\tR\x06nodeId\x12\x1d\n" + + "\n" + + "mount_path\x18\b \x01(\tR\tmountPath\"E\n" + + "\x12CreateDiskResponse\x12/\n" + + "\x04disk\x18\x01 \x01(\v2\x1b.persys.control.v1.DiskViewR\x04disk\"\x12\n" + + "\x10ListDisksRequest\"F\n" + + "\x11ListDisksResponse\x121\n" + + "\x05disks\x18\x01 \x03(\v2\x1b.persys.control.v1.DiskViewR\x05disks\")\n" + + "\x0eGetDiskRequest\x12\x17\n" + + "\adisk_id\x18\x01 \x01(\tR\x06diskId\"B\n" + + "\x0fGetDiskResponse\x12/\n" + + "\x04disk\x18\x01 \x01(\v2\x1b.persys.control.v1.DiskViewR\x04disk\"B\n" + + "\x11DeleteDiskRequest\x12\x17\n" + + "\adisk_id\x18\x01 \x01(\tR\x06diskId\x12\x14\n" + + "\x05force\x18\x02 \x01(\bR\x05force\"S\n" + + "\x12DeleteDiskResponse\x12\x18\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\x12#\n" + + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage\"\xa5\x04\n" + + "\bDiskView\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x16\n" + + "\x06driver\x18\x03 \x01(\tR\x06driver\x12\x17\n" + + "\asize_gb\x18\x04 \x01(\x03R\x06sizeGb\x12\x17\n" + + "\afs_type\x18\x05 \x01(\tR\x06fsType\x12\x1f\n" + + "\vaccess_mode\x18\x06 \x01(\tR\n" + + "accessMode\x12#\n" + + "\rretain_policy\x18\a \x01(\tR\fretainPolicy\x12\x14\n" + + "\x05phase\x18\b \x01(\tR\x05phase\x12\x1d\n" + + "\n" + + "last_error\x18\t \x01(\tR\tlastError\x12\x17\n" + + "\anode_id\x18\n" + + " \x01(\tR\x06nodeId\x12\x16\n" + + "\x06device\x18\v \x01(\tR\x06device\x12\x1e\n" + + "\n" + + "standalone\x18\f \x01(\bR\n" + + "standalone\x12\x1d\n" + + "\n" + + "mount_path\x18\r \x01(\tR\tmountPath\x12#\n" + + "\rworkload_refs\x18\x0e \x03(\tR\fworkloadRefs\x12%\n" + + "\x0eattached_nodes\x18\x0f \x03(\tR\rattachedNodes\x129\n" + + "\n" + + "created_at\x18\x10 \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + + "\n" + + "updated_at\x18\x11 \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\"a\n" + + "\x13CreateBucketRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + + "\x06region\x18\x02 \x01(\tR\x06region\x12\x1e\n" + + "\n" + + "versioning\x18\x03 \x01(\bR\n" + + "versioning\"\x86\x01\n" + + "\x14CreateBucketResponse\x125\n" + + "\x06bucket\x18\x01 \x01(\v2\x1d.persys.control.v1.BucketViewR\x06bucket\x127\n" + + "\x06access\x18\x02 \x01(\v2\x1f.persys.control.v1.BucketAccessR\x06access\"\x14\n" + + "\x12ListBucketsRequest\"N\n" + + "\x13ListBucketsResponse\x127\n" + + "\abuckets\x18\x01 \x03(\v2\x1d.persys.control.v1.BucketViewR\abuckets\"/\n" + + "\x10GetBucketRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\"J\n" + + "\x11GetBucketResponse\x125\n" + + "\x06bucket\x18\x01 \x01(\v2\x1d.persys.control.v1.BucketViewR\x06bucket\"H\n" + + "\x13DeleteBucketRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\x12\x14\n" + + "\x05force\x18\x02 \x01(\bR\x05force\"U\n" + + "\x14DeleteBucketResponse\x12\x18\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\x12#\n" + + "\rerror_message\x18\x02 \x01(\tR\ferrorMessage\"5\n" + + "\x16GetBucketAccessRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\"R\n" + + "\x17GetBucketAccessResponse\x127\n" + + "\x06access\x18\x01 \x01(\v2\x1f.persys.control.v1.BucketAccessR\x06access\"\x99\x01\n" + + "\x18ListBucketObjectsRequest\x12\x1b\n" + + "\tbucket_id\x18\x01 \x01(\tR\bbucketId\x12\x16\n" + + "\x06prefix\x18\x02 \x01(\tR\x06prefix\x12-\n" + + "\x12continuation_token\x18\x03 \x01(\tR\x11continuationToken\x12\x19\n" + + "\bmax_keys\x18\x04 \x01(\x05R\amaxKeys\"\xc7\x01\n" + + "\x19ListBucketObjectsResponse\x127\n" + + "\aobjects\x18\x01 \x03(\v2\x1d.persys.control.v1.ObjectInfoR\aobjects\x126\n" + + "\x17next_continuation_token\x18\x02 \x01(\tR\x15nextContinuationToken\x12!\n" + + "\fis_truncated\x18\x03 \x01(\bR\visTruncated\x12\x16\n" + + "\x06prefix\x18\x04 \x01(\tR\x06prefix\"\x87\x03\n" + + "\n" + + "BucketView\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x16\n" + + "\x06region\x18\x03 \x01(\tR\x06region\x12\x14\n" + + "\x05owner\x18\x04 \x01(\tR\x05owner\x12\x1a\n" + + "\bendpoint\x18\x05 \x01(\tR\bendpoint\x12\x1e\n" + + "\n" + + "versioning\x18\x06 \x01(\bR\n" + + "versioning\x12!\n" + + "\fobject_count\x18\a \x01(\x03R\vobjectCount\x12\x1d\n" + + "\n" + + "size_bytes\x18\b \x01(\x03R\tsizeBytes\x12\x14\n" + + "\x05phase\x18\t \x01(\tR\x05phase\x12\x1d\n" + + "\n" + + "last_error\x18\n" + + " \x01(\tR\tlastError\x129\n" + + "\n" + + "created_at\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x129\n" + + "\n" + + "updated_at\x18\f \x01(\v2\x1a.google.protobuf.TimestampR\tupdatedAt\"\xce\x01\n" + + "\fBucketAccess\x12\x1a\n" + + "\bendpoint\x18\x01 \x01(\tR\bendpoint\x12\x16\n" + + "\x06region\x18\x02 \x01(\tR\x06region\x12\x16\n" + + "\x06bucket\x18\x03 \x01(\tR\x06bucket\x12\x1d\n" + + "\n" + + "access_key\x18\x04 \x01(\tR\taccessKey\x12\x1d\n" + + "\n" + + "secret_key\x18\x05 \x01(\tR\tsecretKey\x12\x1d\n" + + "\n" + + "vault_path\x18\x06 \x01(\tR\tvaultPath\x12\x15\n" + + "\x06s3_url\x18\a \x01(\tR\x05s3Url\"\x9b\x01\n" + + "\n" + + "ObjectInfo\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x1d\n" + + "\n" + + "size_bytes\x18\x02 \x01(\x03R\tsizeBytes\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12#\n" + + "\rlast_modified\x18\x04 \x01(\tR\flastModified\x12#\n" + + "\rstorage_class\x18\x05 \x01(\tR\fstorageClass*\xda\x01\n" + "\x14AutomationActionType\x12&\n" + "\"AUTOMATION_ACTION_TYPE_UNSPECIFIED\x10\x00\x12'\n" + "#AUTOMATION_ACTION_SET_DESIRED_STATE\x10\x01\x12$\n" + @@ -4376,7 +6298,7 @@ const file_control_proto_rawDesc = "" + "\rRUNTIME_ERROR\x10\x05\x12\x11\n" + "\rNETWORK_ERROR\x10\x06\x12\x11\n" + "\rSTORAGE_ERROR\x10\a\x12\x12\n" + - "\x0eVM_BOOT_FAILED\x10\b2\xf0\r\n" + + "\x0eVM_BOOT_FAILED\x10\b2\xdc\x16\n" + "\fAgentControl\x12_\n" + "\fRegisterNode\x12&.persys.control.v1.RegisterNodeRequest\x1a'.persys.control.v1.RegisterNodeResponse\x12V\n" + "\tHeartbeat\x12#.persys.control.v1.HeartbeatRequest\x1a$.persys.control.v1.HeartbeatResponse\x12b\n" + @@ -4395,7 +6317,22 @@ const file_control_proto_rawDesc = "" + "\rListWorkloads\x12'.persys.control.v1.ListWorkloadsRequest\x1a(.persys.control.v1.ListWorkloadsResponse\x12\\\n" + "\vGetWorkload\x12%.persys.control.v1.GetWorkloadRequest\x1a&.persys.control.v1.GetWorkloadResponse\x12n\n" + "\x11GetClusterSummary\x12+.persys.control.v1.GetClusterSummaryRequest\x1a,.persys.control.v1.GetClusterSummaryResponse\x12Y\n" + - "\rControlStream\x12!.persys.control.v1.ControlMessage\x1a!.persys.control.v1.ControlMessage(\x010\x01B7Z5github.com/persys-dev/persys/api/control/v1;controlv1b\x06proto3" + "\n" + + "ListEvents\x12$.persys.control.v1.ListEventsRequest\x1a%.persys.control.v1.ListEventsResponse\x12]\n" + + "\vWatchEvents\x12%.persys.control.v1.WatchEventsRequest\x1a%.persys.control.v1.SchedulerEventView0\x01\x12Y\n" + + "\rControlStream\x12!.persys.control.v1.ControlMessage\x1a!.persys.control.v1.ControlMessage(\x010\x01\x12Y\n" + + "\n" + + "CreateDisk\x12$.persys.control.v1.CreateDiskRequest\x1a%.persys.control.v1.CreateDiskResponse\x12V\n" + + "\tListDisks\x12#.persys.control.v1.ListDisksRequest\x1a$.persys.control.v1.ListDisksResponse\x12P\n" + + "\aGetDisk\x12!.persys.control.v1.GetDiskRequest\x1a\".persys.control.v1.GetDiskResponse\x12Y\n" + + "\n" + + "DeleteDisk\x12$.persys.control.v1.DeleteDiskRequest\x1a%.persys.control.v1.DeleteDiskResponse\x12_\n" + + "\fCreateBucket\x12&.persys.control.v1.CreateBucketRequest\x1a'.persys.control.v1.CreateBucketResponse\x12\\\n" + + "\vListBuckets\x12%.persys.control.v1.ListBucketsRequest\x1a&.persys.control.v1.ListBucketsResponse\x12V\n" + + "\tGetBucket\x12#.persys.control.v1.GetBucketRequest\x1a$.persys.control.v1.GetBucketResponse\x12_\n" + + "\fDeleteBucket\x12&.persys.control.v1.DeleteBucketRequest\x1a'.persys.control.v1.DeleteBucketResponse\x12h\n" + + "\x0fGetBucketAccess\x12).persys.control.v1.GetBucketAccessRequest\x1a*.persys.control.v1.GetBucketAccessResponse\x12n\n" + + "\x11ListBucketObjects\x12+.persys.control.v1.ListBucketObjectsRequest\x1a,.persys.control.v1.ListBucketObjectsResponseB7Z5github.com/persys-dev/persys/api/control/v1;controlv1b\x06proto3" var ( file_control_proto_rawDescOnce sync.Once @@ -4410,7 +6347,7 @@ func file_control_proto_rawDescGZIP() []byte { } var file_control_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_control_proto_msgTypes = make([]protoimpl.MessageInfo, 61) +var file_control_proto_msgTypes = make([]protoimpl.MessageInfo, 90) var file_control_proto_goTypes = []any{ (AutomationActionType)(0), // 0: persys.control.v1.AutomationActionType (FailureReason)(0), // 1: persys.control.v1.FailureReason @@ -4463,124 +6400,193 @@ var file_control_proto_goTypes = []any{ (*GetNodeResponse)(nil), // 48: persys.control.v1.GetNodeResponse (*NodeView)(nil), // 49: persys.control.v1.NodeView (*ListWorkloadsRequest)(nil), // 50: persys.control.v1.ListWorkloadsRequest - (*GetWorkloadRequest)(nil), // 51: persys.control.v1.GetWorkloadRequest - (*ListWorkloadsResponse)(nil), // 52: persys.control.v1.ListWorkloadsResponse - (*GetWorkloadResponse)(nil), // 53: persys.control.v1.GetWorkloadResponse - (*WorkloadView)(nil), // 54: persys.control.v1.WorkloadView - (*GetClusterSummaryRequest)(nil), // 55: persys.control.v1.GetClusterSummaryRequest - (*GetClusterSummaryResponse)(nil), // 56: persys.control.v1.GetClusterSummaryResponse - (*ControlMessage)(nil), // 57: persys.control.v1.ControlMessage - nil, // 58: persys.control.v1.RegisterNodeRequest.LabelsEntry - nil, // 59: persys.control.v1.WorkloadSpec.MetadataEntry - nil, // 60: persys.control.v1.ContainerSpec.EnvEntry - nil, // 61: persys.control.v1.ComposeSpec.EnvEntry - nil, // 62: persys.control.v1.NodeView.LabelsEntry - (*timestamppb.Timestamp)(nil), // 63: google.protobuf.Timestamp + (*SchedulerEventView)(nil), // 51: persys.control.v1.SchedulerEventView + (*ListEventsRequest)(nil), // 52: persys.control.v1.ListEventsRequest + (*ListEventsResponse)(nil), // 53: persys.control.v1.ListEventsResponse + (*WatchEventsRequest)(nil), // 54: persys.control.v1.WatchEventsRequest + (*GetWorkloadRequest)(nil), // 55: persys.control.v1.GetWorkloadRequest + (*ListWorkloadsResponse)(nil), // 56: persys.control.v1.ListWorkloadsResponse + (*GetWorkloadResponse)(nil), // 57: persys.control.v1.GetWorkloadResponse + (*WorkloadView)(nil), // 58: persys.control.v1.WorkloadView + (*GetClusterSummaryRequest)(nil), // 59: persys.control.v1.GetClusterSummaryRequest + (*GetClusterSummaryResponse)(nil), // 60: persys.control.v1.GetClusterSummaryResponse + (*ControlMessage)(nil), // 61: persys.control.v1.ControlMessage + (*CreateDiskRequest)(nil), // 62: persys.control.v1.CreateDiskRequest + (*CreateDiskResponse)(nil), // 63: persys.control.v1.CreateDiskResponse + (*ListDisksRequest)(nil), // 64: persys.control.v1.ListDisksRequest + (*ListDisksResponse)(nil), // 65: persys.control.v1.ListDisksResponse + (*GetDiskRequest)(nil), // 66: persys.control.v1.GetDiskRequest + (*GetDiskResponse)(nil), // 67: persys.control.v1.GetDiskResponse + (*DeleteDiskRequest)(nil), // 68: persys.control.v1.DeleteDiskRequest + (*DeleteDiskResponse)(nil), // 69: persys.control.v1.DeleteDiskResponse + (*DiskView)(nil), // 70: persys.control.v1.DiskView + (*CreateBucketRequest)(nil), // 71: persys.control.v1.CreateBucketRequest + (*CreateBucketResponse)(nil), // 72: persys.control.v1.CreateBucketResponse + (*ListBucketsRequest)(nil), // 73: persys.control.v1.ListBucketsRequest + (*ListBucketsResponse)(nil), // 74: persys.control.v1.ListBucketsResponse + (*GetBucketRequest)(nil), // 75: persys.control.v1.GetBucketRequest + (*GetBucketResponse)(nil), // 76: persys.control.v1.GetBucketResponse + (*DeleteBucketRequest)(nil), // 77: persys.control.v1.DeleteBucketRequest + (*DeleteBucketResponse)(nil), // 78: persys.control.v1.DeleteBucketResponse + (*GetBucketAccessRequest)(nil), // 79: persys.control.v1.GetBucketAccessRequest + (*GetBucketAccessResponse)(nil), // 80: persys.control.v1.GetBucketAccessResponse + (*ListBucketObjectsRequest)(nil), // 81: persys.control.v1.ListBucketObjectsRequest + (*ListBucketObjectsResponse)(nil), // 82: persys.control.v1.ListBucketObjectsResponse + (*BucketView)(nil), // 83: persys.control.v1.BucketView + (*BucketAccess)(nil), // 84: persys.control.v1.BucketAccess + (*ObjectInfo)(nil), // 85: persys.control.v1.ObjectInfo + nil, // 86: persys.control.v1.RegisterNodeRequest.LabelsEntry + nil, // 87: persys.control.v1.WorkloadSpec.MetadataEntry + nil, // 88: persys.control.v1.ContainerSpec.EnvEntry + nil, // 89: persys.control.v1.ComposeSpec.EnvEntry + nil, // 90: persys.control.v1.NodeView.LabelsEntry + nil, // 91: persys.control.v1.SchedulerEventView.DetailsEntry + (*timestamppb.Timestamp)(nil), // 92: google.protobuf.Timestamp } var file_control_proto_depIdxs = []int32{ - 0, // 0: persys.control.v1.AutomationSuggestion.action_type:type_name -> persys.control.v1.AutomationActionType - 63, // 1: persys.control.v1.AutomationSuggestion.suggested_at:type_name -> google.protobuf.Timestamp - 2, // 2: persys.control.v1.SubmitAutomationSuggestionRequest.suggestion:type_name -> persys.control.v1.AutomationSuggestion - 63, // 3: persys.control.v1.SubmitAutomationSuggestionResponse.decided_at:type_name -> google.protobuf.Timestamp - 6, // 4: persys.control.v1.RegisterNodeRequest.capabilities:type_name -> persys.control.v1.NodeCapabilities - 58, // 5: persys.control.v1.RegisterNodeRequest.labels:type_name -> persys.control.v1.RegisterNodeRequest.LabelsEntry - 63, // 6: persys.control.v1.RegisterNodeRequest.timestamp:type_name -> google.protobuf.Timestamp - 7, // 7: persys.control.v1.NodeCapabilities.storage_pools:type_name -> persys.control.v1.StoragePool - 63, // 8: persys.control.v1.RegisterNodeResponse.lease_expires_at:type_name -> google.protobuf.Timestamp - 10, // 9: persys.control.v1.HeartbeatRequest.usage:type_name -> persys.control.v1.NodeUsage - 29, // 10: persys.control.v1.HeartbeatRequest.workload_statuses:type_name -> persys.control.v1.WorkloadStatus - 63, // 11: persys.control.v1.HeartbeatRequest.timestamp:type_name -> google.protobuf.Timestamp - 27, // 12: persys.control.v1.HeartbeatRequest.workload_usage:type_name -> persys.control.v1.WorkloadUsageSnapshot - 63, // 13: persys.control.v1.HeartbeatResponse.lease_expires_at:type_name -> google.protobuf.Timestamp - 16, // 14: persys.control.v1.ApplyWorkloadRequest.spec:type_name -> persys.control.v1.WorkloadSpec - 1, // 15: persys.control.v1.ApplyWorkloadResponse.failure_reason:type_name -> persys.control.v1.FailureReason - 17, // 16: persys.control.v1.WorkloadSpec.resources:type_name -> persys.control.v1.ResourceRequirements - 18, // 17: persys.control.v1.WorkloadSpec.container:type_name -> persys.control.v1.ContainerSpec - 21, // 18: persys.control.v1.WorkloadSpec.compose:type_name -> persys.control.v1.ComposeSpec - 22, // 19: persys.control.v1.WorkloadSpec.vm:type_name -> persys.control.v1.VMSpec - 59, // 20: persys.control.v1.WorkloadSpec.metadata:type_name -> persys.control.v1.WorkloadSpec.MetadataEntry - 60, // 21: persys.control.v1.ContainerSpec.env:type_name -> persys.control.v1.ContainerSpec.EnvEntry - 19, // 22: persys.control.v1.ContainerSpec.volumes:type_name -> persys.control.v1.VolumeMount - 20, // 23: persys.control.v1.ContainerSpec.ports:type_name -> persys.control.v1.Port - 26, // 24: persys.control.v1.ContainerSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec - 61, // 25: persys.control.v1.ComposeSpec.env:type_name -> persys.control.v1.ComposeSpec.EnvEntry - 23, // 26: persys.control.v1.VMSpec.disks:type_name -> persys.control.v1.DiskConfig - 24, // 27: persys.control.v1.VMSpec.networks:type_name -> persys.control.v1.NetworkConfig - 25, // 28: persys.control.v1.VMSpec.cloud_init:type_name -> persys.control.v1.CloudInitConfig - 26, // 29: persys.control.v1.VMSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec - 63, // 30: persys.control.v1.WorkloadUsageSnapshot.collected_at:type_name -> google.protobuf.Timestamp - 63, // 31: persys.control.v1.ReasonDetail.last_transition:type_name -> google.protobuf.Timestamp - 63, // 32: persys.control.v1.ReasonDetail.next_retry_at:type_name -> google.protobuf.Timestamp - 1, // 33: persys.control.v1.WorkloadStatus.failure_reason:type_name -> persys.control.v1.FailureReason - 63, // 34: persys.control.v1.WorkloadStatus.last_transition:type_name -> google.protobuf.Timestamp - 28, // 35: persys.control.v1.WorkloadStatus.reason:type_name -> persys.control.v1.ReasonDetail - 27, // 36: persys.control.v1.WorkloadStatus.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot - 49, // 37: persys.control.v1.DrainNodeResponse.node:type_name -> persys.control.v1.NodeView - 49, // 38: persys.control.v1.UndrainNodeResponse.node:type_name -> persys.control.v1.NodeView - 36, // 39: persys.control.v1.TaintNodeRequest.taint:type_name -> persys.control.v1.NodeTaint - 49, // 40: persys.control.v1.TaintNodeResponse.node:type_name -> persys.control.v1.NodeView - 49, // 41: persys.control.v1.UntaintNodeResponse.node:type_name -> persys.control.v1.NodeView - 49, // 42: persys.control.v1.SetNodeLabelResponse.node:type_name -> persys.control.v1.NodeView - 49, // 43: persys.control.v1.DeleteNodeLabelResponse.node:type_name -> persys.control.v1.NodeView - 49, // 44: persys.control.v1.ListNodesResponse.nodes:type_name -> persys.control.v1.NodeView - 49, // 45: persys.control.v1.GetNodeResponse.node:type_name -> persys.control.v1.NodeView - 63, // 46: persys.control.v1.NodeView.status_updated_at:type_name -> google.protobuf.Timestamp - 63, // 47: persys.control.v1.NodeView.last_heartbeat:type_name -> google.protobuf.Timestamp - 62, // 48: persys.control.v1.NodeView.labels:type_name -> persys.control.v1.NodeView.LabelsEntry - 36, // 49: persys.control.v1.NodeView.taints:type_name -> persys.control.v1.NodeTaint - 54, // 50: persys.control.v1.ListWorkloadsResponse.workloads:type_name -> persys.control.v1.WorkloadView - 54, // 51: persys.control.v1.GetWorkloadResponse.workload:type_name -> persys.control.v1.WorkloadView - 63, // 52: persys.control.v1.WorkloadView.retry_next_at:type_name -> google.protobuf.Timestamp - 63, // 53: persys.control.v1.WorkloadView.last_updated:type_name -> google.protobuf.Timestamp - 28, // 54: persys.control.v1.WorkloadView.reason:type_name -> persys.control.v1.ReasonDetail - 27, // 55: persys.control.v1.WorkloadView.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot - 63, // 56: persys.control.v1.WorkloadView.created_at:type_name -> google.protobuf.Timestamp - 63, // 57: persys.control.v1.GetClusterSummaryResponse.generated_at:type_name -> google.protobuf.Timestamp - 5, // 58: persys.control.v1.ControlMessage.register:type_name -> persys.control.v1.RegisterNodeRequest - 9, // 59: persys.control.v1.ControlMessage.heartbeat:type_name -> persys.control.v1.HeartbeatRequest - 12, // 60: persys.control.v1.ControlMessage.apply:type_name -> persys.control.v1.ApplyWorkloadRequest - 14, // 61: persys.control.v1.ControlMessage.delete:type_name -> persys.control.v1.DeleteWorkloadRequest - 5, // 62: persys.control.v1.AgentControl.RegisterNode:input_type -> persys.control.v1.RegisterNodeRequest - 9, // 63: persys.control.v1.AgentControl.Heartbeat:input_type -> persys.control.v1.HeartbeatRequest - 12, // 64: persys.control.v1.AgentControl.ApplyWorkload:input_type -> persys.control.v1.ApplyWorkloadRequest - 14, // 65: persys.control.v1.AgentControl.DeleteWorkload:input_type -> persys.control.v1.DeleteWorkloadRequest - 30, // 66: persys.control.v1.AgentControl.RetryWorkload:input_type -> persys.control.v1.RetryWorkloadRequest - 32, // 67: persys.control.v1.AgentControl.DrainNode:input_type -> persys.control.v1.DrainNodeRequest - 34, // 68: persys.control.v1.AgentControl.UndrainNode:input_type -> persys.control.v1.UndrainNodeRequest - 37, // 69: persys.control.v1.AgentControl.TaintNode:input_type -> persys.control.v1.TaintNodeRequest - 39, // 70: persys.control.v1.AgentControl.UntaintNode:input_type -> persys.control.v1.UntaintNodeRequest - 41, // 71: persys.control.v1.AgentControl.SetNodeLabel:input_type -> persys.control.v1.SetNodeLabelRequest - 43, // 72: persys.control.v1.AgentControl.DeleteNodeLabel:input_type -> persys.control.v1.DeleteNodeLabelRequest - 3, // 73: persys.control.v1.AgentControl.SubmitAutomationSuggestion:input_type -> persys.control.v1.SubmitAutomationSuggestionRequest - 45, // 74: persys.control.v1.AgentControl.ListNodes:input_type -> persys.control.v1.ListNodesRequest - 46, // 75: persys.control.v1.AgentControl.GetNode:input_type -> persys.control.v1.GetNodeRequest - 50, // 76: persys.control.v1.AgentControl.ListWorkloads:input_type -> persys.control.v1.ListWorkloadsRequest - 51, // 77: persys.control.v1.AgentControl.GetWorkload:input_type -> persys.control.v1.GetWorkloadRequest - 55, // 78: persys.control.v1.AgentControl.GetClusterSummary:input_type -> persys.control.v1.GetClusterSummaryRequest - 57, // 79: persys.control.v1.AgentControl.ControlStream:input_type -> persys.control.v1.ControlMessage - 8, // 80: persys.control.v1.AgentControl.RegisterNode:output_type -> persys.control.v1.RegisterNodeResponse - 11, // 81: persys.control.v1.AgentControl.Heartbeat:output_type -> persys.control.v1.HeartbeatResponse - 13, // 82: persys.control.v1.AgentControl.ApplyWorkload:output_type -> persys.control.v1.ApplyWorkloadResponse - 15, // 83: persys.control.v1.AgentControl.DeleteWorkload:output_type -> persys.control.v1.DeleteWorkloadResponse - 31, // 84: persys.control.v1.AgentControl.RetryWorkload:output_type -> persys.control.v1.RetryWorkloadResponse - 33, // 85: persys.control.v1.AgentControl.DrainNode:output_type -> persys.control.v1.DrainNodeResponse - 35, // 86: persys.control.v1.AgentControl.UndrainNode:output_type -> persys.control.v1.UndrainNodeResponse - 38, // 87: persys.control.v1.AgentControl.TaintNode:output_type -> persys.control.v1.TaintNodeResponse - 40, // 88: persys.control.v1.AgentControl.UntaintNode:output_type -> persys.control.v1.UntaintNodeResponse - 42, // 89: persys.control.v1.AgentControl.SetNodeLabel:output_type -> persys.control.v1.SetNodeLabelResponse - 44, // 90: persys.control.v1.AgentControl.DeleteNodeLabel:output_type -> persys.control.v1.DeleteNodeLabelResponse - 4, // 91: persys.control.v1.AgentControl.SubmitAutomationSuggestion:output_type -> persys.control.v1.SubmitAutomationSuggestionResponse - 47, // 92: persys.control.v1.AgentControl.ListNodes:output_type -> persys.control.v1.ListNodesResponse - 48, // 93: persys.control.v1.AgentControl.GetNode:output_type -> persys.control.v1.GetNodeResponse - 52, // 94: persys.control.v1.AgentControl.ListWorkloads:output_type -> persys.control.v1.ListWorkloadsResponse - 53, // 95: persys.control.v1.AgentControl.GetWorkload:output_type -> persys.control.v1.GetWorkloadResponse - 56, // 96: persys.control.v1.AgentControl.GetClusterSummary:output_type -> persys.control.v1.GetClusterSummaryResponse - 57, // 97: persys.control.v1.AgentControl.ControlStream:output_type -> persys.control.v1.ControlMessage - 80, // [80:98] is the sub-list for method output_type - 62, // [62:80] is the sub-list for method input_type - 62, // [62:62] is the sub-list for extension type_name - 62, // [62:62] is the sub-list for extension extendee - 0, // [0:62] is the sub-list for field type_name + 0, // 0: persys.control.v1.AutomationSuggestion.action_type:type_name -> persys.control.v1.AutomationActionType + 92, // 1: persys.control.v1.AutomationSuggestion.suggested_at:type_name -> google.protobuf.Timestamp + 2, // 2: persys.control.v1.SubmitAutomationSuggestionRequest.suggestion:type_name -> persys.control.v1.AutomationSuggestion + 92, // 3: persys.control.v1.SubmitAutomationSuggestionResponse.decided_at:type_name -> google.protobuf.Timestamp + 6, // 4: persys.control.v1.RegisterNodeRequest.capabilities:type_name -> persys.control.v1.NodeCapabilities + 86, // 5: persys.control.v1.RegisterNodeRequest.labels:type_name -> persys.control.v1.RegisterNodeRequest.LabelsEntry + 92, // 6: persys.control.v1.RegisterNodeRequest.timestamp:type_name -> google.protobuf.Timestamp + 7, // 7: persys.control.v1.NodeCapabilities.storage_pools:type_name -> persys.control.v1.StoragePool + 92, // 8: persys.control.v1.RegisterNodeResponse.lease_expires_at:type_name -> google.protobuf.Timestamp + 10, // 9: persys.control.v1.HeartbeatRequest.usage:type_name -> persys.control.v1.NodeUsage + 29, // 10: persys.control.v1.HeartbeatRequest.workload_statuses:type_name -> persys.control.v1.WorkloadStatus + 92, // 11: persys.control.v1.HeartbeatRequest.timestamp:type_name -> google.protobuf.Timestamp + 27, // 12: persys.control.v1.HeartbeatRequest.workload_usage:type_name -> persys.control.v1.WorkloadUsageSnapshot + 92, // 13: persys.control.v1.HeartbeatResponse.lease_expires_at:type_name -> google.protobuf.Timestamp + 16, // 14: persys.control.v1.ApplyWorkloadRequest.spec:type_name -> persys.control.v1.WorkloadSpec + 1, // 15: persys.control.v1.ApplyWorkloadResponse.failure_reason:type_name -> persys.control.v1.FailureReason + 17, // 16: persys.control.v1.WorkloadSpec.resources:type_name -> persys.control.v1.ResourceRequirements + 18, // 17: persys.control.v1.WorkloadSpec.container:type_name -> persys.control.v1.ContainerSpec + 21, // 18: persys.control.v1.WorkloadSpec.compose:type_name -> persys.control.v1.ComposeSpec + 22, // 19: persys.control.v1.WorkloadSpec.vm:type_name -> persys.control.v1.VMSpec + 87, // 20: persys.control.v1.WorkloadSpec.metadata:type_name -> persys.control.v1.WorkloadSpec.MetadataEntry + 88, // 21: persys.control.v1.ContainerSpec.env:type_name -> persys.control.v1.ContainerSpec.EnvEntry + 19, // 22: persys.control.v1.ContainerSpec.volumes:type_name -> persys.control.v1.VolumeMount + 20, // 23: persys.control.v1.ContainerSpec.ports:type_name -> persys.control.v1.Port + 26, // 24: persys.control.v1.ContainerSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec + 89, // 25: persys.control.v1.ComposeSpec.env:type_name -> persys.control.v1.ComposeSpec.EnvEntry + 23, // 26: persys.control.v1.VMSpec.disks:type_name -> persys.control.v1.DiskConfig + 24, // 27: persys.control.v1.VMSpec.networks:type_name -> persys.control.v1.NetworkConfig + 25, // 28: persys.control.v1.VMSpec.cloud_init:type_name -> persys.control.v1.CloudInitConfig + 26, // 29: persys.control.v1.VMSpec.managed_volumes:type_name -> persys.control.v1.ManagedVolumeSpec + 92, // 30: persys.control.v1.WorkloadUsageSnapshot.collected_at:type_name -> google.protobuf.Timestamp + 92, // 31: persys.control.v1.ReasonDetail.last_transition:type_name -> google.protobuf.Timestamp + 92, // 32: persys.control.v1.ReasonDetail.next_retry_at:type_name -> google.protobuf.Timestamp + 1, // 33: persys.control.v1.WorkloadStatus.failure_reason:type_name -> persys.control.v1.FailureReason + 92, // 34: persys.control.v1.WorkloadStatus.last_transition:type_name -> google.protobuf.Timestamp + 28, // 35: persys.control.v1.WorkloadStatus.reason:type_name -> persys.control.v1.ReasonDetail + 27, // 36: persys.control.v1.WorkloadStatus.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot + 49, // 37: persys.control.v1.DrainNodeResponse.node:type_name -> persys.control.v1.NodeView + 49, // 38: persys.control.v1.UndrainNodeResponse.node:type_name -> persys.control.v1.NodeView + 36, // 39: persys.control.v1.TaintNodeRequest.taint:type_name -> persys.control.v1.NodeTaint + 49, // 40: persys.control.v1.TaintNodeResponse.node:type_name -> persys.control.v1.NodeView + 49, // 41: persys.control.v1.UntaintNodeResponse.node:type_name -> persys.control.v1.NodeView + 49, // 42: persys.control.v1.SetNodeLabelResponse.node:type_name -> persys.control.v1.NodeView + 49, // 43: persys.control.v1.DeleteNodeLabelResponse.node:type_name -> persys.control.v1.NodeView + 49, // 44: persys.control.v1.ListNodesResponse.nodes:type_name -> persys.control.v1.NodeView + 49, // 45: persys.control.v1.GetNodeResponse.node:type_name -> persys.control.v1.NodeView + 92, // 46: persys.control.v1.NodeView.status_updated_at:type_name -> google.protobuf.Timestamp + 92, // 47: persys.control.v1.NodeView.last_heartbeat:type_name -> google.protobuf.Timestamp + 90, // 48: persys.control.v1.NodeView.labels:type_name -> persys.control.v1.NodeView.LabelsEntry + 36, // 49: persys.control.v1.NodeView.taints:type_name -> persys.control.v1.NodeTaint + 92, // 50: persys.control.v1.SchedulerEventView.timestamp:type_name -> google.protobuf.Timestamp + 91, // 51: persys.control.v1.SchedulerEventView.details:type_name -> persys.control.v1.SchedulerEventView.DetailsEntry + 51, // 52: persys.control.v1.ListEventsResponse.events:type_name -> persys.control.v1.SchedulerEventView + 58, // 53: persys.control.v1.ListWorkloadsResponse.workloads:type_name -> persys.control.v1.WorkloadView + 58, // 54: persys.control.v1.GetWorkloadResponse.workload:type_name -> persys.control.v1.WorkloadView + 92, // 55: persys.control.v1.WorkloadView.retry_next_at:type_name -> google.protobuf.Timestamp + 92, // 56: persys.control.v1.WorkloadView.last_updated:type_name -> google.protobuf.Timestamp + 28, // 57: persys.control.v1.WorkloadView.reason:type_name -> persys.control.v1.ReasonDetail + 27, // 58: persys.control.v1.WorkloadView.usage:type_name -> persys.control.v1.WorkloadUsageSnapshot + 92, // 59: persys.control.v1.WorkloadView.created_at:type_name -> google.protobuf.Timestamp + 92, // 60: persys.control.v1.GetClusterSummaryResponse.generated_at:type_name -> google.protobuf.Timestamp + 5, // 61: persys.control.v1.ControlMessage.register:type_name -> persys.control.v1.RegisterNodeRequest + 9, // 62: persys.control.v1.ControlMessage.heartbeat:type_name -> persys.control.v1.HeartbeatRequest + 12, // 63: persys.control.v1.ControlMessage.apply:type_name -> persys.control.v1.ApplyWorkloadRequest + 14, // 64: persys.control.v1.ControlMessage.delete:type_name -> persys.control.v1.DeleteWorkloadRequest + 70, // 65: persys.control.v1.CreateDiskResponse.disk:type_name -> persys.control.v1.DiskView + 70, // 66: persys.control.v1.ListDisksResponse.disks:type_name -> persys.control.v1.DiskView + 70, // 67: persys.control.v1.GetDiskResponse.disk:type_name -> persys.control.v1.DiskView + 92, // 68: persys.control.v1.DiskView.created_at:type_name -> google.protobuf.Timestamp + 92, // 69: persys.control.v1.DiskView.updated_at:type_name -> google.protobuf.Timestamp + 83, // 70: persys.control.v1.CreateBucketResponse.bucket:type_name -> persys.control.v1.BucketView + 84, // 71: persys.control.v1.CreateBucketResponse.access:type_name -> persys.control.v1.BucketAccess + 83, // 72: persys.control.v1.ListBucketsResponse.buckets:type_name -> persys.control.v1.BucketView + 83, // 73: persys.control.v1.GetBucketResponse.bucket:type_name -> persys.control.v1.BucketView + 84, // 74: persys.control.v1.GetBucketAccessResponse.access:type_name -> persys.control.v1.BucketAccess + 85, // 75: persys.control.v1.ListBucketObjectsResponse.objects:type_name -> persys.control.v1.ObjectInfo + 92, // 76: persys.control.v1.BucketView.created_at:type_name -> google.protobuf.Timestamp + 92, // 77: persys.control.v1.BucketView.updated_at:type_name -> google.protobuf.Timestamp + 5, // 78: persys.control.v1.AgentControl.RegisterNode:input_type -> persys.control.v1.RegisterNodeRequest + 9, // 79: persys.control.v1.AgentControl.Heartbeat:input_type -> persys.control.v1.HeartbeatRequest + 12, // 80: persys.control.v1.AgentControl.ApplyWorkload:input_type -> persys.control.v1.ApplyWorkloadRequest + 14, // 81: persys.control.v1.AgentControl.DeleteWorkload:input_type -> persys.control.v1.DeleteWorkloadRequest + 30, // 82: persys.control.v1.AgentControl.RetryWorkload:input_type -> persys.control.v1.RetryWorkloadRequest + 32, // 83: persys.control.v1.AgentControl.DrainNode:input_type -> persys.control.v1.DrainNodeRequest + 34, // 84: persys.control.v1.AgentControl.UndrainNode:input_type -> persys.control.v1.UndrainNodeRequest + 37, // 85: persys.control.v1.AgentControl.TaintNode:input_type -> persys.control.v1.TaintNodeRequest + 39, // 86: persys.control.v1.AgentControl.UntaintNode:input_type -> persys.control.v1.UntaintNodeRequest + 41, // 87: persys.control.v1.AgentControl.SetNodeLabel:input_type -> persys.control.v1.SetNodeLabelRequest + 43, // 88: persys.control.v1.AgentControl.DeleteNodeLabel:input_type -> persys.control.v1.DeleteNodeLabelRequest + 3, // 89: persys.control.v1.AgentControl.SubmitAutomationSuggestion:input_type -> persys.control.v1.SubmitAutomationSuggestionRequest + 45, // 90: persys.control.v1.AgentControl.ListNodes:input_type -> persys.control.v1.ListNodesRequest + 46, // 91: persys.control.v1.AgentControl.GetNode:input_type -> persys.control.v1.GetNodeRequest + 50, // 92: persys.control.v1.AgentControl.ListWorkloads:input_type -> persys.control.v1.ListWorkloadsRequest + 55, // 93: persys.control.v1.AgentControl.GetWorkload:input_type -> persys.control.v1.GetWorkloadRequest + 59, // 94: persys.control.v1.AgentControl.GetClusterSummary:input_type -> persys.control.v1.GetClusterSummaryRequest + 52, // 95: persys.control.v1.AgentControl.ListEvents:input_type -> persys.control.v1.ListEventsRequest + 54, // 96: persys.control.v1.AgentControl.WatchEvents:input_type -> persys.control.v1.WatchEventsRequest + 61, // 97: persys.control.v1.AgentControl.ControlStream:input_type -> persys.control.v1.ControlMessage + 62, // 98: persys.control.v1.AgentControl.CreateDisk:input_type -> persys.control.v1.CreateDiskRequest + 64, // 99: persys.control.v1.AgentControl.ListDisks:input_type -> persys.control.v1.ListDisksRequest + 66, // 100: persys.control.v1.AgentControl.GetDisk:input_type -> persys.control.v1.GetDiskRequest + 68, // 101: persys.control.v1.AgentControl.DeleteDisk:input_type -> persys.control.v1.DeleteDiskRequest + 71, // 102: persys.control.v1.AgentControl.CreateBucket:input_type -> persys.control.v1.CreateBucketRequest + 73, // 103: persys.control.v1.AgentControl.ListBuckets:input_type -> persys.control.v1.ListBucketsRequest + 75, // 104: persys.control.v1.AgentControl.GetBucket:input_type -> persys.control.v1.GetBucketRequest + 77, // 105: persys.control.v1.AgentControl.DeleteBucket:input_type -> persys.control.v1.DeleteBucketRequest + 79, // 106: persys.control.v1.AgentControl.GetBucketAccess:input_type -> persys.control.v1.GetBucketAccessRequest + 81, // 107: persys.control.v1.AgentControl.ListBucketObjects:input_type -> persys.control.v1.ListBucketObjectsRequest + 8, // 108: persys.control.v1.AgentControl.RegisterNode:output_type -> persys.control.v1.RegisterNodeResponse + 11, // 109: persys.control.v1.AgentControl.Heartbeat:output_type -> persys.control.v1.HeartbeatResponse + 13, // 110: persys.control.v1.AgentControl.ApplyWorkload:output_type -> persys.control.v1.ApplyWorkloadResponse + 15, // 111: persys.control.v1.AgentControl.DeleteWorkload:output_type -> persys.control.v1.DeleteWorkloadResponse + 31, // 112: persys.control.v1.AgentControl.RetryWorkload:output_type -> persys.control.v1.RetryWorkloadResponse + 33, // 113: persys.control.v1.AgentControl.DrainNode:output_type -> persys.control.v1.DrainNodeResponse + 35, // 114: persys.control.v1.AgentControl.UndrainNode:output_type -> persys.control.v1.UndrainNodeResponse + 38, // 115: persys.control.v1.AgentControl.TaintNode:output_type -> persys.control.v1.TaintNodeResponse + 40, // 116: persys.control.v1.AgentControl.UntaintNode:output_type -> persys.control.v1.UntaintNodeResponse + 42, // 117: persys.control.v1.AgentControl.SetNodeLabel:output_type -> persys.control.v1.SetNodeLabelResponse + 44, // 118: persys.control.v1.AgentControl.DeleteNodeLabel:output_type -> persys.control.v1.DeleteNodeLabelResponse + 4, // 119: persys.control.v1.AgentControl.SubmitAutomationSuggestion:output_type -> persys.control.v1.SubmitAutomationSuggestionResponse + 47, // 120: persys.control.v1.AgentControl.ListNodes:output_type -> persys.control.v1.ListNodesResponse + 48, // 121: persys.control.v1.AgentControl.GetNode:output_type -> persys.control.v1.GetNodeResponse + 56, // 122: persys.control.v1.AgentControl.ListWorkloads:output_type -> persys.control.v1.ListWorkloadsResponse + 57, // 123: persys.control.v1.AgentControl.GetWorkload:output_type -> persys.control.v1.GetWorkloadResponse + 60, // 124: persys.control.v1.AgentControl.GetClusterSummary:output_type -> persys.control.v1.GetClusterSummaryResponse + 53, // 125: persys.control.v1.AgentControl.ListEvents:output_type -> persys.control.v1.ListEventsResponse + 51, // 126: persys.control.v1.AgentControl.WatchEvents:output_type -> persys.control.v1.SchedulerEventView + 61, // 127: persys.control.v1.AgentControl.ControlStream:output_type -> persys.control.v1.ControlMessage + 63, // 128: persys.control.v1.AgentControl.CreateDisk:output_type -> persys.control.v1.CreateDiskResponse + 65, // 129: persys.control.v1.AgentControl.ListDisks:output_type -> persys.control.v1.ListDisksResponse + 67, // 130: persys.control.v1.AgentControl.GetDisk:output_type -> persys.control.v1.GetDiskResponse + 69, // 131: persys.control.v1.AgentControl.DeleteDisk:output_type -> persys.control.v1.DeleteDiskResponse + 72, // 132: persys.control.v1.AgentControl.CreateBucket:output_type -> persys.control.v1.CreateBucketResponse + 74, // 133: persys.control.v1.AgentControl.ListBuckets:output_type -> persys.control.v1.ListBucketsResponse + 76, // 134: persys.control.v1.AgentControl.GetBucket:output_type -> persys.control.v1.GetBucketResponse + 78, // 135: persys.control.v1.AgentControl.DeleteBucket:output_type -> persys.control.v1.DeleteBucketResponse + 80, // 136: persys.control.v1.AgentControl.GetBucketAccess:output_type -> persys.control.v1.GetBucketAccessResponse + 82, // 137: persys.control.v1.AgentControl.ListBucketObjects:output_type -> persys.control.v1.ListBucketObjectsResponse + 108, // [108:138] is the sub-list for method output_type + 78, // [78:108] is the sub-list for method input_type + 78, // [78:78] is the sub-list for extension type_name + 78, // [78:78] is the sub-list for extension extendee + 0, // [0:78] is the sub-list for field type_name } func init() { file_control_proto_init() } @@ -4593,7 +6599,7 @@ func file_control_proto_init() { (*WorkloadSpec_Compose)(nil), (*WorkloadSpec_Vm)(nil), } - file_control_proto_msgTypes[55].OneofWrappers = []any{ + file_control_proto_msgTypes[59].OneofWrappers = []any{ (*ControlMessage_Register)(nil), (*ControlMessage_Heartbeat)(nil), (*ControlMessage_Apply)(nil), @@ -4605,7 +6611,7 @@ func file_control_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_control_proto_rawDesc), len(file_control_proto_rawDesc)), NumEnums: 2, - NumMessages: 61, + NumMessages: 90, NumExtensions: 0, NumServices: 1, }, diff --git a/persys-scheduler/internal/controlv1/control_grpc.pb.go b/persys-scheduler/internal/controlv1/control_grpc.pb.go index 4a70e36..1ccea93 100644 --- a/persys-scheduler/internal/controlv1/control_grpc.pb.go +++ b/persys-scheduler/internal/controlv1/control_grpc.pb.go @@ -36,7 +36,19 @@ const ( AgentControl_ListWorkloads_FullMethodName = "/persys.control.v1.AgentControl/ListWorkloads" AgentControl_GetWorkload_FullMethodName = "/persys.control.v1.AgentControl/GetWorkload" AgentControl_GetClusterSummary_FullMethodName = "/persys.control.v1.AgentControl/GetClusterSummary" + AgentControl_ListEvents_FullMethodName = "/persys.control.v1.AgentControl/ListEvents" + AgentControl_WatchEvents_FullMethodName = "/persys.control.v1.AgentControl/WatchEvents" AgentControl_ControlStream_FullMethodName = "/persys.control.v1.AgentControl/ControlStream" + AgentControl_CreateDisk_FullMethodName = "/persys.control.v1.AgentControl/CreateDisk" + AgentControl_ListDisks_FullMethodName = "/persys.control.v1.AgentControl/ListDisks" + AgentControl_GetDisk_FullMethodName = "/persys.control.v1.AgentControl/GetDisk" + AgentControl_DeleteDisk_FullMethodName = "/persys.control.v1.AgentControl/DeleteDisk" + AgentControl_CreateBucket_FullMethodName = "/persys.control.v1.AgentControl/CreateBucket" + AgentControl_ListBuckets_FullMethodName = "/persys.control.v1.AgentControl/ListBuckets" + AgentControl_GetBucket_FullMethodName = "/persys.control.v1.AgentControl/GetBucket" + AgentControl_DeleteBucket_FullMethodName = "/persys.control.v1.AgentControl/DeleteBucket" + AgentControl_GetBucketAccess_FullMethodName = "/persys.control.v1.AgentControl/GetBucketAccess" + AgentControl_ListBucketObjects_FullMethodName = "/persys.control.v1.AgentControl/ListBucketObjects" ) // AgentControlClient is the client API for AgentControl service. @@ -66,8 +78,32 @@ type AgentControlClient interface { ListWorkloads(ctx context.Context, in *ListWorkloadsRequest, opts ...grpc.CallOption) (*ListWorkloadsResponse, error) GetWorkload(ctx context.Context, in *GetWorkloadRequest, opts ...grpc.CallOption) (*GetWorkloadResponse, error) GetClusterSummary(ctx context.Context, in *GetClusterSummaryRequest, opts ...grpc.CallOption) (*GetClusterSummaryResponse, error) + // Cluster-wide events: node joined, node lost, node left, workload + // scheduled, drift detected, retries, reschedules, etc (see + // internal/scheduler/events.go for producers). ListEvents is a + // plain unary call (auto-bridged to REST by persys-gateway's + // reflection-based grpcbridge, no gateway changes needed). WatchEvents + // is a server-streaming call — grpcbridge explicitly does not bridge + // streaming RPCs, so consumers that need HTTP (e.g. a browser + // dashboard) go through a hand-written SSE endpoint on the gateway + // instead of the generic bridge; a gRPC client (e.g. persysctl) can + // call it directly. + ListEvents(ctx context.Context, in *ListEventsRequest, opts ...grpc.CallOption) (*ListEventsResponse, error) + WatchEvents(ctx context.Context, in *WatchEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SchedulerEventView], error) // Optional future streaming channel ControlStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ControlMessage, ControlMessage], error) + // Standalone disk inventory (managed volumes) + CreateDisk(ctx context.Context, in *CreateDiskRequest, opts ...grpc.CallOption) (*CreateDiskResponse, error) + ListDisks(ctx context.Context, in *ListDisksRequest, opts ...grpc.CallOption) (*ListDisksResponse, error) + GetDisk(ctx context.Context, in *GetDiskRequest, opts ...grpc.CallOption) (*GetDiskResponse, error) + DeleteDisk(ctx context.Context, in *DeleteDiskRequest, opts ...grpc.CallOption) (*DeleteDiskResponse, error) + // Object storage (Ceph RGW / S3-compatible buckets) + CreateBucket(ctx context.Context, in *CreateBucketRequest, opts ...grpc.CallOption) (*CreateBucketResponse, error) + ListBuckets(ctx context.Context, in *ListBucketsRequest, opts ...grpc.CallOption) (*ListBucketsResponse, error) + GetBucket(ctx context.Context, in *GetBucketRequest, opts ...grpc.CallOption) (*GetBucketResponse, error) + DeleteBucket(ctx context.Context, in *DeleteBucketRequest, opts ...grpc.CallOption) (*DeleteBucketResponse, error) + GetBucketAccess(ctx context.Context, in *GetBucketAccessRequest, opts ...grpc.CallOption) (*GetBucketAccessResponse, error) + ListBucketObjects(ctx context.Context, in *ListBucketObjectsRequest, opts ...grpc.CallOption) (*ListBucketObjectsResponse, error) } type agentControlClient struct { @@ -248,9 +284,38 @@ func (c *agentControlClient) GetClusterSummary(ctx context.Context, in *GetClust return out, nil } +func (c *agentControlClient) ListEvents(ctx context.Context, in *ListEventsRequest, opts ...grpc.CallOption) (*ListEventsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListEventsResponse) + err := c.cc.Invoke(ctx, AgentControl_ListEvents_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) WatchEvents(ctx context.Context, in *WatchEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SchedulerEventView], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &AgentControl_ServiceDesc.Streams[0], AgentControl_WatchEvents_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[WatchEventsRequest, SchedulerEventView]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type AgentControl_WatchEventsClient = grpc.ServerStreamingClient[SchedulerEventView] + func (c *agentControlClient) ControlStream(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ControlMessage, ControlMessage], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &AgentControl_ServiceDesc.Streams[0], AgentControl_ControlStream_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &AgentControl_ServiceDesc.Streams[1], AgentControl_ControlStream_FullMethodName, cOpts...) if err != nil { return nil, err } @@ -261,6 +326,106 @@ func (c *agentControlClient) ControlStream(ctx context.Context, opts ...grpc.Cal // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type AgentControl_ControlStreamClient = grpc.BidiStreamingClient[ControlMessage, ControlMessage] +func (c *agentControlClient) CreateDisk(ctx context.Context, in *CreateDiskRequest, opts ...grpc.CallOption) (*CreateDiskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateDiskResponse) + err := c.cc.Invoke(ctx, AgentControl_CreateDisk_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) ListDisks(ctx context.Context, in *ListDisksRequest, opts ...grpc.CallOption) (*ListDisksResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListDisksResponse) + err := c.cc.Invoke(ctx, AgentControl_ListDisks_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) GetDisk(ctx context.Context, in *GetDiskRequest, opts ...grpc.CallOption) (*GetDiskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetDiskResponse) + err := c.cc.Invoke(ctx, AgentControl_GetDisk_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) DeleteDisk(ctx context.Context, in *DeleteDiskRequest, opts ...grpc.CallOption) (*DeleteDiskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeleteDiskResponse) + err := c.cc.Invoke(ctx, AgentControl_DeleteDisk_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) CreateBucket(ctx context.Context, in *CreateBucketRequest, opts ...grpc.CallOption) (*CreateBucketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CreateBucketResponse) + err := c.cc.Invoke(ctx, AgentControl_CreateBucket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) ListBuckets(ctx context.Context, in *ListBucketsRequest, opts ...grpc.CallOption) (*ListBucketsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBucketsResponse) + err := c.cc.Invoke(ctx, AgentControl_ListBuckets_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) GetBucket(ctx context.Context, in *GetBucketRequest, opts ...grpc.CallOption) (*GetBucketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBucketResponse) + err := c.cc.Invoke(ctx, AgentControl_GetBucket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) DeleteBucket(ctx context.Context, in *DeleteBucketRequest, opts ...grpc.CallOption) (*DeleteBucketResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(DeleteBucketResponse) + err := c.cc.Invoke(ctx, AgentControl_DeleteBucket_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) GetBucketAccess(ctx context.Context, in *GetBucketAccessRequest, opts ...grpc.CallOption) (*GetBucketAccessResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBucketAccessResponse) + err := c.cc.Invoke(ctx, AgentControl_GetBucketAccess_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *agentControlClient) ListBucketObjects(ctx context.Context, in *ListBucketObjectsRequest, opts ...grpc.CallOption) (*ListBucketObjectsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListBucketObjectsResponse) + err := c.cc.Invoke(ctx, AgentControl_ListBucketObjects_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // AgentControlServer is the server API for AgentControl service. // All implementations must embed UnimplementedAgentControlServer // for forward compatibility. @@ -288,8 +453,32 @@ type AgentControlServer interface { ListWorkloads(context.Context, *ListWorkloadsRequest) (*ListWorkloadsResponse, error) GetWorkload(context.Context, *GetWorkloadRequest) (*GetWorkloadResponse, error) GetClusterSummary(context.Context, *GetClusterSummaryRequest) (*GetClusterSummaryResponse, error) + // Cluster-wide events: node joined, node lost, node left, workload + // scheduled, drift detected, retries, reschedules, etc (see + // internal/scheduler/events.go for producers). ListEvents is a + // plain unary call (auto-bridged to REST by persys-gateway's + // reflection-based grpcbridge, no gateway changes needed). WatchEvents + // is a server-streaming call — grpcbridge explicitly does not bridge + // streaming RPCs, so consumers that need HTTP (e.g. a browser + // dashboard) go through a hand-written SSE endpoint on the gateway + // instead of the generic bridge; a gRPC client (e.g. persysctl) can + // call it directly. + ListEvents(context.Context, *ListEventsRequest) (*ListEventsResponse, error) + WatchEvents(*WatchEventsRequest, grpc.ServerStreamingServer[SchedulerEventView]) error // Optional future streaming channel ControlStream(grpc.BidiStreamingServer[ControlMessage, ControlMessage]) error + // Standalone disk inventory (managed volumes) + CreateDisk(context.Context, *CreateDiskRequest) (*CreateDiskResponse, error) + ListDisks(context.Context, *ListDisksRequest) (*ListDisksResponse, error) + GetDisk(context.Context, *GetDiskRequest) (*GetDiskResponse, error) + DeleteDisk(context.Context, *DeleteDiskRequest) (*DeleteDiskResponse, error) + // Object storage (Ceph RGW / S3-compatible buckets) + CreateBucket(context.Context, *CreateBucketRequest) (*CreateBucketResponse, error) + ListBuckets(context.Context, *ListBucketsRequest) (*ListBucketsResponse, error) + GetBucket(context.Context, *GetBucketRequest) (*GetBucketResponse, error) + DeleteBucket(context.Context, *DeleteBucketRequest) (*DeleteBucketResponse, error) + GetBucketAccess(context.Context, *GetBucketAccessRequest) (*GetBucketAccessResponse, error) + ListBucketObjects(context.Context, *ListBucketObjectsRequest) (*ListBucketObjectsResponse, error) mustEmbedUnimplementedAgentControlServer() } @@ -351,9 +540,45 @@ func (UnimplementedAgentControlServer) GetWorkload(context.Context, *GetWorkload func (UnimplementedAgentControlServer) GetClusterSummary(context.Context, *GetClusterSummaryRequest) (*GetClusterSummaryResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetClusterSummary not implemented") } +func (UnimplementedAgentControlServer) ListEvents(context.Context, *ListEventsRequest) (*ListEventsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListEvents not implemented") +} +func (UnimplementedAgentControlServer) WatchEvents(*WatchEventsRequest, grpc.ServerStreamingServer[SchedulerEventView]) error { + return status.Error(codes.Unimplemented, "method WatchEvents not implemented") +} func (UnimplementedAgentControlServer) ControlStream(grpc.BidiStreamingServer[ControlMessage, ControlMessage]) error { return status.Error(codes.Unimplemented, "method ControlStream not implemented") } +func (UnimplementedAgentControlServer) CreateDisk(context.Context, *CreateDiskRequest) (*CreateDiskResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CreateDisk not implemented") +} +func (UnimplementedAgentControlServer) ListDisks(context.Context, *ListDisksRequest) (*ListDisksResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListDisks not implemented") +} +func (UnimplementedAgentControlServer) GetDisk(context.Context, *GetDiskRequest) (*GetDiskResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetDisk not implemented") +} +func (UnimplementedAgentControlServer) DeleteDisk(context.Context, *DeleteDiskRequest) (*DeleteDiskResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteDisk not implemented") +} +func (UnimplementedAgentControlServer) CreateBucket(context.Context, *CreateBucketRequest) (*CreateBucketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CreateBucket not implemented") +} +func (UnimplementedAgentControlServer) ListBuckets(context.Context, *ListBucketsRequest) (*ListBucketsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBuckets not implemented") +} +func (UnimplementedAgentControlServer) GetBucket(context.Context, *GetBucketRequest) (*GetBucketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetBucket not implemented") +} +func (UnimplementedAgentControlServer) DeleteBucket(context.Context, *DeleteBucketRequest) (*DeleteBucketResponse, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteBucket not implemented") +} +func (UnimplementedAgentControlServer) GetBucketAccess(context.Context, *GetBucketAccessRequest) (*GetBucketAccessResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetBucketAccess not implemented") +} +func (UnimplementedAgentControlServer) ListBucketObjects(context.Context, *ListBucketObjectsRequest) (*ListBucketObjectsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListBucketObjects not implemented") +} func (UnimplementedAgentControlServer) mustEmbedUnimplementedAgentControlServer() {} func (UnimplementedAgentControlServer) testEmbeddedByValue() {} @@ -681,6 +906,35 @@ func _AgentControl_GetClusterSummary_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _AgentControl_ListEvents_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListEventsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).ListEvents(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_ListEvents_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).ListEvents(ctx, req.(*ListEventsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_WatchEvents_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(WatchEventsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(AgentControlServer).WatchEvents(m, &grpc.GenericServerStream[WatchEventsRequest, SchedulerEventView]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type AgentControl_WatchEventsServer = grpc.ServerStreamingServer[SchedulerEventView] + func _AgentControl_ControlStream_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(AgentControlServer).ControlStream(&grpc.GenericServerStream[ControlMessage, ControlMessage]{ServerStream: stream}) } @@ -688,6 +942,186 @@ func _AgentControl_ControlStream_Handler(srv interface{}, stream grpc.ServerStre // This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. type AgentControl_ControlStreamServer = grpc.BidiStreamingServer[ControlMessage, ControlMessage] +func _AgentControl_CreateDisk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateDiskRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).CreateDisk(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_CreateDisk_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).CreateDisk(ctx, req.(*CreateDiskRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_ListDisks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListDisksRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).ListDisks(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_ListDisks_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).ListDisks(ctx, req.(*ListDisksRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_GetDisk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetDiskRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).GetDisk(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_GetDisk_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).GetDisk(ctx, req.(*GetDiskRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_DeleteDisk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteDiskRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).DeleteDisk(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_DeleteDisk_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).DeleteDisk(ctx, req.(*DeleteDiskRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_CreateBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateBucketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).CreateBucket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_CreateBucket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).CreateBucket(ctx, req.(*CreateBucketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_ListBuckets_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBucketsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).ListBuckets(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_ListBuckets_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).ListBuckets(ctx, req.(*ListBucketsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_GetBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBucketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).GetBucket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_GetBucket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).GetBucket(ctx, req.(*GetBucketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_DeleteBucket_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteBucketRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).DeleteBucket(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_DeleteBucket_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).DeleteBucket(ctx, req.(*DeleteBucketRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_GetBucketAccess_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBucketAccessRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).GetBucketAccess(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_GetBucketAccess_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).GetBucketAccess(ctx, req.(*GetBucketAccessRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AgentControl_ListBucketObjects_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListBucketObjectsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AgentControlServer).ListBucketObjects(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AgentControl_ListBucketObjects_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AgentControlServer).ListBucketObjects(ctx, req.(*ListBucketObjectsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // AgentControl_ServiceDesc is the grpc.ServiceDesc for AgentControl service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -763,8 +1197,57 @@ var AgentControl_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetClusterSummary", Handler: _AgentControl_GetClusterSummary_Handler, }, + { + MethodName: "ListEvents", + Handler: _AgentControl_ListEvents_Handler, + }, + { + MethodName: "CreateDisk", + Handler: _AgentControl_CreateDisk_Handler, + }, + { + MethodName: "ListDisks", + Handler: _AgentControl_ListDisks_Handler, + }, + { + MethodName: "GetDisk", + Handler: _AgentControl_GetDisk_Handler, + }, + { + MethodName: "DeleteDisk", + Handler: _AgentControl_DeleteDisk_Handler, + }, + { + MethodName: "CreateBucket", + Handler: _AgentControl_CreateBucket_Handler, + }, + { + MethodName: "ListBuckets", + Handler: _AgentControl_ListBuckets_Handler, + }, + { + MethodName: "GetBucket", + Handler: _AgentControl_GetBucket_Handler, + }, + { + MethodName: "DeleteBucket", + Handler: _AgentControl_DeleteBucket_Handler, + }, + { + MethodName: "GetBucketAccess", + Handler: _AgentControl_GetBucketAccess_Handler, + }, + { + MethodName: "ListBucketObjects", + Handler: _AgentControl_ListBucketObjects_Handler, + }, }, Streams: []grpc.StreamDesc{ + { + StreamName: "WatchEvents", + Handler: _AgentControl_WatchEvents_Handler, + ServerStreams: true, + }, { StreamName: "ControlStream", Handler: _AgentControl_ControlStream_Handler, From 11e4640ebf78a74bcdbc6f32f85c7ff08a0ec730 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:52:16 +0330 Subject: [PATCH 25/31] Feat: Add Redis event configuration and HA mode settings to scheduler config --- persys-scheduler/internal/config/config.go | 33 ++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/persys-scheduler/internal/config/config.go b/persys-scheduler/internal/config/config.go index 24879de..bf787f4 100644 --- a/persys-scheduler/internal/config/config.go +++ b/persys-scheduler/internal/config/config.go @@ -27,10 +27,15 @@ type Config struct { SchedulerAdvertisePort int // Redis - RedisAddr string - RedisPassword string - RedisDB int - RedisReconcileTTL time.Duration + RedisAddr string + RedisPassword string + RedisDB int + RedisReconcileTTL time.Duration + // RedisEventTTL / RedisEventMaxEntries bound the shared cluster-events + // Redis Stream (see emitEvent/redis_store.go) — events are stored only + // in Redis, not etcd, so these are the sole retention knobs: TTL + // refreshed on every write, and MAXLEN ~ trimming applied at write + // time (no separate sweep needed). RedisEventTTL time.Duration RedisEventMaxEntries int64 @@ -73,6 +78,20 @@ type Config struct { // Reconciliation / drift SchedulerReconcileInterval time.Duration + SchedulerReconcileConcurrency int + + // HA mode: "failover" (default) runs one active scheduler instance + // across all replicas, with automatic takeover on crash/expiry (see + // leader.go). "active-active" partitions nodes across + // SchedulerShardCount shards by a stable hash of node ID, and this + // instance only drives reconciliation/monitoring for nodes in its own + // SchedulerShardIndex — letting multiple replicas process different + // shards concurrently, at the cost of the caveats documented in + // sharding.go. Run more than one replica per shard index for HA + // within a shard in active-active mode. + SchedulerHAMode string + SchedulerShardCount int + SchedulerShardIndex int SchedulerDriftDetectInterval time.Duration SchedulerNodeUnavailableGrace time.Duration SchedulerReapplyGuard time.Duration @@ -121,7 +140,7 @@ func Load(insecureFlag bool) (*Config, error) { TLSKeyPath: envOr("PERSYS_TLS_KEY", "/etc/persys/certs/persys_scheduler/persys_scheduler-key.key"), VaultEnabled: envBoolOr("PERSYS_VAULT_ENABLED", true), - VaultManagerAddr: envOr("PERSYS_VAULT_MANAGER_ADDR","vault-manager:50069"), + VaultManagerAddr: envOr("PERSYS_VAULT_MANAGER_ADDR", "vault-manager:50069"), VaultAddr: envOr("PERSYS_VAULT_ADDR", "http://localhost:8200"), VaultAuthMethod: strings.ToLower(envOr("PERSYS_VAULT_AUTH_METHOD", "token")), VaultToken: strings.TrimSpace(os.Getenv("PERSYS_VAULT_TOKEN")), @@ -142,6 +161,10 @@ func Load(insecureFlag bool) (*Config, error) { SchedulerAgentRPCTimeout: envDurationOrFlexibleSeconds("SCHEDULER_AGENT_RPC_TIMEOUT", 10*time.Second), SchedulerReconcileInterval: envDurationOrFlexibleSeconds("SCHEDULER_RECONCILE_INTERVAL", 5*time.Second), + SchedulerReconcileConcurrency: envIntOr("SCHEDULER_RECONCILE_CONCURRENCY", 64), + SchedulerHAMode: strings.ToLower(envOr("SCHEDULER_HA_MODE", "failover")), + SchedulerShardCount: envIntOr("SCHEDULER_SHARD_COUNT", 1), + SchedulerShardIndex: envIntOr("SCHEDULER_SHARD_INDEX", 0), SchedulerDriftDetectInterval: envDurationOrFlexibleSeconds("SCHEDULER_DRIFT_DETECT_INTERVAL", 300*time.Second), SchedulerNodeUnavailableGrace: envDurationOrFlexibleSeconds("SCHEDULER_NODE_UNAVAILABLE_GRACE", 3*time.Minute), SchedulerReapplyGuard: envDurationOrFlexibleSeconds("SCHEDULER_REAPPLY_GUARD", 45*time.Second), From beb39dc2b87f2894f213b225a8652a47bdf6d623 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:52:28 +0330 Subject: [PATCH 26/31] Feat: Remove Vault certificate manager implementation to streamline authentication process --- persys-scheduler/internal/auth/vault.go | 584 ------------------------ 1 file changed, 584 deletions(-) delete mode 100644 persys-scheduler/internal/auth/vault.go diff --git a/persys-scheduler/internal/auth/vault.go b/persys-scheduler/internal/auth/vault.go deleted file mode 100644 index 7f12677..0000000 --- a/persys-scheduler/internal/auth/vault.go +++ /dev/null @@ -1,584 +0,0 @@ -package auth - -import ( - "context" - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "net" - "net/url" - "os" - "path/filepath" - "strings" - "sync" - "time" - - vault "github.com/hashicorp/vault/api" - "github.com/sirupsen/logrus" -) - -const ( - rotationFractionNumerator = 80 - rotationFractionDenominator = 100 - minRotationWait = 30 * time.Second -) - -type Config struct { - TLSEnabled bool - ExternalIP string - TLSCertPath string - TLSKeyPath string - TLSCAPath string - - VaultEnabled bool - VaultAddr string - VaultAuthMethod string - VaultToken string - VaultAppRoleID string - VaultAppSecretID string - VaultPKIMount string - VaultPKIRole string - VaultCertTTL time.Duration - VaultServiceName string - VaultServiceDomain string - VaultRetryInterval time.Duration - - BindHost string -} - -type Manager struct { - cfg Config - logger *logrus.Entry - - mu sync.RWMutex - current certMeta -} - -type certMeta struct { - notBefore time.Time - notAfter time.Time -} - -func NewManager(cfg Config, logger *logrus.Logger) *Manager { - return &Manager{ - cfg: cfg, - logger: logger.WithField("component", "vault-cert-manager"), - } -} - -func (m *Manager) Validate() error { - if !m.cfg.TLSEnabled { - return nil - } - if !m.cfg.VaultEnabled { - return nil - } - if strings.TrimSpace(m.cfg.VaultAddr) == "" { - return fmt.Errorf("vault is enabled but PERSYS_VAULT_ADDR is empty") - } - if strings.TrimSpace(m.cfg.VaultPKIMount) == "" || strings.TrimSpace(m.cfg.VaultPKIRole) == "" { - return fmt.Errorf("vault is enabled but PKI mount/role is not configured") - } - switch strings.ToLower(strings.TrimSpace(m.cfg.VaultAuthMethod)) { - case "token": - if strings.TrimSpace(m.cfg.VaultToken) == "" { - return fmt.Errorf("vault token auth selected but PERSYS_VAULT_TOKEN is empty") - } - case "approle": - if strings.TrimSpace(m.cfg.VaultAppRoleID) == "" || strings.TrimSpace(m.cfg.VaultAppSecretID) == "" { - return fmt.Errorf("vault approle auth selected but role_id/secret_id is missing") - } - default: - return fmt.Errorf("unsupported vault auth method %q (expected token|approle)", m.cfg.VaultAuthMethod) - } - if m.cfg.VaultCertTTL <= 0 { - return fmt.Errorf("vault cert TTL must be positive") - } - if m.cfg.VaultRetryInterval <= 0 { - return fmt.Errorf("vault retry interval must be positive") - } - return nil -} - -func (m *Manager) Start(ctx context.Context) error { - if !m.cfg.TLSEnabled { - return nil - } - if !m.cfg.VaultEnabled { - m.logger.Info("Vault cert manager disabled; using manual certificate files") - return nil - } - if err := m.Validate(); err != nil { - return err - } - if existingMeta, ok := m.loadExistingCertMeta(); ok { - m.mu.Lock() - m.current = existingMeta - m.mu.Unlock() - m.logger.WithFields(logrus.Fields{ - "not_before": existingMeta.notBefore.UTC().Format(time.RFC3339), - "not_after": existingMeta.notAfter.UTC().Format(time.RFC3339), - }).Info("Using existing valid certificate from disk") - go m.rotationLoop(ctx) - return nil - } - - cli, err := m.newVaultClient() - if err != nil { - if m.manualCertAvailable() { - m.logger.WithError(err).Warn("Vault unavailable on startup, falling back to manual certificates") - go m.recoveryLoop(ctx) - return nil - } - return fmt.Errorf("vault unavailable and no manual cert fallback found: %w", err) - } - - if err := m.issueAndPersist(ctx, cli); err != nil { - if m.manualCertAvailable() { - m.logger.WithError(err).Warn("Vault certificate issuance failed, using manual certificates") - go m.recoveryLoop(ctx) - return nil - } - return fmt.Errorf("vault issuance failed and no manual cert fallback found: %w", err) - } - - go m.rotationLoop(ctx) - return nil -} - -func (m *Manager) rotationLoop(ctx context.Context) { - for { - renewAt := m.nextRenewAt() - wait := time.Until(renewAt) - if wait < minRotationWait { - wait = minRotationWait - } - - m.logger.WithField("next_rotation", renewAt.UTC().Format(time.RFC3339)).Info("Next certificate rotation scheduled") - - select { - case <-ctx.Done(): - return - case <-time.After(wait): - } - - cli, err := m.newVaultClient() - if err != nil { - m.logger.WithError(err).Warn("Vault not reachable during rotation window; retrying later") - continue - } - if err := m.issueAndPersist(ctx, cli); err != nil { - m.logger.WithError(err).Warn("Certificate rotation failed; retrying later") - continue - } - } -} - -func (m *Manager) recoveryLoop(ctx context.Context) { - ticker := time.NewTicker(m.cfg.VaultRetryInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - } - - cli, err := m.newVaultClient() - if err != nil { - m.logger.WithError(err).Debug("Vault still unavailable while running on fallback certs") - continue - } - if err := m.issueAndPersist(ctx, cli); err != nil { - m.logger.WithError(err).Warn("Vault recovered but certificate issuance still failing") - continue - } - - m.logger.Info("Vault certificate provisioning recovered; enabling rotation loop") - go m.rotationLoop(ctx) - return - } -} - -func (m *Manager) newVaultClient() (*vault.Client, error) { - conf := vault.DefaultConfig() - conf.Address = m.cfg.VaultAddr - - client, err := vault.NewClient(conf) - if err != nil { - return nil, err - } - - switch strings.ToLower(strings.TrimSpace(m.cfg.VaultAuthMethod)) { - case "token": - client.SetToken(m.cfg.VaultToken) - if _, err := client.Auth().Token().LookupSelf(); err != nil { - return nil, fmt.Errorf("token auth validation failed: %w", err) - } - case "approle": - secret, err := client.Logical().Write("auth/approle/login", map[string]interface{}{ - "role_id": m.cfg.VaultAppRoleID, - "secret_id": m.cfg.VaultAppSecretID, - }) - if err != nil { - return nil, fmt.Errorf("approle login failed: %w", err) - } - if secret == nil || secret.Auth == nil || secret.Auth.ClientToken == "" { - return nil, errors.New("approle login returned empty client token") - } - client.SetToken(secret.Auth.ClientToken) - default: - return nil, fmt.Errorf("unsupported vault auth method: %s", m.cfg.VaultAuthMethod) - } - - return client, nil -} - -func (m *Manager) issueAndPersist(ctx context.Context, client *vault.Client) error { - dnsSANs, ipSANs := m.detectSANs() - payload := map[string]interface{}{ - "common_name": m.cfg.VaultServiceName, - "ttl": m.cfg.VaultCertTTL.String(), - } - if len(dnsSANs) > 0 { - payload["alt_names"] = strings.Join(dnsSANs, ",") - } - if len(ipSANs) > 0 { - payload["ip_sans"] = strings.Join(ipSANs, ",") - } - - path := fmt.Sprintf("%s/issue/%s", strings.Trim(m.cfg.VaultPKIMount, "/"), m.cfg.VaultPKIRole) - secret, err := client.Logical().WriteWithContext(ctx, path, payload) - if err != nil { - return err - } - if secret == nil || secret.Data == nil { - return errors.New("empty response from vault issue endpoint") - } - - certPEM := asString(secret.Data["certificate"]) - keyPEM := asString(secret.Data["private_key"]) - issuingCA := asString(secret.Data["issuing_ca"]) - caChain := parseCAChain(secret.Data["ca_chain"]) - - if certPEM == "" || keyPEM == "" { - return errors.New("vault response missing certificate or private key") - } - - combinedCA := combineCA(issuingCA, caChain) - if combinedCA == "" { - return errors.New("vault response missing CA chain") - } - - notBefore, notAfter, err := certValidity(certPEM, keyPEM) - if err != nil { - return err - } - - if err := writeCertBundleAtomic(m.cfg.TLSCertPath, certPEM, m.cfg.TLSKeyPath, keyPEM, m.cfg.TLSCAPath, combinedCA); err != nil { - return err - } - - m.mu.Lock() - m.current = certMeta{ - notBefore: notBefore, - notAfter: notAfter, - } - m.mu.Unlock() - - m.logger.WithFields(logrus.Fields{ - "not_before": notBefore.UTC().Format(time.RFC3339), - "not_after": notAfter.UTC().Format(time.RFC3339), - "dns_sans": strings.Join(dnsSANs, ","), - "ip_sans": strings.Join(ipSANs, ","), - }).Info("Issued and installed certificate from Vault") - - return nil -} - -func (m *Manager) detectSANs() ([]string, []string) { - dnsSet := map[string]struct{}{} - ipSet := map[string]struct{}{} - addDNS := func(s string) { - s = strings.TrimSpace(strings.ToLower(s)) - if s != "" { - dnsSet[s] = struct{}{} - } - } - addIP := func(s string) { - s = strings.TrimSpace(s) - if ip := net.ParseIP(s); ip != nil { - ipSet[ip.String()] = struct{}{} - } - } - - service := strings.TrimSpace(m.cfg.VaultServiceName) - if service == "" { - service = "persys-scheduler" - } - addDNS(service) - addDNS("localhost") - addIP("127.0.0.1") - addIP("::1") - if m.cfg.ExternalIP != "" { - addIP(m.cfg.ExternalIP) - } - - if host, err := os.Hostname(); err == nil { - addDNS(host) - } - - domain := strings.Trim(strings.ToLower(m.cfg.VaultServiceDomain), ".") - if domain != "" { - addDNS(service + "." + domain) - if host, err := os.Hostname(); err == nil { - short := strings.Split(host, ".")[0] - addDNS(short + "." + domain) - } - } - - if bindHost := strings.TrimSpace(m.cfg.BindHost); bindHost != "" && bindHost != "0.0.0.0" { - if ip := net.ParseIP(bindHost); ip != nil { - addIP(ip.String()) - } else { - addDNS(bindHost) - } - } - - if u, err := url.Parse(m.cfg.VaultAddr); err == nil { - host := u.Hostname() - if ip := net.ParseIP(host); ip != nil { - addIP(ip.String()) - } - } - - if addrs, err := net.InterfaceAddrs(); err == nil { - for _, addr := range addrs { - ipNet, ok := addr.(*net.IPNet) - if !ok || ipNet.IP == nil || ipNet.IP.IsLoopback() { - continue - } - addIP(ipNet.IP.String()) - } - } - - dnsSANs := make([]string, 0, len(dnsSet)) - for s := range dnsSet { - dnsSANs = append(dnsSANs, s) - } - ipSANs := make([]string, 0, len(ipSet)) - for s := range ipSet { - ipSANs = append(ipSANs, s) - } - return dnsSANs, ipSANs -} - -func (m *Manager) manualCertAvailable() bool { - if _, err := tls.LoadX509KeyPair(m.cfg.TLSCertPath, m.cfg.TLSKeyPath); err != nil { - return false - } - caPEM, err := os.ReadFile(m.cfg.TLSCAPath) - if err != nil { - return false - } - pool := x509.NewCertPool() - return pool.AppendCertsFromPEM(caPEM) -} - -func (m *Manager) loadExistingCertMeta() (certMeta, bool) { - if !m.manualCertAvailable() { - return certMeta{}, false - } - keyPair, err := tls.LoadX509KeyPair(m.cfg.TLSCertPath, m.cfg.TLSKeyPath) - if err != nil || len(keyPair.Certificate) == 0 { - return certMeta{}, false - } - leaf, err := x509.ParseCertificate(keyPair.Certificate[0]) - if err != nil { - return certMeta{}, false - } - now := time.Now() - if now.Before(leaf.NotBefore) || !now.Before(leaf.NotAfter) { - return certMeta{}, false - } - if !m.certMatchesExpectedIdentity(leaf) { - return certMeta{}, false - } - return certMeta{notBefore: leaf.NotBefore, notAfter: leaf.NotAfter}, true -} - -func (m *Manager) certMatchesExpectedIdentity(leaf *x509.Certificate) bool { - expected := make([]string, 0, 3) - serviceName := strings.TrimSpace(m.cfg.VaultServiceName) - serviceDomain := strings.TrimSpace(m.cfg.VaultServiceDomain) - bindHost := strings.TrimSpace(m.cfg.BindHost) - - if bindHost != "" { - expected = append(expected, strings.ToLower(bindHost)) - } - if serviceName != "" { - expected = append(expected, strings.ToLower(serviceName)) - } - if serviceName != "" && serviceDomain != "" { - expected = append(expected, strings.ToLower(serviceName+"."+serviceDomain)) - } - if len(expected) == 0 { - return true - } - - candidates := make([]string, 0, len(leaf.DNSNames)+1) - if cn := strings.ToLower(strings.TrimSpace(leaf.Subject.CommonName)); cn != "" { - candidates = append(candidates, cn) - } - for _, dns := range leaf.DNSNames { - if s := strings.ToLower(strings.TrimSpace(dns)); s != "" { - candidates = append(candidates, s) - } - } - - for _, want := range expected { - for _, got := range candidates { - if got == want { - return true - } - } - } - return false -} - -func (m *Manager) nextRenewAt() time.Time { - m.mu.RLock() - meta := m.current - m.mu.RUnlock() - - if meta.notAfter.IsZero() || meta.notBefore.IsZero() || !meta.notAfter.After(meta.notBefore) { - return time.Now().Add(m.cfg.VaultRetryInterval) - } - - lifetime := meta.notAfter.Sub(meta.notBefore) - rotationPoint := meta.notBefore.Add(lifetime * rotationFractionNumerator / rotationFractionDenominator) - if rotationPoint.Before(time.Now()) { - return time.Now().Add(minRotationWait) - } - return rotationPoint -} - -func certValidity(certPEM, keyPEM string) (time.Time, time.Time, error) { - keyPair, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM)) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("parse issued keypair: %w", err) - } - if len(keyPair.Certificate) == 0 { - return time.Time{}, time.Time{}, errors.New("issued keypair contains no certificate") - } - leaf, err := x509.ParseCertificate(keyPair.Certificate[0]) - if err != nil { - return time.Time{}, time.Time{}, fmt.Errorf("parse issued leaf certificate: %w", err) - } - return leaf.NotBefore, leaf.NotAfter, nil -} - -func combineCA(issuingCA string, chain []string) string { - parts := make([]string, 0, 1+len(chain)) - if trimmed := strings.TrimSpace(issuingCA); trimmed != "" { - parts = append(parts, trimmed) - } - for _, c := range chain { - if trimmed := strings.TrimSpace(c); trimmed != "" { - parts = append(parts, trimmed) - } - } - return strings.Join(parts, "\n") -} - -func parseCAChain(v interface{}) []string { - switch raw := v.(type) { - case []interface{}: - out := make([]string, 0, len(raw)) - for _, item := range raw { - if s := strings.TrimSpace(asString(item)); s != "" { - out = append(out, s) - } - } - return out - case []string: - out := make([]string, 0, len(raw)) - for _, item := range raw { - if s := strings.TrimSpace(item); s != "" { - out = append(out, s) - } - } - return out - default: - s := strings.TrimSpace(asString(v)) - if s == "" { - return nil - } - return []string{s} - } -} - -func asString(v interface{}) string { - switch t := v.(type) { - case string: - return t - case []byte: - return string(t) - case nil: - return "" - default: - return fmt.Sprintf("%v", t) - } -} - -func writeAtomic(path, contents string, mode os.FileMode) error { - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - - tmp, err := os.CreateTemp(dir, ".tmp-cert-*") - if err != nil { - return err - } - tmpName := tmp.Name() - defer os.Remove(tmpName) - - if _, err := tmp.WriteString(contents); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Chmod(mode); err != nil { - _ = tmp.Close() - return err - } - if err := tmp.Close(); err != nil { - return err - } - - return os.Rename(tmpName, path) -} - -func writeCertBundleAtomic(certPath, certPEM, keyPath, keyPEM, caPath, caPEM string) error { - // Validate full bundle before writing so we don't publish an unusable pair. - if _, err := tls.X509KeyPair([]byte(certPEM), []byte(keyPEM)); err != nil { - return fmt.Errorf("invalid cert/key pair: %w", err) - } - pool := x509.NewCertPool() - if !pool.AppendCertsFromPEM([]byte(caPEM)) { - return errors.New("invalid CA PEM") - } - - if err := writeAtomic(keyPath, keyPEM, 0o600); err != nil { - return err - } - if err := writeAtomic(certPath, certPEM, 0o644); err != nil { - return err - } - if err := writeAtomic(caPath, caPEM, 0o644); err != nil { - return err - } - return nil -} From 3b3376268e17a2437bb968fb4f119e9d78352512 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:53:01 +0330 Subject: [PATCH 27/31] Chore: Update Agent gRPC implementations --- persys-scheduler/internal/agentpb/agent.pb.go | 145 +++++++++++++++--- .../internal/agentpb/agent_grpc.pb.go | 2 +- 2 files changed, 128 insertions(+), 19 deletions(-) diff --git a/persys-scheduler/internal/agentpb/agent.pb.go b/persys-scheduler/internal/agentpb/agent.pb.go index 2087cce..d9eb750 100644 --- a/persys-scheduler/internal/agentpb/agent.pb.go +++ b/persys-scheduler/internal/agentpb/agent.pb.go @@ -4,7 +4,7 @@ // protoc v3.21.12 // source: agent.proto -package agentpb +package v1 import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -28,6 +28,8 @@ const ( WorkloadType_WORKLOAD_TYPE_CONTAINER WorkloadType = 1 WorkloadType_WORKLOAD_TYPE_COMPOSE WorkloadType = 2 WorkloadType_WORKLOAD_TYPE_VM WorkloadType = 3 + // Firecracker microVM. Shares WorkloadSpec.vm oneof with KVM VMs. + WorkloadType_WORKLOAD_TYPE_MICROVM WorkloadType = 4 ) // Enum value maps for WorkloadType. @@ -37,12 +39,14 @@ var ( 1: "WORKLOAD_TYPE_CONTAINER", 2: "WORKLOAD_TYPE_COMPOSE", 3: "WORKLOAD_TYPE_VM", + 4: "WORKLOAD_TYPE_MICROVM", } WorkloadType_value = map[string]int32{ "WORKLOAD_TYPE_UNSPECIFIED": 0, "WORKLOAD_TYPE_CONTAINER": 1, "WORKLOAD_TYPE_COMPOSE": 2, "WORKLOAD_TYPE_VM": 3, + "WORKLOAD_TYPE_MICROVM": 4, } ) @@ -1221,8 +1225,15 @@ type VMSpec struct { Metadata map[string]string `protobuf:"bytes,7,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` CloudInitConfig *CloudInitConfig `protobuf:"bytes,8,opt,name=cloud_init_config,json=cloudInitConfig,proto3" json:"cloud_init_config,omitempty"` // advanced cloud-init settings ManagedVolumes []*ManagedVolumeSpec `protobuf:"bytes,9,rep,name=managed_volumes,json=managedVolumes,proto3" json:"managed_volumes,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Happy-path OS image: catalog name or absolute path to a read-only base + // image. Agent creates a writable qcow2 overlay; base is never mutated. + OsImage string `protobuf:"bytes,10,opt,name=os_image,json=osImage,proto3" json:"os_image,omitempty"` + // Root disk size in GB when synthesizing from os_image (default 10). + DiskGb int64 `protobuf:"varint,11,opt,name=disk_gb,json=diskGb,proto3" json:"disk_gb,omitempty"` + // Optional runtime selector: "libvirt" (default) or "firecracker". + Runtime string `protobuf:"bytes,12,opt,name=runtime,proto3" json:"runtime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *VMSpec) Reset() { @@ -1318,12 +1329,36 @@ func (x *VMSpec) GetManagedVolumes() []*ManagedVolumeSpec { return nil } +func (x *VMSpec) GetOsImage() string { + if x != nil { + return x.OsImage + } + return "" +} + +func (x *VMSpec) GetDiskGb() int64 { + if x != nil { + return x.DiskGb + } + return 0 +} + +func (x *VMSpec) GetRuntime() string { + if x != nil { + return x.Runtime + } + return "" +} + type CloudInitConfig struct { state protoimpl.MessageState `protogen:"open.v1"` UserData string `protobuf:"bytes,1,opt,name=user_data,json=userData,proto3" json:"user_data,omitempty"` // cloud-init user-data script MetaData string `protobuf:"bytes,2,opt,name=meta_data,json=metaData,proto3" json:"meta_data,omitempty"` // cloud-init meta-data (JSON) NetworkConfig string `protobuf:"bytes,3,opt,name=network_config,json=networkConfig,proto3" json:"network_config,omitempty"` // cloud-init network config (YAML) VendorData string `protobuf:"bytes,4,opt,name=vendor_data,json=vendorData,proto3" json:"vendor_data,omitempty"` // cloud-init vendor-data + Username string `protobuf:"bytes,5,opt,name=username,proto3" json:"username,omitempty"` // default login user when generating user-data + SshPublicKey string `protobuf:"bytes,6,opt,name=ssh_public_key,json=sshPublicKey,proto3" json:"ssh_public_key,omitempty"` // inject authorized key instead of password + Password string `protobuf:"bytes,7,opt,name=password,proto3" json:"password,omitempty"` // fixed password (otherwise random) unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1386,6 +1421,27 @@ func (x *CloudInitConfig) GetVendorData() string { return "" } +func (x *CloudInitConfig) GetUsername() string { + if x != nil { + return x.Username + } + return "" +} + +func (x *CloudInitConfig) GetSshPublicKey() string { + if x != nil { + return x.SshPublicKey + } + return "" +} + +func (x *CloudInitConfig) GetPassword() string { + if x != nil { + return x.Password + } + return "" +} + type ManagedVolumeSpec struct { state protoimpl.MessageState `protogen:"open.v1"` Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` @@ -1720,12 +1776,14 @@ func (x *RestartPolicy) GetMaxRetryCount() int32 { type DiskConfig struct { state protoimpl.MessageState `protogen:"open.v1"` - Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` // path to disk image or ISO + Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"` // path to disk image or ISO (leave empty with os_image set) Device string `protobuf:"bytes,2,opt,name=device,proto3" json:"device,omitempty"` // vda, vdb, etc. Format string `protobuf:"bytes,3,opt,name=format,proto3" json:"format,omitempty"` // qcow2, raw, iso SizeGb int64 `protobuf:"varint,4,opt,name=size_gb,json=sizeGb,proto3" json:"size_gb,omitempty"` - Type string `protobuf:"bytes,5,opt,name=type,proto3" json:"type,omitempty"` // disk or cdrom (for ISO) - Boot bool `protobuf:"varint,6,opt,name=boot,proto3" json:"boot,omitempty"` // true if this is the boot disk/ISO + Type string `protobuf:"bytes,5,opt,name=type,proto3" json:"type,omitempty"` // disk or cdrom (for ISO) + Boot bool `protobuf:"varint,6,opt,name=boot,proto3" json:"boot,omitempty"` // true if this is the boot disk/ISO + BackingFile string `protobuf:"bytes,7,opt,name=backing_file,json=backingFile,proto3" json:"backing_file,omitempty"` // optional explicit backing image for overlay + Storage string `protobuf:"bytes,8,opt,name=storage,proto3" json:"storage,omitempty"` // local|nfs|ceph-rbd hint unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1802,11 +1860,28 @@ func (x *DiskConfig) GetBoot() bool { return false } +func (x *DiskConfig) GetBackingFile() string { + if x != nil { + return x.BackingFile + } + return "" +} + +func (x *DiskConfig) GetStorage() string { + if x != nil { + return x.Storage + } + return "" +} + type NetworkConfig struct { state protoimpl.MessageState `protogen:"open.v1"` - Network string `protobuf:"bytes,1,opt,name=network,proto3" json:"network,omitempty"` // network name or bridge + Network string `protobuf:"bytes,1,opt,name=network,proto3" json:"network,omitempty"` // libvirt network name or bridge (default: "default") MacAddress string `protobuf:"bytes,2,opt,name=mac_address,json=macAddress,proto3" json:"mac_address,omitempty"` - IpAddress string `protobuf:"bytes,3,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` // optional static IP + IpAddress string `protobuf:"bytes,3,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"` // optional static guest IP + HostDevName string `protobuf:"bytes,4,opt,name=host_dev_name,json=hostDevName,proto3" json:"host_dev_name,omitempty"` // Firecracker host TAP device + Model string `protobuf:"bytes,5,opt,name=model,proto3" json:"model,omitempty"` // virtio (default) + Bridge string `protobuf:"bytes,6,opt,name=bridge,proto3" json:"bridge,omitempty"` // optional explicit bridge unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1862,6 +1937,27 @@ func (x *NetworkConfig) GetIpAddress() string { return "" } +func (x *NetworkConfig) GetHostDevName() string { + if x != nil { + return x.HostDevName + } + return "" +} + +func (x *NetworkConfig) GetModel() string { + if x != nil { + return x.Model + } + return "" +} + +func (x *NetworkConfig) GetBridge() string { + if x != nil { + return x.Bridge + } + return "" +} + type WorkloadStatus struct { state protoimpl.MessageState `protogen:"open.v1"` Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` @@ -2187,7 +2283,7 @@ const file_agent_proto_rawDesc = "" + "\x03env\x18\x03 \x03(\v2%.persys.agent.v1.ComposeSpec.EnvEntryR\x03env\x1a6\n" + "\bEnvEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xf8\x03\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xc6\x04\n" + "\x06VMSpec\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + "\x05vcpus\x18\x02 \x01(\x05R\x05vcpus\x12\x1b\n" + @@ -2198,16 +2294,23 @@ const file_agent_proto_rawDesc = "" + "cloud_init\x18\x06 \x01(\tR\tcloudInit\x12A\n" + "\bmetadata\x18\a \x03(\v2%.persys.agent.v1.VMSpec.MetadataEntryR\bmetadata\x12L\n" + "\x11cloud_init_config\x18\b \x01(\v2 .persys.agent.v1.CloudInitConfigR\x0fcloudInitConfig\x12K\n" + - "\x0fmanaged_volumes\x18\t \x03(\v2\".persys.agent.v1.ManagedVolumeSpecR\x0emanagedVolumes\x1a;\n" + + "\x0fmanaged_volumes\x18\t \x03(\v2\".persys.agent.v1.ManagedVolumeSpecR\x0emanagedVolumes\x12\x19\n" + + "\bos_image\x18\n" + + " \x01(\tR\aosImage\x12\x17\n" + + "\adisk_gb\x18\v \x01(\x03R\x06diskGb\x12\x18\n" + + "\aruntime\x18\f \x01(\tR\aruntime\x1a;\n" + "\rMetadataEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x93\x01\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xf1\x01\n" + "\x0fCloudInitConfig\x12\x1b\n" + "\tuser_data\x18\x01 \x01(\tR\buserData\x12\x1b\n" + "\tmeta_data\x18\x02 \x01(\tR\bmetaData\x12%\n" + "\x0enetwork_config\x18\x03 \x01(\tR\rnetworkConfig\x12\x1f\n" + "\vvendor_data\x18\x04 \x01(\tR\n" + - "vendorData\"\xf3\x01\n" + + "vendorData\x12\x1a\n" + + "\busername\x18\x05 \x01(\tR\busername\x12$\n" + + "\x0essh_public_key\x18\x06 \x01(\tR\fsshPublicKey\x12\x1a\n" + + "\bpassword\x18\a \x01(\tR\bpassword\"\xf3\x01\n" + "\x11ManagedVolumeSpec\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n" + "\x06driver\x18\x02 \x01(\tR\x06driver\x12\x17\n" + @@ -2234,7 +2337,7 @@ const file_agent_proto_rawDesc = "" + "\x11memory_swap_bytes\x18\x03 \x01(\x03R\x0fmemorySwapBytes\"O\n" + "\rRestartPolicy\x12\x16\n" + "\x06policy\x18\x01 \x01(\tR\x06policy\x12&\n" + - "\x0fmax_retry_count\x18\x02 \x01(\x05R\rmaxRetryCount\"\x91\x01\n" + + "\x0fmax_retry_count\x18\x02 \x01(\x05R\rmaxRetryCount\"\xce\x01\n" + "\n" + "DiskConfig\x12\x12\n" + "\x04path\x18\x01 \x01(\tR\x04path\x12\x16\n" + @@ -2242,13 +2345,18 @@ const file_agent_proto_rawDesc = "" + "\x06format\x18\x03 \x01(\tR\x06format\x12\x17\n" + "\asize_gb\x18\x04 \x01(\x03R\x06sizeGb\x12\x12\n" + "\x04type\x18\x05 \x01(\tR\x04type\x12\x12\n" + - "\x04boot\x18\x06 \x01(\bR\x04boot\"i\n" + + "\x04boot\x18\x06 \x01(\bR\x04boot\x12!\n" + + "\fbacking_file\x18\a \x01(\tR\vbackingFile\x12\x18\n" + + "\astorage\x18\b \x01(\tR\astorage\"\xbb\x01\n" + "\rNetworkConfig\x12\x18\n" + "\anetwork\x18\x01 \x01(\tR\anetwork\x12\x1f\n" + "\vmac_address\x18\x02 \x01(\tR\n" + "macAddress\x12\x1d\n" + "\n" + - "ip_address\x18\x03 \x01(\tR\tipAddress\"\x97\x04\n" + + "ip_address\x18\x03 \x01(\tR\tipAddress\x12\"\n" + + "\rhost_dev_name\x18\x04 \x01(\tR\vhostDevName\x12\x14\n" + + "\x05model\x18\x05 \x01(\tR\x05model\x12\x16\n" + + "\x06bridge\x18\x06 \x01(\tR\x06bridge\"\x97\x04\n" + "\x0eWorkloadStatus\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x121\n" + "\x04type\x18\x02 \x01(\x0e2\x1d.persys.agent.v1.WorkloadTypeR\x04type\x12\x1f\n" + @@ -2282,12 +2390,13 @@ const file_agent_proto_rawDesc = "" + "netTxBytes\x12!\n" + "\fcollected_at\x18\t \x01(\x03R\vcollectedAt\x12\x16\n" + "\x06source\x18\n" + - " \x01(\tR\x06source*{\n" + + " \x01(\tR\x06source*\x96\x01\n" + "\fWorkloadType\x12\x1d\n" + "\x19WORKLOAD_TYPE_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17WORKLOAD_TYPE_CONTAINER\x10\x01\x12\x19\n" + "\x15WORKLOAD_TYPE_COMPOSE\x10\x02\x12\x14\n" + - "\x10WORKLOAD_TYPE_VM\x10\x03*c\n" + + "\x10WORKLOAD_TYPE_VM\x10\x03\x12\x19\n" + + "\x15WORKLOAD_TYPE_MICROVM\x10\x04*c\n" + "\fDesiredState\x12\x1d\n" + "\x19DESIRED_STATE_UNSPECIFIED\x10\x00\x12\x19\n" + "\x15DESIRED_STATE_RUNNING\x10\x01\x12\x19\n" + @@ -2305,7 +2414,7 @@ const file_agent_proto_rawDesc = "" + "\x11GetWorkloadStatus\x12).persys.agent.v1.GetWorkloadStatusRequest\x1a*.persys.agent.v1.GetWorkloadStatusResponse\x12^\n" + "\rListWorkloads\x12%.persys.agent.v1.ListWorkloadsRequest\x1a&.persys.agent.v1.ListWorkloadsResponse\x12X\n" + "\vHealthCheck\x12#.persys.agent.v1.HealthCheckRequest\x1a$.persys.agent.v1.HealthCheckResponse\x12X\n" + - "\vListActions\x12#.persys.agent.v1.ListActionsRequest\x1a$.persys.agent.v1.ListActionsResponseBNZLgithub.com/persys-dev/persys-cloud/persys-scheduler/internal/agentpb;agentpbb\x06proto3" + "\vListActions\x12#.persys.agent.v1.ListActionsRequest\x1a$.persys.agent.v1.ListActionsResponseB3Z1github.com/persys-dev/compute-agent/pkg/api/v1;v1b\x06proto3" var ( file_agent_proto_rawDescOnce sync.Once diff --git a/persys-scheduler/internal/agentpb/agent_grpc.pb.go b/persys-scheduler/internal/agentpb/agent_grpc.pb.go index eff89c8..2fb05bf 100644 --- a/persys-scheduler/internal/agentpb/agent_grpc.pb.go +++ b/persys-scheduler/internal/agentpb/agent_grpc.pb.go @@ -4,7 +4,7 @@ // - protoc v3.21.12 // source: agent.proto -package agentpb +package v1 import ( context "context" From a42793138bf107d071fef50c1ef9b6b64af985ea Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:53:15 +0330 Subject: [PATCH 28/31] Feat: Add HAProxy configuration for HTTP and gRPC APIs --- persys-scheduler/haproxy/haproxy.cfg | 58 ++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 persys-scheduler/haproxy/haproxy.cfg diff --git a/persys-scheduler/haproxy/haproxy.cfg b/persys-scheduler/haproxy/haproxy.cfg new file mode 100644 index 0000000..12e40ab --- /dev/null +++ b/persys-scheduler/haproxy/haproxy.cfg @@ -0,0 +1,58 @@ +global + log stdout format raw local0 + maxconn 4096 + +defaults + log global + mode tcp + option tcplog + timeout connect 5s + timeout client 300s + timeout server 300s + retries 3 + +# Docker's embedded DNS resolver. Using it (rather than a one-shot lookup +# at haproxy startup) is what lets this config track persys-scheduler +# being scaled up/down, and containers restarting with new IPs, without +# needing haproxy itself restarted. +resolvers dockerdns + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 10s + +# HTTP (non-mTLS) API +frontend scheduler_http_in + bind *:8084 + default_backend scheduler_http_back + +backend scheduler_http_back + balance roundrobin + # server-template pre-declares N backend slots and lets each one + # independently resolve/re-resolve "persys-scheduler" via Docker's + # embedded DNS, which returns one address per replica of a scaled + # Compose service. Bump the "3" if you scale the scheduler further. + # init-addr none avoids haproxy refusing to start if the scheduler + # containers aren't up yet when this container starts. + server-template scheduler 3 persys-scheduler:8084 check inter 5s resolvers dockerdns init-addr none + +# gRPC + mTLS API. mode tcp (inherited from defaults) is required here, +# not optional: this is mutual TLS between agent and scheduler, so haproxy +# must pass the raw TCP stream through untouched rather than terminate +# TLS itself — terminating here would break client-certificate +# verification on the scheduler side. +frontend scheduler_grpc_in + bind *:8085 + default_backend scheduler_grpc_back + +backend scheduler_grpc_back + balance roundrobin + server-template scheduler 3 persys-scheduler:8085 check inter 5s resolvers dockerdns init-addr none + +frontend prometheus + bind :8405 + mode http + http-request use-service prometheus-exporter + no log + From a065f1aa148b5ae94e6472f873db71bbd24d73fe Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:53:48 +0330 Subject: [PATCH 29/31] Feat: Update documentation and configuration for scheduler's high availability and Redis event streaming --- persys-scheduler/CHANGELOG.md | 62 ++++ persys-scheduler/README.md | 76 +++- persys-scheduler/docs/SCHEDULER_DESIGN_V1.md | 351 ++++++++++++++----- persys-scheduler/sample.env | 19 +- 4 files changed, 406 insertions(+), 102 deletions(-) diff --git a/persys-scheduler/CHANGELOG.md b/persys-scheduler/CHANGELOG.md index 68a0623..967aac4 100644 --- a/persys-scheduler/CHANGELOG.md +++ b/persys-scheduler/CHANGELOG.md @@ -1,5 +1,67 @@ # Changelog +## 2026-07-28 (Unreleased) + +Source: 1000+ node scaling initiative (`persys-scheduler-1000-node-scaling-plan.md`) + +### Summary + +This release targets running persys-scheduler at 1,000-5,000+ node fleets and behind multiple replicas. It cuts agent-facing fan-out from O(workloads) to O(nodes), adds etcd compare-and-swap to every concurrent write path, adds a watch-backed node cache for placement, adds leader election with an optional active-active sharding mode, replaces single-factor node scoring with a weighted placement algorithm that accounts for in-flight (not-yet-heartbeated) resource commitments, and fixes CoreDNS self-registration so persys-gateway can discover more than one running replica at a time. + +### Major Features + +1. **Agent connection pooling** + - Replaced per-RPC dial-then-close with a pooled `map[nodeID]*grpc.ClientConn`, keepalive-checked and reused across calls + - Removes a full TLS handshake from every single agent RPC — previously paid on every apply/delete/status/list call, to every node, every cycle + +2. **Reconciliation fan-out: O(workloads) → O(nodes)** + - The reconciler now fetches each node's full workload list once per cycle (via the batch RPC drift-detection already used) instead of one status RPC per workload + - Bounded-concurrency workload processing (`SCHEDULER_RECONCILE_CONCURRENCY`, default 64), plus a cycle-overlap guard so a slow cycle can't stack with the next tick + - `MonitorNodes`, `MonitorWorkloads`, and `detectDriftOnce` got the same bounded-concurrency treatment + +3. **etcd compare-and-swap on all concurrent write paths** + - New `RetryableEtcdCASPut`; every node-mutating function (heartbeat, drain/ready/taint/label/capability updates, NotReady transitions) and every workload-status-mutating function (`UpdateWorkloadStatus`, `UpdateWorkloadLogs`, `UpdateWorkloadMetadata`, `UpdateWorkloadRuntimeDetails`) now retries against a fresh read on conflict instead of silently overwriting a concurrent writer + - Removed a dead `/retries/{id}` write nothing ever read; fixed a bug where usage telemetry (`Usage`) was silently dropped from the etcd status projection on every read + +4. **Watch-backed live node cache for placement** + - New `node_watch.go`: full resync then a live etcd `Watch` keep the in-memory node cache current + - `selectNodeForWorkload` reads from it, falling back to a live scan only if the cache isn't populated yet (startup, or mid-resync) + +5. **Leader election with failover / active-active HA modes** (new env: `SCHEDULER_HA_MODE`, `SCHEDULER_SHARD_COUNT`, `SCHEDULER_SHARD_INDEX`) + - New `leader.go`: etcd-lease-based election (`go.etcd.io/etcd/client/v3/concurrency`) so multiple scheduler replicas can run against the same etcd cluster with automatic failover + - `SCHEDULER_HA_MODE=failover` (default): exactly one replica active cluster-wide; the others are hot standbys that take over automatically if it dies + - `SCHEDULER_HA_MODE=active-active`: nodes are partitioned across `SCHEDULER_SHARD_COUNT` shards by a stable hash of node ID; each replica only drives reconciliation/monitoring/drift-detection for the nodes in its `SCHEDULER_SHARD_INDEX`, so multiple shards run concurrently — run more than one replica per shard index for HA within a shard + - The gRPC API (`RegisterNode`, `Heartbeat`, `ApplyWorkload`, ...) runs unconditionally on every replica in both modes, since those paths are CAS-protected; only the singleton convergence loops are gated + +6. **Weighted placement algorithm with in-flight resource reservation** + - Replaced single-factor "lowest CPU+memory average" sorting with a weighted score: CPU headroom, memory headroom, and a spread term (workload count relative to the busiest candidate), plus a small deterministic tie-breaker so exact ties don't always resolve to the same node + - New in-flight reservation tracking (`placement.go`): resources committed to a just-assigned workload are counted against its node immediately, before that node's next heartbeat reflects the change — closes a real oversubscription window that gets materially more likely now that reconciliation, monitoring, and (in active-active mode) multiple scheduler replicas can all be placing/converging workloads concurrently + +7. **CoreDNS multi-replica fix** + - Scheduler self-registration (used by persys-gateway to discover the scheduler) previously wrote to one fixed etcd key; every replica overwrote the others on startup, so the gateway could only ever resolve one replica regardless of how many were running + - Now keyed per-instance (mirroring the pattern already used for agent node registration), so CoreDNS returns one record per running replica; added deregistration on clean shutdown so a stopped replica doesn't linger as a dead record until its TTL expires + - Note: this DNS mechanism is for **gateway → scheduler** discovery only; agents do not use CoreDNS to reach the scheduler + +### Deployment note + +Running multiple scheduler replicas under plain `docker compose up` (not Swarm) requires removing the scheduler's own host port publishing and putting a TCP-passthrough load balancer (e.g. HAProxy) in front instead — otherwise replicas past the first fail to bind the same host port. See `docker-compose.scheduler-ha.snippet.yml` and `haproxy.cfg`. Not needed under `docker stack deploy` (Swarm), where the ingress routing mesh already handles this. + +### Breaking Changes + +None. `SCHEDULER_HA_MODE` defaults to `failover`, which preserves prior single-active-instance behavior exactly. All other new env vars have defaults matching prior behavior (`SCHEDULER_RECONCILE_CONCURRENCY=64`, `SCHEDULER_SHARD_COUNT=1`, `SCHEDULER_SHARD_INDEX=0`). + +### Known Limitations + +- Active-active mode has no explicit shard hand-off protocol: if a node fails and its workloads are reassigned to a node owned by a different shard, there's a brief window (expected to self-heal within one reconcile interval) where neither shard is actively driving that workload. See `sharding.go` for details. +- Redis remains a single instance with no Sentinel/cluster failover. +- Not load-tested at target scale in this round; reasoning is from code inspection, not measurement. + +### Changed Files + +See `persys-scheduler-scaling-fixes.patch` for the full diff. New files: `internal/scheduler/leader.go`, `internal/scheduler/sharding.go`, `internal/scheduler/node_watch.go`, `internal/scheduler/placement.go`. + +--- + ## 2026-05-29 (Unreleased) Source: `git diff -- persys-scheduler` diff --git a/persys-scheduler/README.md b/persys-scheduler/README.md index c3bd60b..5326e8b 100644 --- a/persys-scheduler/README.md +++ b/persys-scheduler/README.md @@ -6,8 +6,9 @@ It accepts node registrations, stores cluster state in etcd, places workloads on ## What This Service Does - Exposes a gRPC control API for nodes and workload lifecycle. -- Persists scheduler state in etcd (`/nodes`, `/workloads-spec`, `/workloads-status`, `/volumes`, `/attachments`, assignments, retries, reconciliation records, events). -- Offloads high-churn telemetry data to Redis for automatic cleanup (reconciliation metadata, event logs). +- Persists scheduler state in etcd (`/nodes`, `/workloads-spec`, `/workloads-status`, `/volumes`, `/attachments`, assignments, retries, reconciliation records). +- Emits cluster-wide events (node lost, workload scheduled/failed, drift detected, retries, ...) to a Redis Stream — not etcd — for exactly the reasons under "Cluster Events" below. +- Offloads high-churn telemetry data to Redis for automatic cleanup (reconciliation metadata, event history). - Schedules workloads based on node readiness, resources, labels, supported workload types, and storage driver capabilities. - Reconciles workloads (`Running` / `Stopped` / `Deleted`) against agent-reported state with exponential backoff protection. - Manages workload retry state with failure grace periods to allow transient failures to self-heal. @@ -24,13 +25,56 @@ flowchart LR AG[Compute Agents] -->|RegisterNode + Heartbeat| SCH SCH -->|Apply/Delete/Get/ListWorkloads| AG - SCH -->|State + Assignments + Events + Drift Marks| ETCD[(etcd)] + SCH -->|State + Assignments + Drift Marks| ETCD[(etcd)] + SCH -->|Events + Reconciliation Telemetry| REDIS[(Redis)] SCH -->|A/SRV records| DNS[(CoreDNS)] SCH -->|/metrics| PROM[(Prometheus)] SCH -->|OTLP traces| OTLP[(Jaeger/OTel Collector)] ``` +## High Availability and Sharding + +Multiple scheduler replicas can run against the same etcd cluster. `SCHEDULER_HA_MODE` controls how they coordinate: + +- **`failover`** (default): all replicas contend for a single etcd-lease-based election. Exactly one is ever active — driving reconciliation, node/workload monitoring, drift detection, and the placement node-cache watch. The others are hot standbys; if the active replica dies or its lease expires (crash, GC pause past the lease TTL, network partition), another takes over automatically, typically within one lease TTL (15s by default). +- **`active-active`**: nodes are partitioned across `SCHEDULER_SHARD_COUNT` shards by a stable hash of node ID. Each replica is assigned a `SCHEDULER_SHARD_INDEX` and only drives convergence for the nodes that hash into it — so multiple shards make progress concurrently instead of one replica doing all the work. Run more than one replica per shard index to get failover *within* a shard. + +In both modes, the gRPC API (`RegisterNode`, `Heartbeat`, `ApplyWorkload`, `DeleteWorkload`, ...) runs unconditionally on **every** replica — those write paths are protected by etcd compare-and-swap, so it's safe for an external load balancer or Swarm's ingress mesh to route agent traffic to any replica regardless of which one currently holds an election. Only the background convergence loops are gated. + +**Known limitation of `active-active` mode**: there's no explicit hand-off protocol between shards. If a node fails and its workloads are reassigned (`RelocateWorkloadsFromNode`) to a node owned by a different shard, that shard picks up ownership on its next cycle automatically — but there's a brief window (expected to self-heal within one reconcile interval) where neither shard is actively driving that specific workload. This is a scheduling-latency gap, not a correctness bug, but worth knowing before relying on `active-active` mode for latency-sensitive failover. + +Deploying more than one replica without Docker Swarm requires a TCP-passthrough load balancer in front (see `docker-compose.scheduler-ha.snippet.yml` / `haproxy.cfg` in the repo) — plain `docker compose up` can't have multiple containers of the same service all bind the same host port. This isn't needed under `docker stack deploy`, where Swarm's own ingress routing mesh already handles it. + +## Object storage (Ceph RGW) + +Object buckets are **not** stored in etcd. The scheduler proxies **Ceph RGW** (S3 API): + +| Operation | Backend | +|-----------|---------| +| List / create / delete bucket | RGW | +| List objects | RGW | +| User access key / secret | **Vault KV** (path `secret/data/persys/rgw/buckets/{name}` by default) | + +- Control-plane RGW credentials: `PERSYS_RGW_ENDPOINT`, `PERSYS_RGW_ACCESS_KEY`, `PERSYS_RGW_SECRET_KEY` +- Vault auth: **vault-manager** `GetServiceCredentials(PERSYS_VAULT_SERVICE_NAME)` → AppRole login (same pattern as certmanager). Optional `PERSYS_VAULT_TOKEN` for break-glass only. +- Optional `PERSYS_RGW_ADMIN_PATH` (default `admin`) registers generated keys via RGW Admin Ops so S3 clients can authenticate. +- gRPC: `CreateBucket`, `ListBuckets`, `GetBucket`, `DeleteBucket`, `GetBucketAccess`, `ListBucketObjects` +- REST (via gateway): `/buckets`, `/buckets/:id`, `/buckets/:id/access`, `/buckets/:id/objects` +- **Agents are not involved** in object storage. + +Standalone **block disks** remain a separate control-plane surface (`/disks`, hard pinning for local attach). + +## Placement + +`selectNodeForWorkload` filters candidate nodes for feasibility (status, taints, workload-type capability, storage driver, CPU/memory availability), then scores the survivors and picks the highest score. The score blends: + +- CPU and memory headroom, adjusted for resources already committed to workloads assigned earlier in the same scheduling window but not yet reflected in that node's own heartbeat-reported availability (see "in-flight reservations" below). +- A spread term based on how many workloads are already on the node relative to the busiest candidate, so nodes with similar CPU/memory ratios aren't treated as identical if one is already hosting far more workloads. +- A small deterministic tie-breaker (hashed from workload + node ID) so exact ties — common in a homogeneous fleet — don't always resolve to the same node. + +**In-flight reservations**: a node's `AvailableCPU`/`AvailableMemory` only update via that node's own heartbeat, which lags behind a placement decision. The scheduler tracks recently-assigned-but-not-yet-heartbeat-confirmed commitments in memory and subtracts them from a node's effective headroom during scoring (self-expiring after 90s), so concurrent placement decisions — routine now that reconciliation runs with bounded concurrency and `active-active` mode can have multiple replicas placing workloads simultaneously — don't all pick the same "least loaded" node and oversubscribe it before any of their heartbeats catch up. + ## Operating Modes The scheduler now runs with explicit operating modes: @@ -92,19 +136,26 @@ The scheduler uses Redis to store high-churn telemetry data, significantly reduc ### What Gets Stored in Redis - Reconciliation metadata (per-workload retry attempt tracking, backoff timers) -- Event history (bounded list with TTL and max entries) +- Cluster-wide event history (Redis Stream — see "Cluster Events" below) - Optionally, high-frequency reconciliation status updates ### Data Retention - Reconciliation data: TTL 24 hours (configurable via `REDIS_RECONCILE_TTL`) -- Event history: TTL 24 hours (configurable via `REDIS_EVENT_TTL`) -- Maximum event entries: 1000 (configurable via `REDIS_EVENT_MAX_ENTRIES`) +- Event history: TTL 24 hours, refreshed on every write (configurable via `REDIS_EVENT_TTL`); capped at 1000 entries via approximate stream trimming (configurable via `REDIS_EVENT_MAX_ENTRIES`) ### Graceful Fallback -- If Redis is unavailable, scheduler automatically falls back to etcd for all storage -- Scheduler continues operating normally with etcd-only mode +- Reconciliation metadata falls back to etcd if Redis is unavailable. +- Cluster events do **not** fall back to etcd — see "Cluster Events" below for why. If Redis is unavailable, events are dropped (logged) rather than persisted elsewhere, and `ListEvents`/`WatchEvents` callers see an empty result rather than an error. + +## Cluster Events + +`emitEvent` records cluster-wide, human-readable events — `NodeJoined`, `NodeLost`, `NodeLeft`, `WorkloadScheduled`, `WorkloadFailed`, `DriftDetected`, `RetryTriggered`, `Rescheduled`, `Relocated` — for consumption by `persysctl` and dashboard UIs via the `ListEvents` (recent history) and `WatchEvents` (live tail, replays recent history first) gRPC RPCs, both supporting optional filtering by `type`/`workload_id`/`node_id`. + +Events live **only in Redis** (a single shared Stream, `scheduler:events`), not etcd. All scheduler replicas share the same Redis instance, so this is cluster-wide in exactly the same sense etcd-backed state is — an event emitted by whichever replica handled the triggering request is visible to every replica's `WatchEvents` callers. + +This is a deliberate choice, not an oversight: events are high-churn and purely observability-oriented, so they shouldn't compete for etcd's write throughput with heartbeats and CAS-retried reconciliation — especially since event volume tends to spike at exactly the moments (node flapping, mass retries during an incident) when etcd is already under the most load from everything else. Redis Streams also give bounded retention for free (`MAXLEN ~` trimming applied at write time) instead of needing a separate scan-and-delete sweep, which the previous etcd-backed version required since event IDs are random UUIDs with no cheap time-range key trick available. - No data loss or service interruption ### Storage Benefits @@ -164,10 +215,11 @@ When reconciliation/apply fails, scheduler updates workload retry state: ## DNS and Service Discovery -- Scheduler self-registers in CoreDNS on startup. +- Scheduler self-registers in CoreDNS on startup, for discovery **by persys-gateway** — agents do not use CoreDNS to reach the scheduler (they connect via the address/LB they were configured with). - SRV record: `_persys-scheduler.`. - A record fallback: `persys-scheduler.`. -- Agents register under shard-aware records: `..agents.persys.cloud`. +- Each replica registers under its own instance-keyed child rather than one shared key, so running multiple replicas produces one record per running replica instead of the last one to start silently overwriting the others. Deregisters its own record on clean shutdown. +- Agents register under shard-aware records: `..agents.persys.cloud`. (Note: `SCHEDULER_SHARD_KEY` here is a DNS namespace prefix for multi-environment segregation — unrelated to the reconciliation sharding described under High Availability below, despite the similar name.) - If CoreDNS is unavailable, scheduler logs a warning and continues running. ## API @@ -272,13 +324,15 @@ mTLS: - `PERSYS_VAULT_ADDR` (default `http://127.0.0.1:8200`) - `PERSYS_VAULT_AUTH_METHOD` (`token` or `approle`) - `PERSYS_VAULT_TOKEN` (token auth) -- `PERSYS_VAULT_APPROLE_ROLE_ID` / `PERSYS_VAULT_APPROLE_SECRET_ID` (AppRole auth) +- `PERSYS_VAULT_MANAGER_ADDR` (default `vault-manager:50069`) — AppRole via vault-manager +- `PERSYS_VAULT_APPROLE_ROLE_ID` / `PERSYS_VAULT_APPROLE_SECRET_ID` (optional legacy; prefer vault-manager) - `PERSYS_VAULT_PKI_MOUNT` (default `pki`) - `PERSYS_VAULT_PKI_ROLE` (default `persys-scheduler`) - `PERSYS_VAULT_CERT_TTL` (default `24h`) - `PERSYS_VAULT_RETRY_INTERVAL` (default `1m`) - `PERSYS_VAULT_SERVICE_NAME` (default `persys-scheduler`) - `PERSYS_VAULT_SERVICE_DOMAIN` (optional) +- Object storage: `PERSYS_RGW_*` and Vault KV paths — see **Object storage (Ceph RGW)** above ## Workload Utilization Telemetry diff --git a/persys-scheduler/docs/SCHEDULER_DESIGN_V1.md b/persys-scheduler/docs/SCHEDULER_DESIGN_V1.md index 97beeeb..213a991 100644 --- a/persys-scheduler/docs/SCHEDULER_DESIGN_V1.md +++ b/persys-scheduler/docs/SCHEDULER_DESIGN_V1.md @@ -77,18 +77,26 @@ API server responsibilities: - Persist desired state into etcd - Trigger scheduling/reconciliation workflows - Expose read APIs for status and inventory +- Cluster-wide event streaming (gRPC server-streaming) -API server must not issue runtime execution directly. Runtime actions are emitted via scheduler workers. +API server must not issue runtime execution directly. Runtime actions are emitted via scheduler workers. The API surface runs unconditionally on every scheduler replica — write paths are protected by etcd compare-and-swap, making it safe to route agent traffic to any replica. -Minimum API surface: +gRPC API surface: -- `POST /workloads` -- `PUT /workloads/{id}` -- `DELETE /workloads/{id}` -- `GET /workloads/{id}` -- `GET /workloads` -- `GET /nodes` -- `POST /workloads/{id}/retry` +**Workload Management:** +- `ApplyWorkload` / `DeleteWorkload` / `GetWorkload` / `ListWorkloads` +- `RetryWorkload` + +**Node Management:** +- `RegisterNode` / `Heartbeat` / `GetClusterSummary` + +**Events (cluster-wide, Redis-backed):** +- `ListEvents(type, workload_id, node_id, limit)` - Historical event replay +- `WatchEvents(type, workload_id, node_id)` - Server-streaming live event feed + +**Storage:** +- `CreateDisk` / `ListDisks` / `GetDisk` / `DeleteDisk` (managed volume inventory) +- `CreateBucket` / `ListBuckets` / `GetBucket` / `DeleteBucket` / `GetBucketAccess` / `ListBucketObjects` (Ceph RGW proxy) ### 4.2 State Store (etcd) @@ -136,77 +144,106 @@ Workload record (canonical shape): } ``` -### 4.3 Node Manager +### 4.3 Node Manager and Node Watch -Responsibilities: +**Node Manager** is responsible for: +- Node registration (RegisterNode) +- Heartbeat processing (Heartbeat) +- Node status transitions (Ready/NotReady/Draining) +- Node taints and labels (operator control) +- Capacity accounting (used/available CPU, memory) -- Node registration -- Heartbeat processing -- Capacity accounting -- Node readiness transitions -- Eviction trigger when node is unhealthy +**Node Watch** is a leader-elected singleton (like reconciliation) that maintains a live in-memory cache of all nodes: +- Performs full resync from etcd (read all `/nodes/*`) +- Establishes live etcd watch stream on `/nodes/*` prefix +- Applies incoming watch events to in-memory map +- Falls back to resync on watch stream errors +- Serves as source of truth for placement decisions (via `candidateNodeSnapshot`) -Node record (canonical shape): +**Why Node Watch matters:** +- Placement algorithm needs to scan all nodes to pick candidates +- Without a live cache, every placement decision = O(nodes) etcd scan +- With live cache, placement = O(1) in-memory read of pre-populated map +- Cache is rebuilt on leader failover (new leader starts node watch immediately) -```json -{ - "id": "node-abc", - "status": "Ready | NotReady | Draining", - "resources": { - "cpu_total": 16, - "cpu_allocated": 8, - "memory_total": 32768, - "memory_allocated": 16000, - "disk_total": 500, - "disk_allocated": 200 - }, - "last_heartbeat": "2026-02-17T00:00:00Z", - "labels": { - "rack": "rack-1", - "zone": "zone-a" - } -} -``` +**Fallback:** +- If node cache is not yet ready (startup, resync in progress), placement falls back to live etcd scan +- Graceful degradation: scheduling works even during leadership transitions -Health policy: +### 4.4 Placement Engine -- Missing heartbeat for 3 intervals -> `NotReady` -- `NotReady` beyond grace -> workload eviction/reschedule +The placement engine makes deterministic workload-to-node assignments based on resource availability, labels, and workload-type capabilities. -### 4.4 Placement Engine +**Inputs:** +- Workload resource requests (CPU, memory, disk) +- Workload type (container, compose, vm, etc.) +- Workload label constraints +- Node available capacity (from heartbeats, adjusted for in-flight reservations) +- Node status (Ready/NotReady/Draining) +- Node labels and supported workload types + +**Algorithm:** + +1. **Filter**: Eliminate ineligible nodes: + - Status is not `Ready` (skip draining, not-ready nodes) + - Insufficient CPU/memory/disk capacity + - Labels don't match workload requirements + - Node doesn't support workload type (container, vm, etc.) + - Storage driver requirements not met -Inputs: +2. **Score**: For each candidate, compute weighted score: + - CPU headroom factor (weight 0.4): (available_cpu - in_flight_cpu) / total_cpu + - Memory headroom factor (weight 0.4): (available_memory - in_flight_memory) / total_memory + - Spread factor (weight 0.2): how loaded relative to busiest candidate (workload_count / busiest_count) + - Deterministic tie-breaker: hash(workload_id + node_id) for stable ordering when tied -- Workload resource requests -- Node available capacity -- Node labels and readiness +3. **Assign**: Pick the highest-scoring node, record assignment in etcd, reserve resources in-flight -Algorithm (V1): +4. **Reserve**: Immediately record in-memory reservation for just-assigned workload (CPU, memory, expires 90s) + - Prevents concurrent placement decisions from all reading stale "available" numbers + - Advisory (soft limit); self-expires via TTL rather than explicit confirmation -1. Filter nodes by: - - `Ready` - - CPU capacity - - Memory capacity - - Disk capacity - - Label constraints -2. Sort by ascending utilization -3. Choose first node -4. Persist assignment and decision metadata +**In-Flight Reservations:** + +Node `AvailableCPU` and `AvailableMemory` only update via that node's heartbeat, which lags behind placement decisions. When multiple placement decisions (reconciliation, monitoring, multiple replicas in active-active mode) happen concurrently, they can all read the same stale "available" numbers and pick the same "least loaded" node, causing oversubscription before any heartbeat catches up. + +In-flight reservations track resources committed to just-assigned workloads and subtract them from a node's effective capacity during scoring. They self-expire after 90 seconds (multiple heartbeat intervals), trading small scoring precision loss for not needing to hook into every status-confirmation path. ### 4.5 Reconciliation Engine -Loop interval: default 5 seconds (configurable). +The reconciliation engine continuously converges workload desired state to actual state as reported by agents. + +**Execution Model:** + +- Leader-elected single instance in failover mode, or shard-partitioned replicas in active-active mode (see section 8) +- Runs every `SCHEDULER_RECONCILE_INTERVAL` (default 5s) +- Bounded concurrency: processes at most `SCHEDULER_RECONCILE_CONCURRENCY` workloads concurrently (default 64) +- Cycle-overlap guard: prevents a slow cycle from stacking with the next tick + +**Optimization: O(nodes) instead of O(workloads) agent communication:** + +Traditional approach: one `GetWorkloadStatus` RPC per workload, per cycle = O(workloads) fan-out. + +New approach: +1. Prefetch snapshots: call `GetWorkloads` once per node (batched list) = O(nodes) fan-out +2. Cache result for the cycle duration +3. Within the cycle, `getActualWorkloadState` consults the snapshot before falling back to a live per-workload RPC +4. Dramatically reduces agent load when workload count >> node count + +**Loop interval:** default 5 seconds (configurable via `SCHEDULER_RECONCILE_INTERVAL`). -For each workload: +**For each workload (with bounded concurrency):** -1. Load desired state from etcd -2. Query agent actual state (`GetWorkloadStatus`) +1. Load desired state from etcd (spec + retry metadata) +2. Query agent actual state: + - First check node snapshot from prefetch (if available) + - Fall back to live `GetWorkloadStatus` RPC if not in snapshot 3. Compare desired vs actual -4. Choose action -5. Execute action (`ApplyWorkload`/`DeleteWorkload`) -6. Persist result and timestamps +4. Choose action (see decision matrix below) +5. Execute action (etcd CAS write + agent RPC) +6. Persist result (status, timestamps, metrics) -Decision matrix: +**Decision matrix:** | Desired | Actual | Action | |---------|---------|----------| @@ -217,11 +254,21 @@ Decision matrix: | Deleted | Exists | Delete | | Running | Running | NoAction | -Rules: +**Concurrency Control:** -- Do not execute before grace period expires for transitional states. -- Persist reconciliation metadata for every action. -- No tight retry loops inside one cycle. +All etcd writes use compare-and-swap (`RetryableEtcdCASPut`): +- Heartbeat updates +- Node drain/ready/taint/label transitions +- Workload status updates (status, logs, metadata, runtime details) +- On conflict, reload from etcd and retry (not silent overwrite) + +**Rules:** + +- Do not execute before grace period expires for transitional states (`SCHEDULER_MISSING_GRACE_PERIOD`, default 15s) +- Persist reconciliation metadata for every action +- No tight retry loops inside one cycle +- Metrics: track per-workload attempt count, backoff timer, failure reason +- Failed workloads with terminal failure reasons skip retry and transition to `Failed` state ### 4.6 Retry Engine @@ -243,19 +290,64 @@ On failure: ### 4.7 Event System -Event types: - -- `WorkloadScheduled` -- `WorkloadFailed` -- `NodeLost` -- `RetryTriggered` -- `Rescheduled` - -Rules: - -- Events are immutable -- Events are append-only -- Include workload id, node id, reason, timestamp +Events are cluster-wide, immutable, append-only records of state transitions and significant control-plane occurrences. They are **stored in a Redis Stream, not etcd**, to avoid etcd fan-out issues at scale. + +**Storage:** +- Single shared Redis Stream across all scheduler replicas +- TTL-based retention (configurable via `REDIS_EVENT_TTL`, default 24h) +- Size-based trimming (approximate, configurable via `REDIS_EVENT_MAX_ENTRIES`, default 1000 entries) + +**Why Redis instead of etcd:** +- Events are high-churn data with well-defined retention windows +- etcd is optimized for mutable state; event-only workloads stress etcd unnecessarily +- Redis Streams are purpose-built for event logging with automatic TTL cleanup +- Graceful degradation: if Redis is down, events are dropped (logged) but scheduler continues; if etcd is down, scheduler enters degraded mode + +**Event Types:** + +Topology: +- `NodeJoined`: agent registered / re-registered +- `NodeLost`: heartbeat timeout, marked NotReady +- `NodeLeft`: deregistered (graceful) + +Workload Lifecycle: +- `WorkloadScheduled`: placed on a node +- `WorkloadFailed`: reached terminal failure (max retries exceeded) +- `DriftDetected`: agent state diverged from desired +- `RetryTriggered`: retry backoff timer expired, retrying +- `Rescheduled`: workload moved to different node (same retry attempt) +- `Relocated`: workload moved due to node drain/failure + +Operator Control: +- `NodeDraining`: operator requested drain +- `NodeReady`: operator cleared drain +- `NodeTainted`: operator applied taint +- `NodeUntainted`: operator removed taint +- `NodeLabelSet`: operator set label +- `NodeLabelDeleted`: operator deleted label + +Control-Plane: +- `SchedulerModeChanged`: transitioned between normal/degraded/recovery +- `LeaderElected`: this instance won leader election +- `LeaderLost`: this instance lost leader election + +**Event API:** + +- `ListEvents(type, workload_id, node_id, limit)` - Query historical events (oldest-first) +- `WatchEvents(type, workload_id, node_id)` - Server-streaming, replays recent history then follows live events + +**Consumption:** +- Dashboard via SSE (HTTP gateway endpoint) +- Alerting systems (watch for WorkloadFailed, NodeLost) +- Audit trails +- Observability: correlate events with metrics/traces + +**Rules:** + +- Events are immutable after creation +- Events are append-only (no deletion or replay) +- Every event includes: id (UUID), type, workload_id (optional), node_id (optional), reason, timestamp, details (string map) +- Redis stream ID ensures exactly-once delivery across replay-to-live boundary (no gap, no duplicate) ### 4.8 Automation Hooks @@ -315,17 +407,96 @@ Node crash: ## 7. Concurrency Model -- Worker pool for reconciliation actions -- Context cancellation for all loops and RPCs -- Optimistic etcd writes where feasible -- Avoid global locks across reconciliation workers - -## 8. High Availability (Future) - -- Multiple stateless scheduler replicas -- Shared etcd state -- Leader election via etcd lease -- Only leader executes reconciliation and retries +The scheduler is designed for safe concurrent operation across multiple replicas. + +**Within Single Instance:** +- Background loops (reconciliation, monitoring, drift detection, node watch) use goroutines with context cancellation +- Bounded concurrency for workload/node processing (default 64) prevents resource exhaustion +- Cycle-overlap guard ensures sequential cycles (no interleaving) +- In-memory state (caches, reservations) protected by sync.RWMutex +- etcd is single source of truth; cache is rebuilt on restart + +**Across Multiple Replicas:** +- All etcd writes use compare-and-swap (`RetryableEtcdCASPut`) + - On conflict, reload fresh state and retry, rather than silent overwrite + - Safe for concurrent writer elimination (only one successfully commits each CAS) +- gRPC API runs on every replica (stateless, safe) +- Background singleton loops (reconciliation, drift, monitoring, node watch) are gated by leader election + - In failover mode: exactly one replica active cluster-wide + - In active-active mode: replicas are partitioned by shard; each shard has its own leader + +**Agent Communication (Connection Pooling):** +- gRPC connections to agents are pooled (map[nodeID]*grpc.ClientConn) +- Connections are reused across RPCs with keepalive checks +- TLS handshake happens once per unique node (cached until connection failure) +- On TLS errors, cert manager can force rotation + retry automatically +- Eliminates O(RPCs) TLS handshakes that were previously paid on every apply/delete/status call + +**Monitoring & Observability:** +- All mutations log comprehensively (node, workload, event) +- Metrics track: attempts, failures, retry counts, latencies +- Traces link placement → reconciliation → agent RPC + +## 8. High Availability and Leader Election + +Multiple scheduler replicas can run against the same etcd cluster, with automatic failover. Leadership determines which replica drives cluster-wide singleton background loops. + +### 8.1 Leader Election + +- Uses etcd-lease-based election (`go.etcd.io/etcd/client/v3/concurrency`) +- Session TTL: 15 seconds (balance between fast failover and transient GC/network resilience) +- Winner is determined by etcd atomically +- Lost leader automatically campaigns again after 2s backoff + +### 8.2 Failover Mode (Default) + +Configuration: `SCHEDULER_HA_MODE=failover` (default) + +Behavior: +- All replicas contend for the single `leaderElectionKey` in etcd +- Exactly one replica is ever elected leader cluster-wide +- Leader drives: + - Reconciliation loops (ReconcileAllWorkloads) + - Node/workload monitoring (MonitorNodes, MonitorWorkloads) + - Drift detection (StartDriftDetection) + - Node watch (StartNodeWatch) - live cache updates +- Standbys (non-leaders) are hot spares: + - Still run `StartMonitoring()` for per-replica health checks + - Can handle gRPC API calls (reads, writes with CAS) + - Will take over if leader dies/loses session + +**Example deployment:** 3 replicas, 1 active, 2 standbys. Leader dies → standby wins election within 15s TTL. + +### 8.3 Active-Active Mode (Optional Sharding) + +Configuration: `SCHEDULER_HA_MODE=active-active`, `SCHEDULER_SHARD_COUNT=N`, `SCHEDULER_SHARD_INDEX=0..N-1` + +Behavior: +- Nodes are partitioned by `crc32(nodeID) % SCHEDULER_SHARD_COUNT` hash +- Each replica is assigned a shard index and owns nodes that hash to that index +- Replicas with different shard indices never contend with each other (independent elections per shard) +- Replicas sharing same shard index do elect one leader (for HA within shard) +- Each shard processes its nodes' reconciliation/monitoring/drift independently +- Multiple shards make progress concurrently (unlike failover where only one is active) + +**Example deployment:** 5 replicas, 3 shards, 2 replicas per shard index: +- Shard 0: replicas A, B → A elected leader, B is standby for shard 0 +- Shard 1: replicas C, D → C elected leader, D is standby for shard 1 +- Shard 2: replica E → E is leader of shard 2 +- Nodes are distributed: nodes 0,3,6,... → shard 0; nodes 1,4,7,... → shard 1; nodes 2,5,8,... → shard 2 +- Three shards process workloads concurrently; higher throughput than failover + +**Tradeoff:** Active-active trades slightly higher latency (per-shard overhead) for higher total throughput. Best for large fleets with many nodes. + +**Known limitation:** If a node fails and its workloads are reassigned to a node owned by a different shard, there's a brief window (expected to self-heal within one reconcile interval) where neither shard is actively driving that workload. This is a scheduling-latency gap, not a correctness bug. See `sharding.go` for details. + +### 8.4 API Availability in Both Modes + +The gRPC API (`RegisterNode`, `Heartbeat`, `ApplyWorkload`, etc.) runs unconditionally on every replica in both modes: +- Write paths are protected by etcd compare-and-swap +- Safe for external load balancer to route to any replica +- No leader-gating on API calls +- Scaling: N replicas can handle N× the API throughput (if load-balanced properly) ## 9. Security diff --git a/persys-scheduler/sample.env b/persys-scheduler/sample.env index 05fb417..6f69c69 100644 --- a/persys-scheduler/sample.env +++ b/persys-scheduler/sample.env @@ -22,6 +22,7 @@ PERSYS_TLS_KEY=/etc/persys/certs/persys-scheduler/persys_scheduler-key.key # Scheduler mTLS certificate management (Vault) # When disabled, scheduler uses the manual cert files from PERSYS_TLS_* above. PERSYS_VAULT_ENABLED=true +PERSYS_VAULT_MANAGER_ADDR=vault-manager:50069 PERSYS_VAULT_ADDR=http://vault:8200 PERSYS_VAULT_AUTH_METHOD=token PERSYS_VAULT_TOKEN=root @@ -45,4 +46,20 @@ OTEL_EXPORTER_OTLP_INSECURE=true # Redis REDIS_ADDR=redis:6379 REDIS_PASSWORD= -REDIS_DB=1 \ No newline at end of file +REDIS_DB=1 + +# ============================================================================= +# Ceph RGW — object storage (S3). Scheduler proxies RGW; no etcd bucket inventory. +# User access keys: generated on create, stored in Vault KV only. +# ============================================================================= +# PERSYS_RGW_ENDPOINT=http://rgw:8080 +# PERSYS_RGW_REGION=default +# PERSYS_RGW_ACCESS_KEY= # control-plane admin key (list/create/delete) +# PERSYS_RGW_SECRET_KEY= +# PERSYS_RGW_ADMIN_PATH=admin # optional Admin Ops so generated keys auth on RGW +# PERSYS_RGW_VAULT_PATH_PREFIX=secret/data/persys/rgw/buckets + +# Vault for bucket credentials (and PKI): prefer vault-manager AppRole (same as certmanager) +# PERSYS_VAULT_MANAGER_ADDR=vault-manager:50069 +# PERSYS_VAULT_SERVICE_NAME=persys-scheduler +# PERSYS_VAULT_TOKEN= # break-glass only; production uses vault-manager \ No newline at end of file From 1db1b4f2c7490b71595cef1dad35838f090ba0a7 Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:54:06 +0330 Subject: [PATCH 30/31] Feat: Refactor certmanager integration and update scheduler's background loop handling --- persys-scheduler/cmd/scheduler/main.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/persys-scheduler/cmd/scheduler/main.go b/persys-scheduler/cmd/scheduler/main.go index ec4a731..afc5260 100644 --- a/persys-scheduler/cmd/scheduler/main.go +++ b/persys-scheduler/cmd/scheduler/main.go @@ -17,7 +17,6 @@ import ( "syscall" "time" - // "github.com/persys-dev/persys-cloud/persys-scheduler/internal/auth" "github.com/persys-dev/persys-cloud/pkg/certmanager" cfgpkg "github.com/persys-dev/persys-cloud/persys-scheduler/internal/config" controlv1 "github.com/persys-dev/persys-cloud/persys-scheduler/internal/controlv1" @@ -53,6 +52,7 @@ func main() { var tlsConfig *tls.Config var certCancel context.CancelFunc + var certMgr *certmanager.Manager if !cfg.Insecure { certCfg := certmanager.Config{ TLSEnabled: cfg.TLSEnabled, @@ -77,7 +77,7 @@ func main() { BindHost: cfg.GRPCAddr, } - certMgr := certmanager.NewManager(certCfg, logger.Logger) + certMgr = certmanager.NewManager(certCfg, logger.Logger) certCtx, cancel := context.WithCancel(context.Background()) certCancel = cancel if err := certMgr.Start(certCtx); err != nil { @@ -107,6 +107,8 @@ func main() { logger.WithError(err).Fatal("failed to initialize scheduler") } defer sched.Close() + // Wire certmanager so outbound agent dials/RPCs can ForceRotate + retry on TLS errors. + sched.SetCertManager(certMgr) if err := sched.RefreshStateMetrics(); err != nil { logger.WithError(err).Warn("failed to initialize scheduler state metrics") } @@ -115,7 +117,7 @@ func main() { defer cancel() sched.StartMonitoring(ctx) - sched.StartReconciliation(ctx) + sched.StartLeaderElectedBackgroundLoops(ctx) grpcPort := strconv.Itoa(cfg.GRPCPort) if err := sched.RegisterSchedulerSelfInCoreDNS(cfg.GRPCPort); err != nil { @@ -156,6 +158,20 @@ func main() { _, _ = w.Write(payload) }), "scheduler.health")) metricsMux.Handle("/debug/pprof/", http.DefaultServeMux) + // if certMgr != nil { + // metricsMux.HandleFunc("/debug/force-rotate", func(w http.ResponseWriter, r *http.Request) { + // if r.Method != http.MethodPost { + // http.Error(w, "POST only", http.StatusMethodNotAllowed) + // return + // } + // if err := certMgr.ForceRotate(r.Context()); err != nil { + // http.Error(w, err.Error(), http.StatusInternalServerError) + // return + // } + // w.Header().Set("Content-Type", "application/json") + // _, _ = w.Write([]byte(`{"status":"rotated"}`)) + // }) + // } metricsServer := &http.Server{Addr: net.JoinHostPort(cfg.GRPCAddr, metricsPort), Handler: metricsMux} serverErrCh := make(chan error, 2) From f16fa3c6a243fb7d7dac60d09fb848e68862ad8e Mon Sep 17 00:00:00 2001 From: milx Date: Mon, 7 Sep 2026 19:54:36 +0330 Subject: [PATCH 31/31] Feat: Add gRPC service definitions for agent and control functionalities --- persys-scheduler/api/proto/agent.proto | 25 +- persys-scheduler/api/proto/control.proto | 216 ++++++++ pkg/proto/agent.proto | 260 +++++++++ pkg/proto/control.proto | 674 +++++++++++++++++++++++ 4 files changed, 1171 insertions(+), 4 deletions(-) create mode 100644 pkg/proto/agent.proto create mode 100644 pkg/proto/control.proto diff --git a/persys-scheduler/api/proto/agent.proto b/persys-scheduler/api/proto/agent.proto index bf21006..49e3be3 100644 --- a/persys-scheduler/api/proto/agent.proto +++ b/persys-scheduler/api/proto/agent.proto @@ -2,7 +2,7 @@ syntax = "proto3"; package persys.agent.v1; -option go_package = "github.com/persys-dev/persys-cloud/persys-scheduler/internal/agentpb;agentpb"; +option go_package = "github.com/persys-dev/compute-agent/pkg/api/v1;v1"; // AgentService defines the gRPC interface for workload management service AgentService { @@ -104,6 +104,8 @@ enum WorkloadType { WORKLOAD_TYPE_CONTAINER = 1; WORKLOAD_TYPE_COMPOSE = 2; WORKLOAD_TYPE_VM = 3; + // Firecracker microVM. Shares WorkloadSpec.vm oneof with KVM VMs. + WORKLOAD_TYPE_MICROVM = 4; } enum DesiredState { @@ -158,6 +160,13 @@ message VMSpec { map metadata = 7; CloudInitConfig cloud_init_config = 8; // advanced cloud-init settings repeated ManagedVolumeSpec managed_volumes = 9; + // Happy-path OS image: catalog name or absolute path to a read-only base + // image. Agent creates a writable qcow2 overlay; base is never mutated. + string os_image = 10; + // Root disk size in GB when synthesizing from os_image (default 10). + int64 disk_gb = 11; + // Optional runtime selector: "libvirt" (default) or "firecracker". + string runtime = 12; } message CloudInitConfig { @@ -165,6 +174,9 @@ message CloudInitConfig { string meta_data = 2; // cloud-init meta-data (JSON) string network_config = 3; // cloud-init network config (YAML) string vendor_data = 4; // cloud-init vendor-data + string username = 5; // default login user when generating user-data + string ssh_public_key = 6; // inject authorized key instead of password + string password = 7; // fixed password (otherwise random) } message ManagedVolumeSpec { @@ -202,18 +214,23 @@ message RestartPolicy { } message DiskConfig { - string path = 1; // path to disk image or ISO + string path = 1; // path to disk image or ISO (leave empty with os_image set) string device = 2; // vda, vdb, etc. string format = 3; // qcow2, raw, iso int64 size_gb = 4; string type = 5; // disk or cdrom (for ISO) bool boot = 6; // true if this is the boot disk/ISO + string backing_file = 7; // optional explicit backing image for overlay + string storage = 8; // local|nfs|ceph-rbd hint } message NetworkConfig { - string network = 1; // network name or bridge + string network = 1; // libvirt network name or bridge (default: "default") string mac_address = 2; - string ip_address = 3; // optional static IP + string ip_address = 3; // optional static guest IP + string host_dev_name = 4; // Firecracker host TAP device + string model = 5; // virtio (default) + string bridge = 6; // optional explicit bridge } message WorkloadStatus { diff --git a/persys-scheduler/api/proto/control.proto b/persys-scheduler/api/proto/control.proto index b7ae24f..b195fe5 100644 --- a/persys-scheduler/api/proto/control.proto +++ b/persys-scheduler/api/proto/control.proto @@ -36,8 +36,35 @@ service AgentControl { rpc GetWorkload(GetWorkloadRequest) returns (GetWorkloadResponse); rpc GetClusterSummary(GetClusterSummaryRequest) returns (GetClusterSummaryResponse); + // Cluster-wide events: node joined, node lost, node left, workload + // scheduled, drift detected, retries, reschedules, etc (see + // internal/scheduler/events.go for producers). ListEvents is a + // plain unary call (auto-bridged to REST by persys-gateway's + // reflection-based grpcbridge, no gateway changes needed). WatchEvents + // is a server-streaming call — grpcbridge explicitly does not bridge + // streaming RPCs, so consumers that need HTTP (e.g. a browser + // dashboard) go through a hand-written SSE endpoint on the gateway + // instead of the generic bridge; a gRPC client (e.g. persysctl) can + // call it directly. + rpc ListEvents(ListEventsRequest) returns (ListEventsResponse); + rpc WatchEvents(WatchEventsRequest) returns (stream SchedulerEventView); + // Optional future streaming channel rpc ControlStream(stream ControlMessage) returns (stream ControlMessage); + + // Standalone disk inventory (managed volumes) + rpc CreateDisk(CreateDiskRequest) returns (CreateDiskResponse); + rpc ListDisks(ListDisksRequest) returns (ListDisksResponse); + rpc GetDisk(GetDiskRequest) returns (GetDiskResponse); + rpc DeleteDisk(DeleteDiskRequest) returns (DeleteDiskResponse); + + // Object storage (Ceph RGW / S3-compatible buckets) + rpc CreateBucket(CreateBucketRequest) returns (CreateBucketResponse); + rpc ListBuckets(ListBucketsRequest) returns (ListBucketsResponse); + rpc GetBucket(GetBucketRequest) returns (GetBucketResponse); + rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse); + rpc GetBucketAccess(GetBucketAccessRequest) returns (GetBucketAccessResponse); + rpc ListBucketObjects(ListBucketObjectsRequest) returns (ListBucketObjectsResponse); } enum AutomationActionType { @@ -405,6 +432,42 @@ message ListWorkloadsRequest { string status = 2; // optional filter } +message SchedulerEventView { + string id = 1; + string type = 2; // e.g. "NodeLost", "WorkloadScheduled", "DriftDetected" + string workload_id = 3; // optional, empty if not workload-scoped + string node_id = 4; // optional, empty if not node-scoped + string reason = 5; + google.protobuf.Timestamp timestamp = 6; + // Free-form auxiliary data. Values are stringified on the way out + // (models.SchedulerEvent.Details is map[string]interface{} on the Go + // side) — this is a deliberate simplification over a + // google.protobuf.Struct, since event details are informational/ + // display-oriented, not structured data a client needs to + // round-trip losslessly. + map details = 7; +} + +message ListEventsRequest { + int64 limit = 1; // 0 means server default + string type = 2; // optional filter + string workload_id = 3; // optional filter + string node_id = 4; // optional filter +} + +message ListEventsResponse { + repeated SchedulerEventView events = 1; +} + +message WatchEventsRequest { + // Same optional filters as ListEventsRequest. The stream first replays + // recent matching events (server-side default limit), then continues + // with new matching events as they're emitted. + string type = 1; + string workload_id = 2; + string node_id = 3; +} + message GetWorkloadRequest { string workload_id = 1; } @@ -456,3 +519,156 @@ message ControlMessage { DeleteWorkloadRequest delete = 4; } } + +// --- Standalone disks (control-plane inventory) --- + +message CreateDiskRequest { + string name = 1; + string driver = 2; // local | ceph-rbd | nfs + int64 size_gb = 3; + string fs_type = 4; + string access_mode = 5; + string retain_policy = 6; // Delete | Retain + string node_id = 7; // optional pre-pin for local + string mount_path = 8; +} + +message CreateDiskResponse { + DiskView disk = 1; +} + +message ListDisksRequest {} + +message ListDisksResponse { + repeated DiskView disks = 1; +} + +message GetDiskRequest { + string disk_id = 1; +} + +message GetDiskResponse { + DiskView disk = 1; +} + +message DeleteDiskRequest { + string disk_id = 1; + bool force = 2; +} + +message DeleteDiskResponse { + bool success = 1; + string error_message = 2; +} + +message DiskView { + string id = 1; + string name = 2; + string driver = 3; + int64 size_gb = 4; + string fs_type = 5; + string access_mode = 6; + string retain_policy = 7; + string phase = 8; + string last_error = 9; + string node_id = 10; + string device = 11; + bool standalone = 12; + string mount_path = 13; + repeated string workload_refs = 14; + repeated string attached_nodes = 15; + google.protobuf.Timestamp created_at = 16; + google.protobuf.Timestamp updated_at = 17; +} + +// --- Object storage (Ceph RGW / S3-compatible) --- + +message CreateBucketRequest { + string name = 1; + string region = 2; + bool versioning = 3; +} + +message CreateBucketResponse { + BucketView bucket = 1; + BucketAccess access = 2; // credentials returned once on create +} + +message ListBucketsRequest {} + +message ListBucketsResponse { + repeated BucketView buckets = 1; +} + +message GetBucketRequest { + string bucket_id = 1; // id or name +} + +message GetBucketResponse { + BucketView bucket = 1; +} + +message DeleteBucketRequest { + string bucket_id = 1; + bool force = 2; +} + +message DeleteBucketResponse { + bool success = 1; + string error_message = 2; +} + +message GetBucketAccessRequest { + string bucket_id = 1; +} + +message GetBucketAccessResponse { + BucketAccess access = 1; +} + +message ListBucketObjectsRequest { + string bucket_id = 1; + string prefix = 2; + string continuation_token = 3; + int32 max_keys = 4; +} + +message ListBucketObjectsResponse { + repeated ObjectInfo objects = 1; + string next_continuation_token = 2; + bool is_truncated = 3; + string prefix = 4; +} + +message BucketView { + string id = 1; + string name = 2; + string region = 3; + string owner = 4; + string endpoint = 5; + bool versioning = 6; + int64 object_count = 7; + int64 size_bytes = 8; + string phase = 9; + string last_error = 10; + google.protobuf.Timestamp created_at = 11; + google.protobuf.Timestamp updated_at = 12; +} + +message BucketAccess { + string endpoint = 1; + string region = 2; + string bucket = 3; + string access_key = 4; + string secret_key = 5; + string vault_path = 6; // when secrets live in Vault + string s3_url = 7; // e.g. s3://bucket +} + +message ObjectInfo { + string key = 1; + int64 size_bytes = 2; + string etag = 3; + string last_modified = 4; + string storage_class = 5; +} \ No newline at end of file diff --git a/pkg/proto/agent.proto b/pkg/proto/agent.proto new file mode 100644 index 0000000..49e3be3 --- /dev/null +++ b/pkg/proto/agent.proto @@ -0,0 +1,260 @@ +syntax = "proto3"; + +package persys.agent.v1; + +option go_package = "github.com/persys-dev/compute-agent/pkg/api/v1;v1"; + +// AgentService defines the gRPC interface for workload management +service AgentService { + // ApplyWorkload creates or updates a workload + rpc ApplyWorkload(ApplyWorkloadRequest) returns (ApplyWorkloadResponse); + + // DeleteWorkload removes a workload + rpc DeleteWorkload(DeleteWorkloadRequest) returns (DeleteWorkloadResponse); + + // GetWorkloadStatus retrieves current workload status + rpc GetWorkloadStatus(GetWorkloadStatusRequest) returns (GetWorkloadStatusResponse); + + // ListWorkloads returns all managed workloads + rpc ListWorkloads(ListWorkloadsRequest) returns (ListWorkloadsResponse); + + // HealthCheck returns agent health status + rpc HealthCheck(HealthCheckRequest) returns (HealthCheckResponse); + + // ListActions returns action/task history tracked by the agent since startup + rpc ListActions(ListActionsRequest) returns (ListActionsResponse); +} + +message ApplyWorkloadRequest { + string id = 1; + WorkloadType type = 2; + string revision_id = 3; + DesiredState desired_state = 4; + WorkloadSpec spec = 5; +} + +message ApplyWorkloadResponse { + bool applied = 1; + bool skipped = 2; // true if revision already applied + string message = 3; + WorkloadStatus status = 4; +} + +message DeleteWorkloadRequest { + string id = 1; +} + +message DeleteWorkloadResponse { + bool success = 1; + string message = 2; +} + +message GetWorkloadStatusRequest { + string id = 1; +} + +message GetWorkloadStatusResponse { + WorkloadStatus status = 1; +} + +message ListWorkloadsRequest { + WorkloadType type = 1; // optional filter +} + +message ListWorkloadsResponse { + repeated WorkloadStatus workloads = 1; +} + +message HealthCheckRequest {} + +message HealthCheckResponse { + bool healthy = 1; + string version = 2; + map runtime_status = 3; + double memory_utilization = 4; // Memory utilization percentage (0-100) + double cpu_utilization = 5; // CPU utilization percentage (0-100) + double disk_utilization = 6; // Disk utilization percentage for root mount (0-100) +} + +message ListActionsRequest { + string workload_id = 1; // optional filter + string action_type = 2; // optional filter: apply_workload, delete_workload, ... + string status = 3; // optional filter: pending, running, completed, failed + int32 limit = 4; // optional max number of actions (0 = all) + bool newest_first = 5; // if true sorts by created_at descending +} + +message AgentAction { + string task_id = 1; + string workload_id = 2; + string action_type = 3; + string status = 4; + string error = 5; + int64 created_at = 6; + int64 started_at = 7; + int64 ended_at = 8; +} + +message ListActionsResponse { + repeated AgentAction actions = 1; +} + +enum WorkloadType { + WORKLOAD_TYPE_UNSPECIFIED = 0; + WORKLOAD_TYPE_CONTAINER = 1; + WORKLOAD_TYPE_COMPOSE = 2; + WORKLOAD_TYPE_VM = 3; + // Firecracker microVM. Shares WorkloadSpec.vm oneof with KVM VMs. + WORKLOAD_TYPE_MICROVM = 4; +} + +enum DesiredState { + DESIRED_STATE_UNSPECIFIED = 0; + DESIRED_STATE_RUNNING = 1; + DESIRED_STATE_STOPPED = 2; +} + +enum ActualState { + ACTUAL_STATE_UNSPECIFIED = 0; + ACTUAL_STATE_PENDING = 1; + ACTUAL_STATE_RUNNING = 2; + ACTUAL_STATE_STOPPED = 3; + ACTUAL_STATE_FAILED = 4; + ACTUAL_STATE_UNKNOWN = 5; +} + +message WorkloadSpec { + oneof spec { + ContainerSpec container = 1; + ComposeSpec compose = 2; + VMSpec vm = 3; + } +} + +message ContainerSpec { + string image = 1; + repeated string command = 2; + repeated string args = 3; + map env = 4; + repeated VolumeMount volumes = 5; + repeated PortMapping ports = 6; + ResourceLimits resources = 7; + RestartPolicy restart_policy = 8; + map labels = 9; + repeated ManagedVolumeSpec managed_volumes = 10; +} + +message ComposeSpec { + string project_name = 1; + string compose_yaml = 2; // base64 encoded docker-compose.yml + map env = 3; +} + +message VMSpec { + string name = 1; + int32 vcpus = 2; + int64 memory_mb = 3; + repeated DiskConfig disks = 4; + repeated NetworkConfig networks = 5; + string cloud_init = 6; // optional cloud-init user-data (YAML content) + map metadata = 7; + CloudInitConfig cloud_init_config = 8; // advanced cloud-init settings + repeated ManagedVolumeSpec managed_volumes = 9; + // Happy-path OS image: catalog name or absolute path to a read-only base + // image. Agent creates a writable qcow2 overlay; base is never mutated. + string os_image = 10; + // Root disk size in GB when synthesizing from os_image (default 10). + int64 disk_gb = 11; + // Optional runtime selector: "libvirt" (default) or "firecracker". + string runtime = 12; +} + +message CloudInitConfig { + string user_data = 1; // cloud-init user-data script + string meta_data = 2; // cloud-init meta-data (JSON) + string network_config = 3; // cloud-init network config (YAML) + string vendor_data = 4; // cloud-init vendor-data + string username = 5; // default login user when generating user-data + string ssh_public_key = 6; // inject authorized key instead of password + string password = 7; // fixed password (otherwise random) +} + +message ManagedVolumeSpec { + string name = 1; + string driver = 2; // local|nfs|ceph-rbd + int64 size_gb = 3; + string access_mode = 4; + string fs_type = 5; + string mount_path = 6; + bool read_only = 7; + string retain_policy = 8; // Delete|Retain +} + +message VolumeMount { + string host_path = 1; + string container_path = 2; + bool read_only = 3; +} + +message PortMapping { + int32 host_port = 1; + int32 container_port = 2; + string protocol = 3; // tcp or udp +} + +message ResourceLimits { + int64 cpu_shares = 1; + int64 memory_bytes = 2; + int64 memory_swap_bytes = 3; +} + +message RestartPolicy { + string policy = 1; // no, always, on-failure, unless-stopped + int32 max_retry_count = 2; +} + +message DiskConfig { + string path = 1; // path to disk image or ISO (leave empty with os_image set) + string device = 2; // vda, vdb, etc. + string format = 3; // qcow2, raw, iso + int64 size_gb = 4; + string type = 5; // disk or cdrom (for ISO) + bool boot = 6; // true if this is the boot disk/ISO + string backing_file = 7; // optional explicit backing image for overlay + string storage = 8; // local|nfs|ceph-rbd hint +} + +message NetworkConfig { + string network = 1; // libvirt network name or bridge (default: "default") + string mac_address = 2; + string ip_address = 3; // optional static guest IP + string host_dev_name = 4; // Firecracker host TAP device + string model = 5; // virtio (default) + string bridge = 6; // optional explicit bridge +} + +message WorkloadStatus { + string id = 1; + WorkloadType type = 2; + string revision_id = 3; + DesiredState desired_state = 4; + ActualState actual_state = 5; + string message = 6; + int64 created_at = 7; + int64 updated_at = 8; + map metadata = 9; + WorkloadUsageSnapshot usage = 10; +} + +message WorkloadUsageSnapshot { + string workload_id = 1; + WorkloadType type = 2; + double cpu_percent = 3; + int64 memory_bytes = 4; + int64 disk_read_bytes = 5; + int64 disk_write_bytes = 6; + int64 net_rx_bytes = 7; + int64 net_tx_bytes = 8; + int64 collected_at = 9; // unix timestamp + string source = 10; +} diff --git a/pkg/proto/control.proto b/pkg/proto/control.proto new file mode 100644 index 0000000..b195fe5 --- /dev/null +++ b/pkg/proto/control.proto @@ -0,0 +1,674 @@ +syntax = "proto3"; + +package persys.control.v1; + +option go_package = "github.com/persys-dev/persys/api/control/v1;controlv1"; + +import "google/protobuf/timestamp.proto"; + +service AgentControl { + // Registration + rpc RegisterNode(RegisterNodeRequest) returns (RegisterNodeResponse); + + // Heartbeat + rpc Heartbeat(HeartbeatRequest) returns (HeartbeatResponse); + + // Workload lifecycle + rpc ApplyWorkload(ApplyWorkloadRequest) returns (ApplyWorkloadResponse); + rpc DeleteWorkload(DeleteWorkloadRequest) returns (DeleteWorkloadResponse); + + // Retry trigger + rpc RetryWorkload(RetryWorkloadRequest) returns (RetryWorkloadResponse); + + // Node management + rpc DrainNode(DrainNodeRequest) returns (DrainNodeResponse); + rpc UndrainNode(UndrainNodeRequest) returns (UndrainNodeResponse); + rpc TaintNode(TaintNodeRequest) returns (TaintNodeResponse); + rpc UntaintNode(UntaintNodeRequest) returns (UntaintNodeResponse); + rpc SetNodeLabel(SetNodeLabelRequest) returns (SetNodeLabelResponse); + rpc DeleteNodeLabel(DeleteNodeLabelRequest) returns (DeleteNodeLabelResponse); + rpc SubmitAutomationSuggestion(SubmitAutomationSuggestionRequest) returns (SubmitAutomationSuggestionResponse); + + // Cluster and node management visibility + rpc ListNodes(ListNodesRequest) returns (ListNodesResponse); + rpc GetNode(GetNodeRequest) returns (GetNodeResponse); + rpc ListWorkloads(ListWorkloadsRequest) returns (ListWorkloadsResponse); + rpc GetWorkload(GetWorkloadRequest) returns (GetWorkloadResponse); + rpc GetClusterSummary(GetClusterSummaryRequest) returns (GetClusterSummaryResponse); + + // Cluster-wide events: node joined, node lost, node left, workload + // scheduled, drift detected, retries, reschedules, etc (see + // internal/scheduler/events.go for producers). ListEvents is a + // plain unary call (auto-bridged to REST by persys-gateway's + // reflection-based grpcbridge, no gateway changes needed). WatchEvents + // is a server-streaming call — grpcbridge explicitly does not bridge + // streaming RPCs, so consumers that need HTTP (e.g. a browser + // dashboard) go through a hand-written SSE endpoint on the gateway + // instead of the generic bridge; a gRPC client (e.g. persysctl) can + // call it directly. + rpc ListEvents(ListEventsRequest) returns (ListEventsResponse); + rpc WatchEvents(WatchEventsRequest) returns (stream SchedulerEventView); + + // Optional future streaming channel + rpc ControlStream(stream ControlMessage) returns (stream ControlMessage); + + // Standalone disk inventory (managed volumes) + rpc CreateDisk(CreateDiskRequest) returns (CreateDiskResponse); + rpc ListDisks(ListDisksRequest) returns (ListDisksResponse); + rpc GetDisk(GetDiskRequest) returns (GetDiskResponse); + rpc DeleteDisk(DeleteDiskRequest) returns (DeleteDiskResponse); + + // Object storage (Ceph RGW / S3-compatible buckets) + rpc CreateBucket(CreateBucketRequest) returns (CreateBucketResponse); + rpc ListBuckets(ListBucketsRequest) returns (ListBucketsResponse); + rpc GetBucket(GetBucketRequest) returns (GetBucketResponse); + rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse); + rpc GetBucketAccess(GetBucketAccessRequest) returns (GetBucketAccessResponse); + rpc ListBucketObjects(ListBucketObjectsRequest) returns (ListBucketObjectsResponse); +} + +enum AutomationActionType { + AUTOMATION_ACTION_TYPE_UNSPECIFIED = 0; + AUTOMATION_ACTION_SET_DESIRED_STATE = 1; + AUTOMATION_ACTION_RETRY_WORKLOAD = 2; + AUTOMATION_ACTION_DELETE_WORKLOAD = 3; + AUTOMATION_ACTION_SCALE_REPLICAS = 4; +} + +message AutomationSuggestion { + string suggestion_id = 1; + string policy_id = 2; + string policy_name = 3; + string target_workload = 4; + AutomationActionType action_type = 5; + string desired_state = 6; + int32 desired_replicas = 7; + int32 replica_delta = 8; + string reason = 9; + google.protobuf.Timestamp suggested_at = 10; +} + +message SubmitAutomationSuggestionRequest { + AutomationSuggestion suggestion = 1; +} + +message SubmitAutomationSuggestionResponse { + bool accepted = 1; + string decision = 2; + string reason = 3; + string applied_action = 4; + google.protobuf.Timestamp decided_at = 5; +} + +message RegisterNodeRequest { + string node_id = 1; + NodeCapabilities capabilities = 2; + map labels = 3; + string agent_version = 4; + string grpc_endpoint = 5; + string cluster_id = 6; + google.protobuf.Timestamp timestamp = 7; +} + +message NodeCapabilities { + int64 cpu_total_millicores = 1; + int64 memory_total_mb = 2; + repeated StoragePool storage_pools = 3; + repeated string supported_workload_types = 4; // container, compose, vm + repeated string supported_storage_drivers = 5; // local, nfs, ceph-rbd +} + +message StoragePool { + string name = 1; + string type = 2; // local, nfs, iscsi + int64 total_gb = 3; +} + +message RegisterNodeResponse { + bool accepted = 1; + string reason = 2; + int32 heartbeat_interval_seconds = 3; + google.protobuf.Timestamp lease_expires_at = 4; +} + +message HeartbeatRequest { + string node_id = 1; + NodeUsage usage = 2; + repeated WorkloadStatus workload_statuses = 3; + google.protobuf.Timestamp timestamp = 4; + repeated WorkloadUsageSnapshot workload_usage = 5; +} + +message NodeUsage { + int64 cpu_allocated_millicores = 1; + int64 cpu_used_millicores = 2; + int64 memory_allocated_mb = 3; + int64 memory_used_mb = 4; + int64 disk_allocated_gb = 5; + int64 disk_used_gb = 6; +} + +message HeartbeatResponse { + bool acknowledged = 1; + bool drain_node = 2; + google.protobuf.Timestamp lease_expires_at = 3; +} + +message ApplyWorkloadRequest { + string workload_id = 1; + WorkloadSpec spec = 2; + + // Compatibility fields aligned with existing agent apply semantics. + string revision_id = 10; + string desired_state = 11; // Running | Stopped +} + +message ApplyWorkloadResponse { + bool success = 1; + FailureReason failure_reason = 2; + string error_message = 3; +} + +message DeleteWorkloadRequest { + string workload_id = 1; +} + +message DeleteWorkloadResponse { + bool success = 1; + string error_message = 2; +} + +message WorkloadSpec { + string type = 1; // container, compose, vm + ResourceRequirements resources = 2; + + oneof workload { + ContainerSpec container = 10; + ComposeSpec compose = 11; + VMSpec vm = 12; + } + + map metadata = 20; +} + +message ResourceRequirements { + int64 cpu_millicores = 1; + int64 memory_mb = 2; + int64 disk_gb = 3; +} + +message ContainerSpec { + string image = 1; + repeated string command = 2; + map env = 3; + repeated VolumeMount volumes = 4; + repeated Port ports = 5; + string restart_policy = 6; + bool privileged = 7; + repeated ManagedVolumeSpec managed_volumes = 8; +} + +message VolumeMount { + string host_path = 1; + string container_path = 2; + bool read_only = 3; +} + +message Port { + int32 host_port = 1; + int32 container_port = 2; + string protocol = 3; // tcp or udp +} + +message ComposeSpec { + string source_type = 1; // git or inline + string git_repo = 2; + string git_ref = 3; + string inline_yaml = 4; + map env = 5; +} + +message VMSpec { + int32 vcpus = 1; + int64 memory_mb = 2; + repeated DiskConfig disks = 3; + repeated NetworkConfig networks = 4; + CloudInitConfig cloud_init = 5; + string os_image = 6; + repeated ManagedVolumeSpec managed_volumes = 7; +} + +message DiskConfig { + string pool_name = 1; + int64 size_gb = 2; + string mount_point = 3; +} + +message NetworkConfig { + string bridge = 1; + bool dhcp = 2; + string static_ip = 3; +} + +message CloudInitConfig { + string user_data = 1; + string meta_data = 2; + string network_config = 3; + string vendor_data = 4; +} + +message ManagedVolumeSpec { + string name = 1; + string driver = 2; // local|nfs|ceph-rbd + int64 size_gb = 3; + string access_mode = 4; + string fs_type = 5; + string mount_path = 6; + bool read_only = 7; + string retain_policy = 8; // Delete|Retain +} + +message WorkloadUsageSnapshot { + string workload_id = 1; + string type = 2; + double cpu_percent = 3; + int64 memory_bytes = 4; + int64 disk_read_bytes = 5; + int64 disk_write_bytes = 6; + int64 net_rx_bytes = 7; + int64 net_tx_bytes = 8; + google.protobuf.Timestamp collected_at = 9; + string source = 10; +} + +message ReasonDetail { + string code = 1; + string message = 2; + google.protobuf.Timestamp last_transition = 3; + google.protobuf.Timestamp next_retry_at = 4; + bool retryable = 5; +} + +message WorkloadStatus { + string workload_id = 1; + string state = 2; // Running, Stopped, Failed + FailureReason failure_reason = 3; + string message = 4; + google.protobuf.Timestamp last_transition = 5; + ReasonDetail reason = 6; + WorkloadUsageSnapshot usage = 7; +} + +enum FailureReason { + FAILURE_REASON_UNSPECIFIED = 0; + IMAGE_PULL_FAILED = 1; + IMAGE_NOT_FOUND = 2; + INSUFFICIENT_RESOURCES = 3; + INVALID_SPEC = 4; + RUNTIME_ERROR = 5; + NETWORK_ERROR = 6; + STORAGE_ERROR = 7; + VM_BOOT_FAILED = 8; +} + +message RetryWorkloadRequest { + string workload_id = 1; +} + +message RetryWorkloadResponse { + bool accepted = 1; +} + +message DrainNodeRequest { + string node_id = 1; + string reason = 2; +} + +message DrainNodeResponse { + bool accepted = 1; + string message = 2; + int32 relocated_workloads = 3; + NodeView node = 4; +} + +message UndrainNodeRequest { + string node_id = 1; + string reason = 2; +} + +message UndrainNodeResponse { + bool accepted = 1; + string message = 2; + NodeView node = 3; +} + +message NodeTaint { + string key = 1; + string value = 2; + string effect = 3; // NoSchedule | PreferNoSchedule +} + +message TaintNodeRequest { + string node_id = 1; + NodeTaint taint = 2; +} + +message TaintNodeResponse { + bool accepted = 1; + string message = 2; + NodeView node = 3; +} + +message UntaintNodeRequest { + string node_id = 1; + string key = 2; + string effect = 3; +} + +message UntaintNodeResponse { + bool accepted = 1; + string message = 2; + NodeView node = 3; +} + +message SetNodeLabelRequest { + string node_id = 1; + string key = 2; + string value = 3; +} + +message SetNodeLabelResponse { + bool accepted = 1; + string message = 2; + NodeView node = 3; +} + +message DeleteNodeLabelRequest { + string node_id = 1; + string key = 2; +} + +message DeleteNodeLabelResponse { + bool accepted = 1; + string message = 2; + NodeView node = 3; +} + +message ListNodesRequest { + string status = 1; // optional filter: Ready | NotReady | Draining +} + +message GetNodeRequest { + string node_id = 1; +} + +message ListNodesResponse { + repeated NodeView nodes = 1; +} + +message GetNodeResponse { + NodeView node = 1; +} + +message NodeView { + string node_id = 1; + string status = 2; + string status_reason = 3; + string status_updated_by = 4; + google.protobuf.Timestamp status_updated_at = 5; + google.protobuf.Timestamp last_heartbeat = 6; + string grpc_endpoint = 7; + double total_cpu_cores = 8; + double available_cpu_cores = 9; + int64 total_memory_mb = 10; + int64 available_memory_mb = 11; + repeated string supported_workload_types = 12; + map labels = 13; + repeated NodeTaint taints = 14; +} + +message ListWorkloadsRequest { + string node_id = 1; // optional filter + string status = 2; // optional filter +} + +message SchedulerEventView { + string id = 1; + string type = 2; // e.g. "NodeLost", "WorkloadScheduled", "DriftDetected" + string workload_id = 3; // optional, empty if not workload-scoped + string node_id = 4; // optional, empty if not node-scoped + string reason = 5; + google.protobuf.Timestamp timestamp = 6; + // Free-form auxiliary data. Values are stringified on the way out + // (models.SchedulerEvent.Details is map[string]interface{} on the Go + // side) — this is a deliberate simplification over a + // google.protobuf.Struct, since event details are informational/ + // display-oriented, not structured data a client needs to + // round-trip losslessly. + map details = 7; +} + +message ListEventsRequest { + int64 limit = 1; // 0 means server default + string type = 2; // optional filter + string workload_id = 3; // optional filter + string node_id = 4; // optional filter +} + +message ListEventsResponse { + repeated SchedulerEventView events = 1; +} + +message WatchEventsRequest { + // Same optional filters as ListEventsRequest. The stream first replays + // recent matching events (server-side default limit), then continues + // with new matching events as they're emitted. + string type = 1; + string workload_id = 2; + string node_id = 3; +} + +message GetWorkloadRequest { + string workload_id = 1; +} + +message ListWorkloadsResponse { + repeated WorkloadView workloads = 1; +} + +message GetWorkloadResponse { + WorkloadView workload = 1; +} + +message WorkloadView { + string workload_id = 1; + string type = 2; + string desired_state = 3; + string status = 4; + string assigned_node_id = 5; + string revision_id = 6; + int32 retry_attempts = 7; + int32 retry_max_attempts = 8; + google.protobuf.Timestamp retry_next_at = 9; + string failure_reason = 10; + google.protobuf.Timestamp last_updated = 11; + ReasonDetail reason = 12; + WorkloadUsageSnapshot usage = 13; + google.protobuf.Timestamp created_at = 14; +} + +message GetClusterSummaryRequest {} + +message GetClusterSummaryResponse { + int32 total_nodes = 1; + int32 ready_nodes = 2; + int32 not_ready_nodes = 3; + int32 total_workloads = 4; + int32 running_workloads = 5; + int32 pending_workloads = 6; + int32 failed_workloads = 7; + int32 deleted_workloads = 8; + google.protobuf.Timestamp generated_at = 9; +} + +message ControlMessage { + oneof message { + RegisterNodeRequest register = 1; + HeartbeatRequest heartbeat = 2; + ApplyWorkloadRequest apply = 3; + DeleteWorkloadRequest delete = 4; + } +} + +// --- Standalone disks (control-plane inventory) --- + +message CreateDiskRequest { + string name = 1; + string driver = 2; // local | ceph-rbd | nfs + int64 size_gb = 3; + string fs_type = 4; + string access_mode = 5; + string retain_policy = 6; // Delete | Retain + string node_id = 7; // optional pre-pin for local + string mount_path = 8; +} + +message CreateDiskResponse { + DiskView disk = 1; +} + +message ListDisksRequest {} + +message ListDisksResponse { + repeated DiskView disks = 1; +} + +message GetDiskRequest { + string disk_id = 1; +} + +message GetDiskResponse { + DiskView disk = 1; +} + +message DeleteDiskRequest { + string disk_id = 1; + bool force = 2; +} + +message DeleteDiskResponse { + bool success = 1; + string error_message = 2; +} + +message DiskView { + string id = 1; + string name = 2; + string driver = 3; + int64 size_gb = 4; + string fs_type = 5; + string access_mode = 6; + string retain_policy = 7; + string phase = 8; + string last_error = 9; + string node_id = 10; + string device = 11; + bool standalone = 12; + string mount_path = 13; + repeated string workload_refs = 14; + repeated string attached_nodes = 15; + google.protobuf.Timestamp created_at = 16; + google.protobuf.Timestamp updated_at = 17; +} + +// --- Object storage (Ceph RGW / S3-compatible) --- + +message CreateBucketRequest { + string name = 1; + string region = 2; + bool versioning = 3; +} + +message CreateBucketResponse { + BucketView bucket = 1; + BucketAccess access = 2; // credentials returned once on create +} + +message ListBucketsRequest {} + +message ListBucketsResponse { + repeated BucketView buckets = 1; +} + +message GetBucketRequest { + string bucket_id = 1; // id or name +} + +message GetBucketResponse { + BucketView bucket = 1; +} + +message DeleteBucketRequest { + string bucket_id = 1; + bool force = 2; +} + +message DeleteBucketResponse { + bool success = 1; + string error_message = 2; +} + +message GetBucketAccessRequest { + string bucket_id = 1; +} + +message GetBucketAccessResponse { + BucketAccess access = 1; +} + +message ListBucketObjectsRequest { + string bucket_id = 1; + string prefix = 2; + string continuation_token = 3; + int32 max_keys = 4; +} + +message ListBucketObjectsResponse { + repeated ObjectInfo objects = 1; + string next_continuation_token = 2; + bool is_truncated = 3; + string prefix = 4; +} + +message BucketView { + string id = 1; + string name = 2; + string region = 3; + string owner = 4; + string endpoint = 5; + bool versioning = 6; + int64 object_count = 7; + int64 size_bytes = 8; + string phase = 9; + string last_error = 10; + google.protobuf.Timestamp created_at = 11; + google.protobuf.Timestamp updated_at = 12; +} + +message BucketAccess { + string endpoint = 1; + string region = 2; + string bucket = 3; + string access_key = 4; + string secret_key = 5; + string vault_path = 6; // when secrets live in Vault + string s3_url = 7; // e.g. s3://bucket +} + +message ObjectInfo { + string key = 1; + int64 size_bytes = 2; + string etag = 3; + string last_modified = 4; + string storage_class = 5; +} \ No newline at end of file