feat: system integration rev1 (Vault cert manager + Redis streams + observability)

infra/docker/alloy/alloy-config.alloy

@@ -0,0 +1,14 @@
+pyroscope.scrape "persys_cloud" {
+  targets = [
+    {"__address__" = "persys-scheduler:8084", "service_name" = "persys-cloud.persys-scheduler"},
+    {"__address__" = "compute-agent:8089", "service_name" = "persys-cloud.compute-agent"},
+    {"__address__" = "persys-gateway:6060", "service_name" = "persys-cloud.persys-gateway"},
+  ]
+  forward_to = [pyroscope.write.local.receiver]
+}
+
+pyroscope.write "local" {
+  endpoint {
+    url = "http://pyroscope:4040"
+  }
+}

infra/docker/docker-compose.monitoring.yml

@@ -0,0 +1,108 @@
+# Persys Cloud Monitoring Stack
+
+services:
+  # --- Prometheus (Metrics collection) ---
+  prometheus:
+    image: prom/prometheus:latest
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus_data:/prometheus
+    command:
+      - '--config.file=/etc/prometheus/prometheus.yml'
+      - '--storage.tsdb.path=/prometheus'
+      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
+      - '--web.console.templates=/usr/share/prometheus/consoles'
+    ports:
+      - "9090:9090"
+    networks:
+      - persys-cloud-net
+
+  # --- Grafana (Metrics visualization) ---
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana_container
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - ./grafana/provisioning:/etc/grafana/provisioning
+      - grafana_data:/var/lib/grafana
+    environment:
+      - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin}
+      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
+      - GF_USERS_ALLOW_SIGN_UP=${GF_USERS_ALLOW_SIGN_UP:-false}
+    ports:
+      - "3000:3000"
+    networks:
+      - persys-cloud-net
+    depends_on:
+      - prometheus
+
+  # --- Pyroscope (Profiling) ---
+  pyroscope:
+    image: grafana/pyroscope:latest
+    ports:
+      - "4040:4040"
+    command: ["server"]
+    networks:
+      - persys-cloud-net
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+      - pyroscope_data:/data
+
+  # --- Alloy (Profile Scraping) ---
+  alloy:
+    image: grafana/alloy:latest
+    volumes:
+      - ./alloy/alloy-config.alloy:/etc/alloy/config.alloy
+      - /etc/localtime:/etc/localtime:ro
+    command: ["run", "/etc/alloy/config.alloy"]
+    networks:
+      - persys-cloud-net
+    depends_on:
+      - pyroscope
+
+  # --- Jaeger (Distributed tracing) ---
+  jaeger:
+    image: jaegertracing/all-in-one:latest
+    environment:
+      - COLLECTOR_OTLP_ENABLED=${COLLECTOR_OTLP_ENABLED:-true}
+    ports:
+      - "5775:5775/udp"
+      - "6831:6831/udp"
+      - "6832:6832/udp"
+      - "5778:5778"
+      - "16686:16686"
+      - "14250:14250"
+      - "14268:14268"
+      - "14269:14269"
+      - "4317:4317"
+      - "4318:4318"
+    volumes:
+      - /etc/localtime:/etc/localtime:ro
+    networks:
+      - persys-cloud-net
+
+  # --- Node Exporter (Host metrics for Prometheus) ---
+  node_exporter:
+    container_name: node_exporter_container
+    restart: always
+    image: prom/node-exporter
+    ports:
+      - '9100:9100'
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    networks:
+      - persys-cloud-net
+
+# --- Volumes for persistent data ---
+volumes:
+  pyroscope_data:
+  prometheus_data:
+  grafana_data:
+
+# --- Shared overlay network for all services ---
+networks:
+  persys-cloud-net:
+    driver: bridge

infra/docker/docker-compose.yml

@@ -1,7 +1,7 @@
 # Persys Cloud Full Stack Docker Compose
 #
 # This file defines all core services for local development and integration:
-# - Persys Gateway, Persys Scheduler, Persys Automation, Persys Forgery, Persys Federation, Persys Intelligence,CoreDNS, Hashicorp Vault, MongoDB, MySQL, Postgres, Prometheus, Grafana, Jaeger, etcd, Node Exporter
+# - Persys Gateway, Persys Scheduler, Persys Automation, Persys Forgery, Persys Federation, Persys Intelligence, CoreDNS, Hashicorp Vault, MongoDB, MySQL, Postgres, etcd
 # - All services are connected via the 'persys-cloud-net' bridge network
 
 services:
@@ -12,10 +12,10 @@ services:
     ports:
       - 8551:8551
     depends_on:
+      - vault-manager
       - mongodb
       - vault
       - coredns
-      - jaeger
     environment:
       - OTEL_EXPORTER_JAEGER_ENDPOINT=${OTEL_EXPORTER_JAEGER_ENDPOINT:-jaeger:4318}
       - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-jaeger:4318}
@@ -47,9 +47,8 @@ services:
       - 8085:8085   # gRPC + mTLS API
     depends_on:
       - etcd
-      - vault
+      - vault-manager
       - coredns
-      - jaeger
       - redis
     environment:
       - ETCD_ENDPOINTS=${ETCD_ENDPOINTS:-etcd:2379}
@@ -80,7 +79,7 @@ services:
     ports:
       - 8087:8087
     depends_on:
-      - vault
+      - vault-manager
       - mysql
       - redis
     environment:
@@ -112,7 +111,6 @@ services:
       - 8092:8092
     depends_on:
       - persys-scheduler
-      - prometheus
       - postgres
       - persys-intelligence
     environment:
@@ -165,7 +163,6 @@ services:
       - 8094:8094
     depends_on:
       - persys-scheduler
-      - prometheus
     environment:
       - INTELLIGENCE_HTTP_ADDR=0.0.0.0
       - INTELLIGENCE_HTTP_PORT=8093
@@ -230,11 +227,11 @@ services:
       - /etc/localtime:/etc/localtime:ro
       - ${DOCKER_SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock
     depends_on:
-      - vault
+      - vault-manager
       - persys-scheduler
     ports:
       - "50051:50051"
-      - "8089:8080"
+      - "8089:8089"
     networks:
       - persys-cloud-net
 
@@ -258,7 +255,7 @@ services:
     networks:
       - persys-cloud-net
 
-  # --- Vault Bootstrap (runs once to initialize/provision Vault) ---
+  # --- Vault Manager (Vault Automatic Management) ---
   vault-manager:
     build: ../../vault-manager
     restart: "no"
@@ -290,77 +287,6 @@ services:
     networks:
       - persys-cloud-net
 
-  # --- Prometheus (Metrics collection) ---
-  prometheus:
-    image: prom/prometheus:latest
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-      - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
-      - prometheus_data:/prometheus
-    command:
-      - '--config.file=/etc/prometheus/prometheus.yml'
-      - '--storage.tsdb.path=/prometheus'
-      - '--web.console.libraries=/usr/share/prometheus/console_libraries'
-      - '--web.console.templates=/usr/share/prometheus/consoles'
-    ports:
-      - "9090:9090"
-    networks:
-      - persys-cloud-net
-
-  # --- Grafana (Metrics visualization) ---
-  grafana:
-    image: grafana/grafana:latest
-    container_name: grafana_container
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-      - ./grafana/provisioning:/etc/grafana/provisioning
-      - grafana_data:/var/lib/grafana
-    environment:
-      - GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin}
-      - GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
-      - GF_USERS_ALLOW_SIGN_UP=${GF_USERS_ALLOW_SIGN_UP:-false}
-    ports:
-      - "3000:3000"
-    networks:
-      - persys-cloud-net
-    depends_on:
-      - prometheus
-
-  # --- Jaeger (Distributed tracing) ---
-  jaeger:
-    image: jaegertracing/all-in-one:latest
-    environment:
-      - COLLECTOR_OTLP_ENABLED=${COLLECTOR_OTLP_ENABLED:-true}
-    ports:
-      - "5775:5775/udp"
-      - "6831:6831/udp"
-      - "6832:6832/udp"
-      - "5778:5778"
-      - "16686:16686"
-      - "14250:14250"
-      - "14268:14268"
-      - "14269:14269"
-      - "4317:4317"
-      - "4318:4318"
-    volumes:
-      - /etc/localtime:/etc/localtime:ro
-    networks:
-      - persys-cloud-net
-
-  # --- Node Exporter (Host metrics for Prometheus) ---
-  node_exporter:
-    container_name: node_exporter_container
-    restart: always
-    image: prom/node-exporter
-    ports:
-      - '9100:9100'
-    volumes:
-      - /proc:/host/proc:ro
-      - /sys:/host/sys:ro
-      - /:/rootfs:ro
-    networks:
-      - persys-cloud-net
-
 # --- MySQL (Forgery persistence) ---
   mysql:
     image: mysql:8.0
@@ -444,8 +370,6 @@ volumes:
   mysql_data:
   redis_data:
   automation_postgres_data:
-  prometheus_data:
-  grafana_data:
   vault_data:
   etcd_data:
 

persys-forgery/cmd/main.go

@@ -16,7 +16,7 @@ import (
 	"time"
 
 	"github.com/persys-dev/persys-cloud/persys-forgery/internal/build"
-	"github.com/persys-dev/persys-cloud/persys-forgery/internal/certmanager"
+	"github.com/persys-dev/persys-cloud/pkg/certmanager"
 	"github.com/persys-dev/persys-cloud/persys-forgery/internal/db"
 	forgeryv1 "github.com/persys-dev/persys-cloud/persys-forgery/internal/forgeryv1"
 	"github.com/persys-dev/persys-cloud/persys-forgery/internal/grpcapi"
@@ -63,8 +63,30 @@ func main() {
 	go queue.StartRedisWorker(cfg, orchestrator)
 	go queue.StartWebhookWorker(cfg)
 
-	certMgr, err := certmanager.NewFromConfig(cfg, logrus.New())
-	if err != nil {
+	certcfg := certmanager.Config{
+		TLSEnabled: cfg.TLS.Enabled,
+
+		TLSCertPath: cfg.TLS.CertPath,
+		TLSKeyPath:  cfg.TLS.KeyPath,
+		TLSCAPath:   cfg.TLS.CAPath,
+
+		VaultEnabled:       cfg.Vault.Enabled,
+		VaultManagerAddr:   cfg.Vault.ManagerAddr,
+		VaultAddr:          cfg.Vault.Addr,
+		VaultAuthMethod:    cfg.Vault.AuthMethod,
+		VaultToken:         cfg.Vault.Token,
+		VaultAppRoleID:     cfg.Vault.AppRoleID,
+		VaultAppSecretID:   cfg.Vault.AppSecretID,
+		VaultPKIMount:      cfg.Vault.PKIMount,
+		VaultPKIRole:       cfg.Vault.PKIRole,
+		VaultCertTTL:       cfg.Vault.CertTTL,
+		VaultServiceName:   cfg.Vault.ServiceName,
+		VaultServiceDomain: cfg.Vault.ServiceDomain,
+		VaultRetryInterval: cfg.Vault.RetryInterval,
+	}	
+
+	certMgr := certmanager.NewManager(certcfg, logrus.New())
+	if certMgr == nil {
 		log.Fatalf("failed to initialize cert manager: %v", err)
 	}
 	if err := certMgr.Start(ctx); err != nil {