Skip to content

Commit 4026348

Browse files
miladhzzzzmiladhzzzz
authored
Merge pull request #27 from persys-dev/system-integration-rev1
feat: system integration rev1 (Vault cert manager + Redis streams + observability)
2 parents 9cb9482 + 2e0cc73 commit 4026348

34 files changed

Lines changed: 1201 additions & 431 deletions

File tree

‎compute-agent‎

‎infra/docker/alloy/alloy-config.alloy‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
pyroscope.scrape "persys_cloud" {
2+
targets = [
3+
{"__address__" = "persys-scheduler:8084", "service_name" = "persys-cloud.persys-scheduler"},
4+
{"__address__" = "compute-agent:8089", "service_name" = "persys-cloud.compute-agent"},
5+
{"__address__" = "persys-gateway:6060", "service_name" = "persys-cloud.persys-gateway"},
6+
]
7+
forward_to = [pyroscope.write.local.receiver]
8+
}
9+
10+
pyroscope.write "local" {
11+
endpoint {
12+
url = "http://pyroscope:4040"
13+
}
14+
}

‎infra/docker/docker-compose.monitoring.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# Persys Cloud Monitoring Stack
2+
3+
services:
4+
# --- Prometheus (Metrics collection) ---
5+
prometheus:
6+
image: prom/prometheus:latest
7+
volumes:
8+
- /etc/localtime:/etc/localtime:ro
9+
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
10+
- prometheus_data:/prometheus
11+
command:
12+
- '--config.file=/etc/prometheus/prometheus.yml'
13+
- '--storage.tsdb.path=/prometheus'
14+
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
15+
- '--web.console.templates=/usr/share/prometheus/consoles'
16+
ports:
17+
- "9090:9090"
18+
networks:
19+
- persys-cloud-net
20+
21+
# --- Grafana (Metrics visualization) ---
22+
grafana:
23+
image: grafana/grafana:latest
24+
container_name: grafana_container
25+
volumes:
26+
- /etc/localtime:/etc/localtime:ro
27+
- ./grafana/provisioning:/etc/grafana/provisioning
28+
- grafana_data:/var/lib/grafana
29+
environment:
30+
- GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin}
31+
- GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
32+
- GF_USERS_ALLOW_SIGN_UP=${GF_USERS_ALLOW_SIGN_UP:-false}
33+
ports:
34+
- "3000:3000"
35+
networks:
36+
- persys-cloud-net
37+
depends_on:
38+
- prometheus
39+
40+
# --- Pyroscope (Profiling) ---
41+
pyroscope:
42+
image: grafana/pyroscope:latest
43+
ports:
44+
- "4040:4040"
45+
command: ["server"]
46+
networks:
47+
- persys-cloud-net
48+
volumes:
49+
- /etc/localtime:/etc/localtime:ro
50+
- pyroscope_data:/data
51+
52+
# --- Alloy (Profile Scraping) ---
53+
alloy:
54+
image: grafana/alloy:latest
55+
volumes:
56+
- ./alloy/alloy-config.alloy:/etc/alloy/config.alloy
57+
- /etc/localtime:/etc/localtime:ro
58+
command: ["run", "/etc/alloy/config.alloy"]
59+
networks:
60+
- persys-cloud-net
61+
depends_on:
62+
- pyroscope
63+
64+
# --- Jaeger (Distributed tracing) ---
65+
jaeger:
66+
image: jaegertracing/all-in-one:latest
67+
environment:
68+
- COLLECTOR_OTLP_ENABLED=${COLLECTOR_OTLP_ENABLED:-true}
69+
ports:
70+
- "5775:5775/udp"
71+
- "6831:6831/udp"
72+
- "6832:6832/udp"
73+
- "5778:5778"
74+
- "16686:16686"
75+
- "14250:14250"
76+
- "14268:14268"
77+
- "14269:14269"
78+
- "4317:4317"
79+
- "4318:4318"
80+
volumes:
81+
- /etc/localtime:/etc/localtime:ro
82+
networks:
83+
- persys-cloud-net
84+
85+
# --- Node Exporter (Host metrics for Prometheus) ---
86+
node_exporter:
87+
container_name: node_exporter_container
88+
restart: always
89+
image: prom/node-exporter
90+
ports:
91+
- '9100:9100'
92+
volumes:
93+
- /proc:/host/proc:ro
94+
- /sys:/host/sys:ro
95+
- /:/rootfs:ro
96+
networks:
97+
- persys-cloud-net
98+
99+
# --- Volumes for persistent data ---
100+
volumes:
101+
pyroscope_data:
102+
prometheus_data:
103+
grafana_data:
104+
105+
# --- Shared overlay network for all services ---
106+
networks:
107+
persys-cloud-net:
108+
driver: bridge

‎infra/docker/docker-compose.yml‎

Lines changed: 7 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Persys Cloud Full Stack Docker Compose
22
#
33
# This file defines all core services for local development and integration:
4-
# - Persys Gateway, Persys Scheduler, Persys Automation, Persys Forgery, Persys Federation, Persys Intelligence,CoreDNS, Hashicorp Vault, MongoDB, MySQL, Postgres, Prometheus, Grafana, Jaeger, etcd, Node Exporter
4+
# - Persys Gateway, Persys Scheduler, Persys Automation, Persys Forgery, Persys Federation, Persys Intelligence, CoreDNS, Hashicorp Vault, MongoDB, MySQL, Postgres, etcd
55
# - All services are connected via the 'persys-cloud-net' bridge network
66

77
services:
@@ -12,10 +12,10 @@ services:
1212
ports:
1313
- 8551:8551
1414
depends_on:
15+
- vault-manager
1516
- mongodb
1617
- vault
1718
- coredns
18-
- jaeger
1919
environment:
2020
- OTEL_EXPORTER_JAEGER_ENDPOINT=${OTEL_EXPORTER_JAEGER_ENDPOINT:-jaeger:4318}
2121
- OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-jaeger:4318}
@@ -47,9 +47,8 @@ services:
4747
- 8085:8085 # gRPC + mTLS API
4848
depends_on:
4949
- etcd
50-
- vault
50+
- vault-manager
5151
- coredns
52-
- jaeger
5352
- redis
5453
environment:
5554
- ETCD_ENDPOINTS=${ETCD_ENDPOINTS:-etcd:2379}
@@ -80,7 +79,7 @@ services:
8079
ports:
8180
- 8087:8087
8281
depends_on:
83-
- vault
82+
- vault-manager
8483
- mysql
8584
- redis
8685
environment:
@@ -112,7 +111,6 @@ services:
112111
- 8092:8092
113112
depends_on:
114113
- persys-scheduler
115-
- prometheus
116114
- postgres
117115
- persys-intelligence
118116
environment:
@@ -165,7 +163,6 @@ services:
165163
- 8094:8094
166164
depends_on:
167165
- persys-scheduler
168-
- prometheus
169166
environment:
170167
- INTELLIGENCE_HTTP_ADDR=0.0.0.0
171168
- INTELLIGENCE_HTTP_PORT=8093
@@ -230,11 +227,11 @@ services:
230227
- /etc/localtime:/etc/localtime:ro
231228
- ${DOCKER_SOCK_PATH:-/var/run/docker.sock}:/var/run/docker.sock
232229
depends_on:
233-
- vault
230+
- vault-manager
234231
- persys-scheduler
235232
ports:
236233
- "50051:50051"
237-
- "8089:8080"
234+
- "8089:8089"
238235
networks:
239236
- persys-cloud-net
240237

@@ -258,7 +255,7 @@ services:
258255
networks:
259256
- persys-cloud-net
260257

261-
# --- Vault Bootstrap (runs once to initialize/provision Vault) ---
258+
# --- Vault Manager (Vault Automatic Management) ---
262259
vault-manager:
263260
build: ../../vault-manager
264261
restart: "no"
@@ -290,77 +287,6 @@ services:
290287
networks:
291288
- persys-cloud-net
292289

293-
# --- Prometheus (Metrics collection) ---
294-
prometheus:
295-
image: prom/prometheus:latest
296-
volumes:
297-
- /etc/localtime:/etc/localtime:ro
298-
- ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
299-
- prometheus_data:/prometheus
300-
command:
301-
- '--config.file=/etc/prometheus/prometheus.yml'
302-
- '--storage.tsdb.path=/prometheus'
303-
- '--web.console.libraries=/usr/share/prometheus/console_libraries'
304-
- '--web.console.templates=/usr/share/prometheus/consoles'
305-
ports:
306-
- "9090:9090"
307-
networks:
308-
- persys-cloud-net
309-
310-
# --- Grafana (Metrics visualization) ---
311-
grafana:
312-
image: grafana/grafana:latest
313-
container_name: grafana_container
314-
volumes:
315-
- /etc/localtime:/etc/localtime:ro
316-
- ./grafana/provisioning:/etc/grafana/provisioning
317-
- grafana_data:/var/lib/grafana
318-
environment:
319-
- GF_SECURITY_ADMIN_USER=${GF_SECURITY_ADMIN_USER:-admin}
320-
- GF_SECURITY_ADMIN_PASSWORD=${GF_SECURITY_ADMIN_PASSWORD:-admin}
321-
- GF_USERS_ALLOW_SIGN_UP=${GF_USERS_ALLOW_SIGN_UP:-false}
322-
ports:
323-
- "3000:3000"
324-
networks:
325-
- persys-cloud-net
326-
depends_on:
327-
- prometheus
328-
329-
# --- Jaeger (Distributed tracing) ---
330-
jaeger:
331-
image: jaegertracing/all-in-one:latest
332-
environment:
333-
- COLLECTOR_OTLP_ENABLED=${COLLECTOR_OTLP_ENABLED:-true}
334-
ports:
335-
- "5775:5775/udp"
336-
- "6831:6831/udp"
337-
- "6832:6832/udp"
338-
- "5778:5778"
339-
- "16686:16686"
340-
- "14250:14250"
341-
- "14268:14268"
342-
- "14269:14269"
343-
- "4317:4317"
344-
- "4318:4318"
345-
volumes:
346-
- /etc/localtime:/etc/localtime:ro
347-
networks:
348-
- persys-cloud-net
349-
350-
# --- Node Exporter (Host metrics for Prometheus) ---
351-
node_exporter:
352-
container_name: node_exporter_container
353-
restart: always
354-
image: prom/node-exporter
355-
ports:
356-
- '9100:9100'
357-
volumes:
358-
- /proc:/host/proc:ro
359-
- /sys:/host/sys:ro
360-
- /:/rootfs:ro
361-
networks:
362-
- persys-cloud-net
363-
364290
# --- MySQL (Forgery persistence) ---
365291
mysql:
366292
image: mysql:8.0
@@ -444,8 +370,6 @@ volumes:
444370
mysql_data:
445371
redis_data:
446372
automation_postgres_data:
447-
prometheus_data:
448-
grafana_data:
449373
vault_data:
450374
etcd_data:
451375

‎persys-forgery/cmd/main.go‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
"time"
1717

1818
"github.com/persys-dev/persys-cloud/persys-forgery/internal/build"
19-
"github.com/persys-dev/persys-cloud/persys-forgery/internal/certmanager"
19+
"github.com/persys-dev/persys-cloud/pkg/certmanager"
2020
"github.com/persys-dev/persys-cloud/persys-forgery/internal/db"
2121
forgeryv1 "github.com/persys-dev/persys-cloud/persys-forgery/internal/forgeryv1"
2222
"github.com/persys-dev/persys-cloud/persys-forgery/internal/grpcapi"
@@ -63,8 +63,30 @@ func main() {
6363
go queue.StartRedisWorker(cfg, orchestrator)
6464
go queue.StartWebhookWorker(cfg)
6565

66-
certMgr, err := certmanager.NewFromConfig(cfg, logrus.New())
67-
if err != nil {
66+
certcfg := certmanager.Config{
67+
TLSEnabled: cfg.TLS.Enabled,
68+
69+
TLSCertPath: cfg.TLS.CertPath,
70+
TLSKeyPath: cfg.TLS.KeyPath,
71+
TLSCAPath: cfg.TLS.CAPath,
72+
73+
VaultEnabled: cfg.Vault.Enabled,
74+
VaultManagerAddr: cfg.Vault.ManagerAddr,
75+
VaultAddr: cfg.Vault.Addr,
76+
VaultAuthMethod: cfg.Vault.AuthMethod,
77+
VaultToken: cfg.Vault.Token,
78+
VaultAppRoleID: cfg.Vault.AppRoleID,
79+
VaultAppSecretID: cfg.Vault.AppSecretID,
80+
VaultPKIMount: cfg.Vault.PKIMount,
81+
VaultPKIRole: cfg.Vault.PKIRole,
82+
VaultCertTTL: cfg.Vault.CertTTL,
83+
VaultServiceName: cfg.Vault.ServiceName,
84+
VaultServiceDomain: cfg.Vault.ServiceDomain,
85+
VaultRetryInterval: cfg.Vault.RetryInterval,
86+
}
87+
88+
certMgr := certmanager.NewManager(certcfg, logrus.New())
89+
if certMgr == nil {
6890
log.Fatalf("failed to initialize cert manager: %v", err)
6991
}
7092
if err := certMgr.Start(ctx); err != nil {

0 commit comments

Comments
 (0)