feat(postgres): support configurable SSL/TLS connections

Makefile

@@ -33,6 +33,9 @@ LDFLAGS_BUILDDATE := $(LDFLAGS_PACKAGE).BuildDate
 # $(CERT_DIR); the e2e tests then authenticate with them instead of a service
 # token. All paths can be overridden by the caller (e.g. CI).
 CERT_DIR ?= tools/dev/cert
+# Only "require" by default: the dev certificate cannot name the minikube node IP.
+POSTGRES_SSL_MODE ?= require
+POSTGRES_SSL_ROOT_CERT ?= $(CERT_DIR)/postgres-tls.crt
 # Go test binary timeout for the e2e suites. Go's default of 10m is too tight
 # when parallel suites slow down the async event pipeline: it panics the whole
 # binary while slow-but-healthy waits are still within their own retry budgets,
@@ -331,6 +334,8 @@ create-postgres-event-tables:
 		--postgres-user=${POSTGRES_USER} \
 		--postgres-pass=${POSTGRES_PASS} \
 		--postgres-db-name=${POSTGRES_DB_NAME} \
+		--postgres-ssl-mode=${POSTGRES_SSL_MODE} \
+		--postgres-ssl-root-cert=${POSTGRES_SSL_ROOT_CERT} \
 		--no-profile \
 		--no-gcp-trace-enabled \
 		--log-level=debug
@@ -715,6 +720,7 @@ deploy-bucketeer: delete-bucketeer-from-minikube
 	make -C tools/dev service-cert-secret
 	make -C tools/dev service-token-secret
 	make -C tools/dev oauth-key-secret
+	make -C tools/dev postgres-cert-secret
 	make -C ./ build-go-embed
 	make -C ./ pull-dev-images
 	TAG=localenv make -C ./ build-docker-images
@@ -793,6 +799,12 @@ docker-compose-setup:
 	else \
 		echo "docker-compose/secrets directory already exists"; \
 	fi
+	@if [ ! -f "$(CERT_DIR)/postgres-tls.key" ]; then \
+		echo "Generating PostgreSQL TLS certificate..."; \
+		make -C tools/dev generate-postgres-tls-certificate; \
+	else \
+		echo "PostgreSQL TLS certificate already exists"; \
+	fi
 	@echo "Docker Compose setup complete"
 
 .PHONY: docker-compose-init-env
@@ -920,6 +932,7 @@ docker-compose-create-postgres-event-tables:
 	POSTGRES_HOST=localhost \
 	POSTGRES_PORT=5432 \
 	POSTGRES_DB_NAME=bucketeer \
+	POSTGRES_SSL_MODE=verify-full \
 	make -C ./ create-postgres-event-tables
 
 .PHONY: docker-compose-delete-mysql-data-warehouse-data

docker-compose/compose.yml

@@ -33,6 +33,23 @@ services:
     networks:
       - bucketeer
 
+  # PostgreSQL needs the key owned by its own user and not group readable, which
+  # neither a bind mount nor a git clone can guarantee. Runs the same image as
+  # postgres below so the user can be resolved by name rather than by uid.
+  postgres-certs:
+    image: timescale/timescaledb:${POSTGRES_VERSION:-2.25.2-pg18}
+    container_name: bucketeer-postgres-certs
+    network_mode: none
+    entrypoint:
+      - sh
+      - -c
+      - install -o postgres -g postgres -m 600 /src/tls.key /certs/tls.key &&
+        install -o postgres -g postgres -m 644 /src/tls.crt /certs/tls.crt
+    volumes:
+      - ../tools/dev/cert/postgres-tls.crt:/src/tls.crt:ro
+      - ../tools/dev/cert/postgres-tls.key:/src/tls.key:ro
+      - postgres_certs:/certs
+
   # NOTE: If you are switching from the official postgres image to timescaledb (or vice versa),
   # you must remove the existing volume or reset the database before starting, otherwise the
   # container will fail to initialize due to incompatible data directory formats.
@@ -41,6 +58,17 @@ services:
     image: timescale/timescaledb:${POSTGRES_VERSION:-2.25.2-pg18}
     container_name: bucketeer-postgres
     restart: unless-stopped
+    depends_on:
+      postgres-certs:
+        condition: service_completed_successfully
+    # Plaintext keeps working, so tools that set no SSL mode are unaffected.
+    command:
+      - -c
+      - ssl=on
+      - -c
+      - ssl_cert_file=/usr/local/certs/postgres/tls.crt
+      - -c
+      - ssl_key_file=/usr/local/certs/postgres/tls.key
     environment:
       POSTGRES_USER: bucketeer
       POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
@@ -52,6 +80,7 @@ services:
     volumes:
       - postgres_data:/var/lib/postgresql
       - ./init-db/postgres_dump.sql:/docker-entrypoint-initdb.d/postgres_dump.sql
+      - postgres_certs:/usr/local/certs/postgres:ro
     healthcheck:
       test: [ "CMD-SHELL", "export PGPASSWORD=`cat /run/secrets/postgres_password` && pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB" ]
       interval: 10s
@@ -116,6 +145,8 @@ services:
       - BUCKETEER_WEB_POSTGRES_HOST=postgres
       - BUCKETEER_WEB_POSTGRES_PORT=5432
       - BUCKETEER_WEB_POSTGRES_DB_NAME=bucketeer
+      - BUCKETEER_WEB_POSTGRES_SSL_MODE=verify-full
+      - BUCKETEER_WEB_POSTGRES_SSL_ROOT_CERT=/usr/local/certs/postgres/tls.crt
       - BUCKETEER_WEB_PERSISTENT_REDIS_SERVER_NAME=persistent-redis
       - BUCKETEER_WEB_PERSISTENT_REDIS_ADDR=redis:6379
       - BUCKETEER_WEB_PERSISTENT_REDIS_POOL_MAX_IDLE=25
@@ -197,6 +228,7 @@ services:
     volumes:
       - ../tools/dev/cert/tls.crt:/usr/local/certs/service/tls.crt:ro
       - ../tools/dev/cert/tls.key:/usr/local/certs/service/tls.key:ro
+      - ../tools/dev/cert/postgres-tls.crt:/usr/local/certs/postgres/tls.crt:ro
       - ../tools/dev/cert/service-token:/usr/local/service-token/token:ro
       - ../tools/dev/cert/oauth-public.pem:/usr/local/oauth-key/public.pem:ro
       - ../tools/dev/cert/oauth-private.pem:/usr/local/oauth-key/private.pem:ro
@@ -229,6 +261,8 @@ services:
       - BUCKETEER_API_POSTGRES_HOST=postgres
       - BUCKETEER_API_POSTGRES_PORT=5432
       - BUCKETEER_API_POSTGRES_DB_NAME=bucketeer
+      - BUCKETEER_API_POSTGRES_SSL_MODE=verify-full
+      - BUCKETEER_API_POSTGRES_SSL_ROOT_CERT=/usr/local/certs/postgres/tls.crt
       - BUCKETEER_API_GOAL_TOPIC=goal
       - BUCKETEER_API_EVALUATION_TOPIC=evaluation
       - BUCKETEER_API_METRICS_TOPIC=metrics
@@ -280,6 +314,7 @@ services:
     volumes:
       - ../tools/dev/cert/tls.crt:/usr/local/certs/service/tls.crt:ro
       - ../tools/dev/cert/tls.key:/usr/local/certs/service/tls.key:ro
+      - ../tools/dev/cert/postgres-tls.crt:/usr/local/certs/postgres/tls.crt:ro
       - ../tools/dev/cert/service-token:/usr/local/service-token/token:ro
     networks:
       - bucketeer
@@ -309,6 +344,8 @@ services:
       - BUCKETEER_BATCH_POSTGRES_HOST=postgres
       - BUCKETEER_BATCH_POSTGRES_PORT=5432
       - BUCKETEER_BATCH_POSTGRES_DB_NAME=bucketeer
+      - BUCKETEER_BATCH_POSTGRES_SSL_MODE=verify-full
+      - BUCKETEER_BATCH_POSTGRES_SSL_ROOT_CERT=/usr/local/certs/postgres/tls.crt
       - BUCKETEER_BATCH_POSTGRES_DB_OPEN_CONNS=50
       - BUCKETEER_BATCH_PERSISTENT_REDIS_SERVER_NAME=batch-persistent-redis
       - BUCKETEER_BATCH_PERSISTENT_REDIS_ADDR=redis:6379
@@ -361,6 +398,7 @@ services:
     volumes:
       - ../tools/dev/cert/tls.crt:/usr/local/certs/service/tls.crt:ro
       - ../tools/dev/cert/tls.key:/usr/local/certs/service/tls.key:ro
+      - ../tools/dev/cert/postgres-tls.crt:/usr/local/certs/postgres/tls.crt:ro
       - ../tools/dev/cert/service-token:/usr/local/service-token/token:ro
       - ../tools/dev/cert/oauth-public.pem:/usr/local/oauth-key/public.pem:ro
     networks:
@@ -391,6 +429,8 @@ services:
       - BUCKETEER_SUBSCRIBER_POSTGRES_HOST=postgres
       - BUCKETEER_SUBSCRIBER_POSTGRES_PORT=5432
       - BUCKETEER_SUBSCRIBER_POSTGRES_DB_NAME=bucketeer
+      - BUCKETEER_SUBSCRIBER_POSTGRES_SSL_MODE=verify-full
+      - BUCKETEER_SUBSCRIBER_POSTGRES_SSL_ROOT_CERT=/usr/local/certs/postgres/tls.crt
       - BUCKETEER_SUBSCRIBER_POSTGRES_DB_OPEN_CONNS=50
       - BUCKETEER_SUBSCRIBER_PERSISTENT_REDIS_SERVER_NAME=batch-persistent-redis
       - BUCKETEER_SUBSCRIBER_PERSISTENT_REDIS_ADDR=redis:6379
@@ -437,6 +477,7 @@ services:
     volumes:
       - ../tools/dev/cert/tls.crt:/usr/local/certs/service/tls.crt:ro
       - ../tools/dev/cert/tls.key:/usr/local/certs/service/tls.key:ro
+      - ../tools/dev/cert/postgres-tls.crt:/usr/local/certs/postgres/tls.crt:ro
       - ../tools/dev/cert/service-token:/usr/local/service-token/token:ro
       - ../tools/dev/cert/oauth-public.pem:/usr/local/oauth-key/public.pem:ro
       - ./config/subscriber-config:/usr/local/conf:ro
@@ -536,6 +577,8 @@ volumes:
     driver: local
   postgres_data:
     driver: local
+  postgres_certs:
+    driver: local
   redis_data:
     driver: local
   prometheus_data:

docker-compose/config/subscriber-config/onDemandProcessors.json

@@ -20,7 +20,8 @@
         "port": 5432,
         "user": "bucketeer",
         "password": "bucketeer",
-        "database": "bucketeer"
+        "database": "bucketeer",
+        "sslMode": "disable"
       },
       "bigquery": {
         "project": "",
@@ -57,7 +58,8 @@
         "port": 5432,
         "user": "bucketeer",
         "password": "bucketeer",
-        "database": "bucketeer"
+        "database": "bucketeer",
+        "sslMode": "disable"
       },
       "bigquery": {
         "project": "",

hack/create-postgres-event-tables/command.go

@@ -38,11 +38,15 @@ var (
 
 type command struct {
 	*kingpin.CmdClause
-	postgresUser   *string
-	postgresPass   *string
-	postgresHost   *string
-	postgresPort   *int
-	postgresDBName *string
+	postgresUser        *string
+	postgresPass        *string
+	postgresHost        *string
+	postgresPort        *int
+	postgresDBName      *string
+	postgresSSLMode     *string
+	postgresSSLRootCert *string
+	postgresSSLCert     *string
+	postgresSSLKey      *string
 }
 
 func registerCommand(r cli.CommandRegistry, p cli.ParentCommand) *command {
@@ -54,6 +58,22 @@ func registerCommand(r cli.CommandRegistry, p cli.ParentCommand) *command {
 		postgresHost:   cmd.Flag("postgres-host", "PostgreSQL host.").Required().String(),
 		postgresPort:   cmd.Flag("postgres-port", "PostgreSQL port.").Default("5432").Int(),
 		postgresDBName: cmd.Flag("postgres-db-name", "PostgreSQL database name.").Required().String(),
+		postgresSSLMode: cmd.Flag(
+			"postgres-ssl-mode",
+			"PostgreSQL SSL mode (disable, allow, prefer, require, verify-ca, verify-full).",
+		).Default(postgres.SSLModeRequire).String(),
+		postgresSSLRootCert: cmd.Flag(
+			"postgres-ssl-root-cert",
+			"Path to the PostgreSQL SSL root certificate (CA) file.",
+		).String(),
+		postgresSSLCert: cmd.Flag(
+			"postgres-ssl-cert",
+			"Path to the PostgreSQL SSL client certificate file.",
+		).String(),
+		postgresSSLKey: cmd.Flag(
+			"postgres-ssl-key",
+			"Path to the PostgreSQL SSL client private key file.",
+		).String(),
 	}
 	r.RegisterCommand(command)
 	return command
@@ -157,6 +177,12 @@ func (c *command) createPostgresClient(ctx context.Context, logger *zap.Logger)
 		*c.postgresUser, *c.postgresPass, *c.postgresHost,
 		*c.postgresPort,
 		*c.postgresDBName,
+		postgres.WithSSL(postgres.SSLConfig{
+			Mode:     *c.postgresSSLMode,
+			RootCert: *c.postgresSSLRootCert,
+			Cert:     *c.postgresSSLCert,
+			Key:      *c.postgresSSLKey,
+		}),
 		postgres.WithLogger(logger),
 	)
 }

manifests/bucketeer/charts/api/templates/deployment.yaml

@@ -42,6 +42,11 @@ spec:
         - name: service-token-secret
           secret:
             secretName: {{ template "service-token-secret" . }}
+        {{- if .Values.global.operationalDatabase.postgres.sslSecretName }}
+        - name: postgres-cert-secret
+          secret:
+            secretName: {{ .Values.global.operationalDatabase.postgres.sslSecretName }}
+        {{- end }}
       {{- if .Values.serviceAccount.annotations }}
       serviceAccountName: {{ template "api.fullname" . }}
       {{- end }}
@@ -79,6 +84,14 @@ spec:
               value: "{{ .Values.env.operationalDatabase.postgres.port | default .Values.global.operationalDatabase.postgres.port }}"
             - name: BUCKETEER_API_POSTGRES_DB_NAME
               value: "{{ .Values.env.operationalDatabase.postgres.name | default .Values.global.operationalDatabase.postgres.name }}"
+            - name: BUCKETEER_API_POSTGRES_SSL_MODE
+              value: "{{ .Values.env.operationalDatabase.postgres.sslMode | default .Values.global.operationalDatabase.postgres.sslMode }}"
+            - name: BUCKETEER_API_POSTGRES_SSL_ROOT_CERT
+              value: "{{ .Values.env.operationalDatabase.postgres.sslRootCert | default .Values.global.operationalDatabase.postgres.sslRootCert }}"
+            - name: BUCKETEER_API_POSTGRES_SSL_CERT
+              value: "{{ .Values.env.operationalDatabase.postgres.sslCert | default .Values.global.operationalDatabase.postgres.sslCert }}"
+            - name: BUCKETEER_API_POSTGRES_SSL_KEY
+              value: "{{ .Values.env.operationalDatabase.postgres.sslKey | default .Values.global.operationalDatabase.postgres.sslKey }}"
             - name: PUBSUB_EMULATOR_HOST
               value: "{{ .Values.global.pubsub.emulatorHost }}"
             - name: BUCKETEER_API_GOAL_TOPIC
@@ -200,6 +213,11 @@ spec:
             - name: service-token-secret
               mountPath: /usr/local/service-token
               readOnly: true
+            {{- if .Values.global.operationalDatabase.postgres.sslSecretName }}
+            - name: postgres-cert-secret
+              mountPath: /usr/local/certs/postgres
+              readOnly: true
+            {{- end }}
           ports:
             - name: service
               containerPort: {{ .Values.env.port }}

manifests/bucketeer/charts/api/values.yaml

@@ -22,6 +22,10 @@ env:
       host:
       port: 5432
       name:
+      sslMode:
+      sslRootCert:
+      sslCert:
+      sslKey:
   enablePprof: false
   pprofAddr: 127.0.0.1:6060
   pubsubEmulatorHost:

manifests/bucketeer/charts/batch/templates/deployment.yaml

@@ -43,6 +43,11 @@ spec:
         - name: service-token-secret
           secret:
             secretName: {{ template "service-token-secret" . }}
+        {{- if .Values.global.operationalDatabase.postgres.sslSecretName }}
+        - name: postgres-cert-secret
+          secret:
+            secretName: {{ .Values.global.operationalDatabase.postgres.sslSecretName }}
+        {{- end }}
         - name: oauth-key-secret
           secret:
             secretName: {{ template "oauth-key-secret" . }}
@@ -131,6 +136,14 @@ spec:
               value: "{{ .Values.env.operationalDatabase.postgres.port | default .Values.global.operationalDatabase.postgres.port }}"
             - name: BUCKETEER_BATCH_POSTGRES_DB_NAME
               value: "{{ .Values.env.operationalDatabase.postgres.name | default .Values.global.operationalDatabase.postgres.name }}"
+            - name: BUCKETEER_BATCH_POSTGRES_SSL_MODE
+              value: "{{ .Values.env.operationalDatabase.postgres.sslMode | default .Values.global.operationalDatabase.postgres.sslMode }}"
+            - name: BUCKETEER_BATCH_POSTGRES_SSL_ROOT_CERT
+              value: "{{ .Values.env.operationalDatabase.postgres.sslRootCert | default .Values.global.operationalDatabase.postgres.sslRootCert }}"
+            - name: BUCKETEER_BATCH_POSTGRES_SSL_CERT
+              value: "{{ .Values.env.operationalDatabase.postgres.sslCert | default .Values.global.operationalDatabase.postgres.sslCert }}"
+            - name: BUCKETEER_BATCH_POSTGRES_SSL_KEY
+              value: "{{ .Values.env.operationalDatabase.postgres.sslKey | default .Values.global.operationalDatabase.postgres.sslKey }}"
             - name: BUCKETEER_BATCH_POSTGRES_DB_OPEN_CONNS
               value: "{{ .Values.env.operationalDatabase.postgres.openConns | default .Values.global.operationalDatabase.postgres.openConns }}"
             - name: BUCKETEER_BATCH_WEB_URL
@@ -205,6 +218,11 @@ spec:
             - name: service-token-secret
               mountPath: /usr/local/service-token
               readOnly: true
+            {{- if .Values.global.operationalDatabase.postgres.sslSecretName }}
+            - name: postgres-cert-secret
+              mountPath: /usr/local/certs/postgres
+              readOnly: true
+            {{- end }}
             - name: oauth-key-secret
               mountPath: /usr/local/oauth-key
               readOnly: true

manifests/bucketeer/charts/batch/values.yaml

@@ -29,6 +29,11 @@ env:
       port: 5432
       name:
       openConns: 50
+      # Leave empty to inherit global.operationalDatabase.postgres.
+      sslMode:
+      sslRootCert:
+      sslCert:
+      sslKey:
   bigqueryWriterEmulatorHost:
   pubsubEmulatorHost:
   accountService: localhost:9001

manifests/bucketeer/charts/subscriber/templates/deployment.yaml

@@ -46,6 +46,11 @@ spec:
         - name: service-token-secret
           secret:
             secretName: {{ template "service-token-secret" . }}
+        {{- if .Values.global.operationalDatabase.postgres.sslSecretName }}
+        - name: postgres-cert-secret
+          secret:
+            secretName: {{ .Values.global.operationalDatabase.postgres.sslSecretName }}
+        {{- end }}
         - name: subscriber-config
           configMap:
             name: {{ template "subscriber.fullname" . }}-subscribers-config
@@ -122,6 +127,14 @@ spec:
               value: "{{ .Values.env.operationalDatabase.postgres.port | default .Values.global.operationalDatabase.postgres.port }}"
             - name: BUCKETEER_SUBSCRIBER_POSTGRES_DB_NAME
               value: "{{ .Values.env.operationalDatabase.postgres.name | default .Values.global.operationalDatabase.postgres.name }}"
+            - name: BUCKETEER_SUBSCRIBER_POSTGRES_SSL_MODE
+              value: "{{ .Values.env.operationalDatabase.postgres.sslMode | default .Values.global.operationalDatabase.postgres.sslMode }}"
+            - name: BUCKETEER_SUBSCRIBER_POSTGRES_SSL_ROOT_CERT
+              value: "{{ .Values.env.operationalDatabase.postgres.sslRootCert | default .Values.global.operationalDatabase.postgres.sslRootCert }}"
+            - name: BUCKETEER_SUBSCRIBER_POSTGRES_SSL_CERT
+              value: "{{ .Values.env.operationalDatabase.postgres.sslCert | default .Values.global.operationalDatabase.postgres.sslCert }}"
+            - name: BUCKETEER_SUBSCRIBER_POSTGRES_SSL_KEY
+              value: "{{ .Values.env.operationalDatabase.postgres.sslKey | default .Values.global.operationalDatabase.postgres.sslKey }}"
             - name: BUCKETEER_SUBSCRIBER_POSTGRES_DB_OPEN_CONNS
               value: "{{ .Values.env.operationalDatabase.postgres.openConns | default .Values.global.operationalDatabase.postgres.openConns }}"
             - name: BUCKETEER_SUBSCRIBER_WEB_URL
@@ -184,6 +197,11 @@ spec:
             - name: service-token-secret
               mountPath: /usr/local/service-token
               readOnly: true
+            {{- if .Values.global.operationalDatabase.postgres.sslSecretName }}
+            - name: postgres-cert-secret
+              mountPath: /usr/local/certs/postgres
+              readOnly: true
+            {{- end }}
             - name: email-config
               mountPath: /usr/local/email-config
               readOnly: true