Migrate persistence to PostgreSQL and manage secrets with Vault (#35)

Use PostgreSQL with the jOOQ Open Source Edition to avoid requiring a
commercial license for Oracle dialect support.

Keep service databases and runtime/migration privileges isolated. Add
vendor-specific Flyway schemas, Vault AppRole and scoped application
secrets, dynamic credential pool replacement, and OTP concurrency checks.

Preserve historical Oracle scripts and document cutover, key rotation,
local verification and remaining limitations. Include only migration
work; the separately owned Identity foundation remains a dependency.

Author: Amir-Golmoradi

.dockerignore

@@ -0,0 +1,11 @@
+.git
+.idea
+.agents
+.codex
+.aws
+.ssh
+.env
+.env.*
+.local
+**/target
+**/__pycache__

.env.example

@@ -1,36 +1,7 @@
+# Secrets are generated into ignored protected files by scripts/local/prepare.py.
+# No runtime database passwords or HMAC keys belong here.
 COMPOSE_PROJECT_NAME=payguard
-
-ORACLE_IMAGE=gvenzl/oracle-free:23-slim-faststart
-WALLET_ORACLE_PORT=1521
-LOAN_ORACLE_PORT=1522
-COLLATERAL_ORACLE_PORT=1523
-WALLET_ORACLE_SYSTEM_PASSWORD=change-wallet-system-password
-LOAN_ORACLE_SYSTEM_PASSWORD=change-loan-system-password
-COLLATERAL_ORACLE_SYSTEM_PASSWORD=change-collateral-system-password
-WALLET_ORACLE_APP_USER=payguard_wallet
-LOAN_ORACLE_APP_USER=payguard_loan
-COLLATERAL_ORACLE_APP_USER=payguard_collateral
-WALLET_ORACLE_APP_PASSWORD=change-wallet-app-password
-LOAN_ORACLE_APP_PASSWORD=change-loan-app-password
-COLLATERAL_ORACLE_APP_PASSWORD=change-collateral-app-password
-WALLET_ORACLE_JDBC_URL=jdbc:oracle:thin:@localhost:1521/FREE
-LOAN_ORACLE_JDBC_URL=jdbc:oracle:thin:@localhost:1522/FREE
-COLLATERAL_ORACLE_JDBC_URL=jdbc:oracle:thin:@localhost:1523/FREE
-OTP_ORACLE_PORT=1524
-OTP_ORACLE_SYSTEM_PASSWORD=change-otp-system-password
-OTP_ORACLE_APP_USER=payguard_otp
-OTP_ORACLE_APP_PASSWORD=change-otp-app-password
-OTP_ORACLE_JDBC_URL=jdbc:oracle:thin:@localhost:1524/FREE
-OTP_SECURITY_PEPPER=replace-with-a-random-secret-at-least-32-characters
-OTP_SERVICE_PORT=8084
-REDIS_HOST=localhost
-REDIS_PORT=6379
-
-WALLET_SERVICE_PORT=8081
-LOAN_SERVICE_PORT=8082
-COLLATERAL_SERVICE_PORT=8083
-
-KAFKA_IMAGE=apache/kafka:latest
+VAULT_PORT=8200
 KAFKA_PORT=9092
-KAFKA_CLUSTER_ID=MkU3OEVBNTcwNTJENDM2Qk
-KAFKA_BOOTSTRAP_SERVERS=localhost:9092
+PAYGUARD_LOCAL_UID=1000
+PAYGUARD_LOCAL_GID=1000

.github/workflows/_database-security.yml

@@ -0,0 +1,55 @@
+name: PostgreSQL and Vault verification
+
+on:
+  workflow_call:
+
+permissions:
+  contents: read
+
+jobs:
+  real-dependencies:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-java@v6
+        with:
+          distribution: temurin
+          java-version: '25'
+          cache: maven
+      - name: PostgreSQL migrations and competing transactions
+        run: >-
+          ./mvnw -B -ntp -pl payguard-identity-service,payguard-loan-service,payguard-collateral-service -am
+          -Dtest=PostgreSqlTlsTest,PostgresqlMigrationRehearsalTest,SanitizedDependencyErrorTest,OtpInfrastructureIntegrationTest,OtpApplicationContractsTest,LoanServiceApplicationTests,CollateralServiceApplicationTests
+          -Dsurefire.failIfNoSpecifiedTests=false clean test
+      - name: Build protected migration and integration runners
+        run: >-
+          ./mvnw -B -ntp -pl payguard-identity-service,payguard-wallet-service,payguard-loan-service,payguard-collateral-service -am -DskipTests package dependency:copy-dependencies -DincludeScope=test
+      - name: Verify synthetic export and approved identifier mapping
+        run: python3 -m unittest discover -s scripts/migration -p 'test_*.py'
+      - name: Prepare isolated TLS infrastructure
+        run: |
+          python3 scripts/local/prepare.py
+          docker compose --env-file .env.example up -d --wait postgres-wallet postgres-loan postgres-collateral postgres-identity vault
+          python3 scripts/local/vault_bootstrap.py
+      - name: Apply separately privileged Flyway migrations
+        run: |
+          python3 scripts/local/migrate.py identity
+          python3 scripts/local/migrate.py loan
+          python3 scripts/local/migrate.py collateral
+      - name: Verify actual pools, scoped secrets and Vault audit
+        run: python3 scripts/local/verify_vault.py
+      - name: Boot actual packaged services with scoped credentials
+        run: python3 scripts/local/verify_applications.py
+      - name: Preserve test reports
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: database-security-reports
+          path: |
+            **/target/surefire-reports/TEST-*.xml
+            payguard-shared/target/vault-test-reports/*.xml
+          if-no-files-found: warn
+      - name: Stop isolated infrastructure
+        if: always()
+        run: docker compose --env-file .env.example stop

.github/workflows/_publish.yml

@@ -34,10 +34,13 @@ jobs:
             | grep -A8 -B2 spring-boot-maven-plugin
 
       - name: Build Wallet Service Image
-        run: docker build -f payguard-wallet-service/Dockerfile -t payguard/wallet-service:latest .
+        run: docker build -f payguard-wallet-service/Dockerfile -t payguard/wallet-service:${{ github.sha }} .
 
       - name: Build Loan Service Image
-        run: docker build -f payguard-loan-service/Dockerfile -t payguard/loan-service:latest .
+        run: docker build -f payguard-loan-service/Dockerfile -t payguard/loan-service:${{ github.sha }} .
 
       - name: Build Collateral Service Image
-        run: docker build -f payguard-collateral-service/Dockerfile -t payguard/collateral-service:latest .
+        run: docker build -f payguard-collateral-service/Dockerfile -t payguard/collateral-service:${{ github.sha }} .
+
+      - name: Build Identity Service Image
+        run: docker build -f payguard-identity-service/Dockerfile -t payguard/identity-service:${{ github.sha }} .

.github/workflows/pipeline.yml

@@ -7,12 +7,15 @@ on:
     types: [ opened, reopened, synchronize ]
 
 jobs:
+  database-security:
+    uses: ./.github/workflows/_database-security.yml
+
   # 1. Always run quality checks on PRs and main branch pushes
   quality-check:
     uses: ./.github/workflows/_quality.yml
 
   # 2. Build container images on every PR and main push.
   # This job validates images; it does not publish to an external registry.
   publish-docker:
-    needs: quality-check
+    needs: [quality-check, database-security]
     uses: ./.github/workflows/_publish.yml

.gitignore

@@ -8,3 +8,8 @@ target/
 
 # Local workflow guidance
 GIT_WORKFLOW_CONVENTION.md
+PAYGUARD_GOAL.md
+
+# Protected local PostgreSQL/Vault rehearsal artifacts
+.local/
+__pycache__/