Deploy Staging #337
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Staging | |
| on: | |
| workflow_run: | |
| workflows: ["CI"] | |
| branches: [staging] | |
| types: [completed] | |
| workflow_dispatch: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| jobs: | |
| deploy: | |
| runs-on: ubuntu-latest | |
| if: >- | |
| (github.event.workflow_run.conclusion == 'success' && | |
| github.event.workflow_run.head_branch == 'staging') || | |
| github.event_name == 'workflow_dispatch' | |
| environment: staging | |
| env: | |
| # Pin to the exact commit that passed CI to prevent version skew. | |
| # workflow_run provides the triggering CI run's commit; workflow_dispatch uses current HEAD. | |
| DEPLOY_SHA: ${{ github.event.workflow_run.head_sha || github.sha }} | |
| steps: | |
| - name: Checkout code at deploy commit | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.DEPLOY_SHA }} | |
| - name: Validate required secrets and variables | |
| run: | | |
| echo "Checking required staging configuration..." | |
| missing="" | |
| if [ -z "${{ secrets.STAGING_HOST }}" ]; then missing="$missing STAGING_HOST"; fi | |
| if [ -z "${{ secrets.STAGING_SSH_USER }}" ]; then missing="$missing STAGING_SSH_USER"; fi | |
| if [ -z "${{ secrets.STAGING_SSH_PRIVATE_KEY }}" ]; then missing="$missing STAGING_SSH_PRIVATE_KEY"; fi | |
| if [ -z "${{ secrets.STAGING_POSTGRES_PASSWORD }}" ]; then missing="$missing STAGING_POSTGRES_PASSWORD"; fi | |
| if [ -z "${{ secrets.STAGING_NEO4J_PASSWORD }}" ]; then missing="$missing STAGING_NEO4J_PASSWORD"; fi | |
| if [ -z "${{ secrets.STAGING_JWT_SECRET }}" ]; then missing="$missing STAGING_JWT_SECRET"; fi | |
| if [ -z "${{ secrets.STAGING_GRAFANA_ADMIN_PASSWORD }}" ]; then missing="$missing STAGING_GRAFANA_ADMIN_PASSWORD"; fi | |
| if [ -z "${{ vars.STAGING_DOMAIN }}" ]; then missing="$missing STAGING_DOMAIN(var)"; fi | |
| if [ -z "${{ vars.STAGING_ACME_EMAIL }}" ]; then missing="$missing STAGING_ACME_EMAIL(var)"; fi | |
| if [ -z "${{ vars.STAGING_ADMIN_DIDS }}" ]; then missing="$missing STAGING_ADMIN_DIDS(var)"; fi | |
| if [ -n "$missing" ]; then | |
| echo "::error::Missing required staging configuration:$missing" | |
| echo "Please configure these in GitHub environment 'staging' settings" | |
| exit 1 | |
| fi | |
| echo "All required staging configuration present" | |
| - name: Create environment files | |
| env: | |
| DOMAIN: ${{ vars.STAGING_DOMAIN }} | |
| ACME_EMAIL: ${{ vars.STAGING_ACME_EMAIL }} | |
| POSTGRES_PASSWORD: ${{ secrets.STAGING_POSTGRES_PASSWORD }} | |
| NEO4J_PASSWORD: ${{ secrets.STAGING_NEO4J_PASSWORD }} | |
| JWT_SECRET: ${{ secrets.STAGING_JWT_SECRET }} | |
| SESSION_SECRET: ${{ secrets.STAGING_SESSION_SECRET }} | |
| GOVERNANCE_PDS_JWT_SECRET: ${{ secrets.STAGING_GOVERNANCE_PDS_JWT_SECRET }} | |
| GOVERNANCE_PDS_ADMIN_PASSWORD: ${{ secrets.STAGING_GOVERNANCE_PDS_ADMIN_PASSWORD }} | |
| GOVERNANCE_PDS_ROTATION_KEY: ${{ secrets.STAGING_GOVERNANCE_PDS_ROTATION_KEY }} | |
| ATPROTO_SERVICE_DID: ${{ vars.STAGING_ATPROTO_SERVICE_DID }} | |
| GRAFANA_ADMIN_PASSWORD: ${{ secrets.STAGING_GRAFANA_ADMIN_PASSWORD }} | |
| ADMIN_DIDS: ${{ vars.STAGING_ADMIN_DIDS }} | |
| ORCID_CLIENT_ID: ${{ secrets.STAGING_ORCID_CLIENT_ID }} | |
| ORCID_CLIENT_SECRET: ${{ secrets.STAGING_ORCID_CLIENT_SECRET }} | |
| run: | | |
| mkdir -p docker-env | |
| cat > docker-env/.env <<EOF | |
| DOMAIN=${DOMAIN} | |
| ACME_EMAIL=${ACME_EMAIL} | |
| CHIVE_VERSION=latest | |
| POSTGRES_DB=chive | |
| POSTGRES_USER=chive | |
| POSTGRES_PASSWORD=${POSTGRES_PASSWORD} | |
| NEO4J_USER=neo4j | |
| NEO4J_PASSWORD=${NEO4J_PASSWORD} | |
| GOVERNANCE_PDS_JWT_SECRET=${GOVERNANCE_PDS_JWT_SECRET} | |
| GOVERNANCE_PDS_ADMIN_PASSWORD=${GOVERNANCE_PDS_ADMIN_PASSWORD} | |
| GOVERNANCE_PDS_ROTATION_KEY=${GOVERNANCE_PDS_ROTATION_KEY} | |
| GRAFANA_ADMIN_PASSWORD=${GRAFANA_ADMIN_PASSWORD} | |
| EOF | |
| cat > docker-env/.env.production <<EOF | |
| DOMAIN=${DOMAIN} | |
| ACME_EMAIL=${ACME_EMAIL} | |
| CHIVE_VERSION=latest | |
| POSTGRES_HOST=postgres | |
| POSTGRES_DB=chive | |
| POSTGRES_USER=chive | |
| POSTGRES_PASSWORD=${POSTGRES_PASSWORD} | |
| NEO4J_USER=neo4j | |
| NEO4J_PASSWORD=${NEO4J_PASSWORD} | |
| REDIS_URL=redis://redis:6379 | |
| JWT_SECRET=${JWT_SECRET} | |
| SESSION_SECRET=${SESSION_SECRET} | |
| GOVERNANCE_PDS_JWT_SECRET=${GOVERNANCE_PDS_JWT_SECRET} | |
| GOVERNANCE_PDS_ADMIN_PASSWORD=${GOVERNANCE_PDS_ADMIN_PASSWORD} | |
| GOVERNANCE_PDS_ROTATION_KEY=${GOVERNANCE_PDS_ROTATION_KEY} | |
| DATABASE_URL=postgresql://chive:${POSTGRES_PASSWORD}@postgres:5432/chive | |
| ELASTICSEARCH_URL=http://elasticsearch:9200 | |
| NEO4J_URI=bolt://neo4j:7687 | |
| NEO4J_ENCRYPTED=ENCRYPTION_OFF | |
| ATPROTO_SERVICE_DID=${ATPROTO_SERVICE_DID} | |
| ATPROTO_RELAY_URL=wss://bsky.network | |
| ADMIN_DIDS=${ADMIN_DIDS} | |
| ORCID_CLIENT_ID=${ORCID_CLIENT_ID} | |
| ORCID_CLIENT_SECRET=${ORCID_CLIENT_SECRET} | |
| ORCID_REDIRECT_URI=https://${DOMAIN}/api/v1/auth/orcid/callback | |
| EOF | |
| sed -i 's/^ //' docker-env/.env docker-env/.env.production | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build and push web frontend image | |
| uses: docker/build-push-action@v6 | |
| with: | |
| context: . | |
| file: web/Dockerfile | |
| push: true | |
| tags: | | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:staging | |
| ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/web:sha-${{ env.DEPLOY_SHA }} | |
| build-args: | | |
| NEXT_PUBLIC_API_URL=https://${{ vars.STAGING_DOMAIN }}/api | |
| NEXT_PUBLIC_FARO_URL=https://faro.${{ vars.STAGING_DOMAIN }}/collect | |
| NEXT_PUBLIC_APP_VERSION=staging-${{ env.DEPLOY_SHA }} | |
| NEXT_PUBLIC_CHIVE_SERVICE_DID=did:web:${{ vars.STAGING_DOMAIN }} | |
| NEXT_PUBLIC_USE_PERMISSION_SETS=true | |
| cache-from: type=gha,scope=web-staging | |
| # `ignore-error=true` because a cache export failure is not a build | |
| # failure. The staging deploy for 0.14.0 built the web image and | |
| # pushed it to ghcr successfully, then died on | |
| # "error writing layer blob: failed to reserve cache" while exporting | |
| # to the Actions cache — so the image existed but staging never | |
| # pulled it, and the branch sat two releases behind with no obvious | |
| # sign why. The cache is an optimisation; losing it should cost time, | |
| # not a deployment. | |
| cache-to: type=gha,mode=max,scope=web-staging,ignore-error=true | |
| - name: Setup Node.js and pnpm for docs build | |
| uses: ./.github/actions/setup-node-pnpm | |
| - name: Build documentation | |
| run: | | |
| pnpm install --frozen-lockfile | |
| pnpm --filter @chive/docs build | |
| - name: Copy docs build to staging server | |
| uses: appleboy/scp-action@v0.1.7 | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| source: "docs/build/*" | |
| target: "/tmp/staging-docs-deploy" | |
| strip_components: 2 | |
| - name: Copy nginx config for docs | |
| uses: appleboy/scp-action@v0.1.7 | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| source: "docker/nginx-docs.conf,docker/docker-compose.staging-docs.yml" | |
| target: "/tmp/staging-docs-config" | |
| strip_components: 1 | |
| - name: Copy env files to staging server | |
| uses: appleboy/scp-action@v0.1.7 | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| source: "docker-env/.env,docker-env/.env.production" | |
| target: "/tmp/chive-deploy" | |
| strip_components: 1 | |
| - name: Deploy to staging server | |
| uses: appleboy/ssh-action@v1.0.3 | |
| env: | |
| REGISTRY: ghcr.io | |
| REPO: ${{ github.repository }} | |
| DEPLOY_SHA: ${{ env.DEPLOY_SHA }} | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| envs: REGISTRY,REPO,DEPLOY_SHA,GH_TOKEN | |
| command_timeout: 30m | |
| script: | | |
| set -e | |
| DEPLOY_DIR="/opt/chive" | |
| REPO_URL="https://github.com/${REPO}.git" | |
| # Pull images by exact commit SHA to prevent version skew between API and web. | |
| # CI tags API images with sha-<7chars> via docker/metadata-action. | |
| # Deploy tags web images with sha-<full_sha> for unambiguous pinning. | |
| SHORT_SHA=$(echo "${DEPLOY_SHA}" | cut -c1-7) | |
| API_IMAGE="${REGISTRY}/${REPO}:sha-${SHORT_SHA}" | |
| WEB_IMAGE="${REGISTRY}/${REPO}/web:sha-${DEPLOY_SHA}" | |
| echo "=== Starting staging deployment ===" | |
| echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" | |
| echo "Deploy SHA: ${DEPLOY_SHA}" | |
| echo "API image: ${API_IMAGE}" | |
| echo "Web image: ${WEB_IMAGE}" | |
| echo "Logging in to container registry..." | |
| echo "${GH_TOKEN}" | docker login "${REGISTRY}" -u github --password-stdin | |
| sudo mkdir -p /opt | |
| sudo chown $USER:$USER /opt 2>/dev/null || true | |
| if [ ! -d "$DEPLOY_DIR" ]; then | |
| git clone "$REPO_URL" "$DEPLOY_DIR" | |
| elif [ ! -d "$DEPLOY_DIR/.git" ]; then | |
| rm -rf "$DEPLOY_DIR" | |
| git clone "$REPO_URL" "$DEPLOY_DIR" | |
| else | |
| cd "$DEPLOY_DIR" | |
| git fetch origin staging | |
| git reset --hard origin/staging | |
| fi | |
| cd "$DEPLOY_DIR" | |
| echo "Current commit: $(git rev-parse --short HEAD) - $(git log -1 --format=%s)" | |
| mkdir -p docker | |
| mv /tmp/chive-deploy/.env docker/.env | |
| mv /tmp/chive-deploy/.env.production docker/.env.production | |
| rm -rf /tmp/chive-deploy | |
| echo "=== Pulling pre-built images ===" | |
| cd docker | |
| # Pull API image by SHA; fall back to :staging tag for workflow_dispatch | |
| if docker pull "$API_IMAGE" 2>/dev/null; then | |
| echo "Pulled API image by SHA: ${API_IMAGE}" | |
| else | |
| echo "SHA-tagged API image not found, falling back to :staging tag" | |
| API_IMAGE="${REGISTRY}/${REPO}:staging" | |
| docker pull "$API_IMAGE" | |
| fi | |
| docker tag "$API_IMAGE" chive:latest | |
| docker pull "$WEB_IMAGE" | |
| docker tag "$WEB_IMAGE" chive-web:latest | |
| echo "Pulling infrastructure images..." | |
| docker compose -f docker-compose.prod.yml pull postgres redis elasticsearch neo4j grobid traefik || true | |
| # Stop all compose services | |
| echo "Stopping all services..." | |
| docker compose -f docker-compose.prod.yml down --remove-orphans 2>/dev/null || true | |
| docker compose -f docker-compose.staging-docs.yml down --remove-orphans 2>/dev/null || true | |
| # Force-remove any leftover chive containers (catches orphans from failed deploys) | |
| echo "Removing any leftover chive containers..." | |
| # `chive-test-*` is the developer test stack (docker/docker-compose.yml). | |
| # It shares the `chive-` prefix, so exclude it by name rather than | |
| # letting this sweep destroy a running test stack on the same host. | |
| docker ps -a --filter "name=chive-" --format '{{.Names}}' \ | |
| | grep -v '^chive-test-' \ | |
| | xargs -r docker rm -f 2>/dev/null || true | |
| # Deploy docs build | |
| echo "Deploying docs build..." | |
| sudo rm -rf /opt/chive/docs/build | |
| sudo mkdir -p /opt/chive/docs | |
| if [ -d /tmp/staging-docs-deploy ]; then | |
| sudo mv /tmp/staging-docs-deploy /opt/chive/docs/build | |
| sudo chown -R root:root /opt/chive/docs/build | |
| sudo chmod -R 755 /opt/chive/docs/build | |
| fi | |
| if [ -d /tmp/staging-docs-config ]; then | |
| cp /tmp/staging-docs-config/nginx-docs.conf /opt/chive/docker/nginx-docs.conf 2>/dev/null || true | |
| cp /tmp/staging-docs-config/docker-compose.staging-docs.yml /opt/chive/docker/docker-compose.staging-docs.yml 2>/dev/null || true | |
| rm -rf /tmp/staging-docs-config | |
| fi | |
| echo "Starting services..." | |
| docker compose -f docker-compose.prod.yml up -d --force-recreate | |
| echo "Starting docs site..." | |
| DOMAIN="${{ vars.STAGING_DOMAIN }}" docker compose -f docker-compose.staging-docs.yml up -d --force-recreate | |
| echo "Waiting for services to initialize..." | |
| sleep 30 | |
| echo "=== Container status ===" | |
| docker compose -f docker-compose.prod.yml ps | |
| echo "Cleaning up old images and build cache..." | |
| docker builder prune -af 2>/dev/null || true | |
| docker system prune -af | |
| echo "=== Staging deployment complete ===" | |
| - name: Setup Elasticsearch Index | |
| uses: appleboy/ssh-action@v1.0.3 | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| command_timeout: 10m | |
| script: | | |
| set -e | |
| cd /opt/chive | |
| docker exec \ | |
| -e ELASTICSEARCH_URL=http://elasticsearch:9200 \ | |
| chive-api \ | |
| node --enable-source-maps dist/scripts/db/recreate-elasticsearch-index.js || { | |
| docker exec \ | |
| -e ELASTICSEARCH_URL=http://elasticsearch:9200 \ | |
| chive-api \ | |
| node --enable-source-maps dist/scripts/db/setup-elasticsearch.js || { | |
| echo "::error::Elasticsearch setup failed" | |
| exit 1 | |
| } | |
| } | |
| - name: Backfill Elasticsearch from Postgres if empty | |
| uses: appleboy/ssh-action@v1.0.3 | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| command_timeout: 30m | |
| script: | | |
| set -e | |
| cd /opt/chive | |
| # Compare Postgres and Elasticsearch eprint counts. When ES is | |
| # behind Postgres — which happens after the setup step recreates | |
| # the index, or when a fresh container comes up against a | |
| # populated Postgres — run the full reindex script. When ES | |
| # already matches (the steady state after any deploy), this is | |
| # a no-op. | |
| PG_COUNT=$(docker exec chive-postgres psql -U chive -d chive -tAc 'SELECT COUNT(*) FROM eprints_index;') | |
| ES_COUNT=$(docker exec chive-elasticsearch curl -s 'http://localhost:9200/eprints/_count' | sed -n 's/.*"count":\([0-9]*\).*/\1/p') | |
| ES_COUNT=${ES_COUNT:-0} | |
| echo "Postgres eprints: $PG_COUNT" | |
| echo "Elasticsearch eprints: $ES_COUNT" | |
| if [ "$PG_COUNT" -gt "$ES_COUNT" ]; then | |
| echo "Elasticsearch is behind Postgres ($ES_COUNT < $PG_COUNT) — running reindex" | |
| # chive-api already has DATABASE_URL, ELASTICSEARCH_URL, | |
| # NEO4J_URI, NEO4J_USER, NEO4J_PASSWORD baked in from | |
| # .env.production, so we only pass overrides needed to use | |
| # in-network hostnames. | |
| docker exec \ | |
| chive-api \ | |
| node --enable-source-maps dist/scripts/reindex-all-eprints.js | |
| else | |
| echo "Elasticsearch is in sync with Postgres — skipping reindex" | |
| fi | |
| - name: Label record nodes and re-match citations | |
| uses: appleboy/ssh-action@v1.0.3 | |
| with: | |
| host: ${{ secrets.STAGING_HOST }} | |
| username: ${{ secrets.STAGING_SSH_USER }} | |
| key: ${{ secrets.STAGING_SSH_PRIVATE_KEY }} | |
| command_timeout: 30m | |
| script: | | |
| set -e | |
| cd /opt/chive | |
| # These steps run on production too. Staging carries them so the | |
| # citation graph can be checked here before it reaches production; | |
| # without them staging's graph stayed empty whatever the code did, | |
| # and the only place to find that out was production. | |
| echo "=== Extracting references never successfully extracted ===" | |
| docker exec \ | |
| chive-api \ | |
| node --enable-source-maps dist/scripts/extract-citations.js --missing || { | |
| echo "::warning::Reference backfill did not complete; it retries next deploy" | |
| } | |
| echo "=== Labelling record nodes ===" | |
| # Always before the citation pass: citation edges attach to | |
| # labelled eprint nodes, and merging one against a label-less node | |
| # carrying the same uri produces a second node rather than reusing | |
| # it. | |
| docker exec \ | |
| chive-api \ | |
| node --enable-source-maps dist/scripts/db/label-unlabelled-record-nodes.js | |
| echo "=== Re-matching citations ===" | |
| docker exec \ | |
| chive-api \ | |
| node --enable-source-maps dist/scripts/db/rematch-citations.js | |
| - name: Verify staging deployment | |
| run: | | |
| echo "Waiting for services to fully start..." | |
| sleep 60 | |
| domain="${{ vars.STAGING_DOMAIN }}" | |
| success=true | |
| echo "Checking API health at https://$domain/api/health..." | |
| if curl -sf --max-time 30 "https://$domain/api/health"; then | |
| echo "API health check passed" | |
| else | |
| echo "API health check failed" | |
| success=false | |
| fi | |
| echo "" | |
| echo "Checking web frontend at https://$domain/..." | |
| if curl -sf --max-time 30 "https://$domain/" > /dev/null; then | |
| echo "Web health check passed" | |
| else | |
| echo "Web health check failed" | |
| success=false | |
| fi | |
| echo "" | |
| echo "Checking docs at https://docs.$domain/..." | |
| if curl -sf --max-time 30 "https://docs.$domain/" > /dev/null; then | |
| echo "Docs health check passed" | |
| else | |
| echo "::warning::Docs health check failed (may need cert provisioning)" | |
| fi | |
| if [ "$success" = "false" ]; then | |
| echo "::error::Staging health checks failed" | |
| exit 1 | |
| fi | |
| echo "All staging health checks passed" | |
| - name: Staging deployment summary | |
| if: always() | |
| run: | | |
| echo "## Staging Deployment" >> $GITHUB_STEP_SUMMARY | |
| echo "" >> $GITHUB_STEP_SUMMARY | |
| echo "**Domain:** ${{ vars.STAGING_DOMAIN }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Commit:** ${{ env.DEPLOY_SHA }}" >> $GITHUB_STEP_SUMMARY | |
| echo "**Timestamp:** $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> $GITHUB_STEP_SUMMARY |