Deploy DSpace #2462
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # DSpace Docker deploy on dataquest servers | |
| name: Deploy DSpace | |
| on: | |
| workflow_call: | |
| inputs: | |
| DEV_MACHINE: | |
| required: false | |
| type: string | |
| default: 'dev-6' | |
| INSTANCE: | |
| required: false | |
| type: string | |
| default: '8603' | |
| IMPORT: | |
| required: false | |
| default: false | |
| type: boolean | |
| ERASE_DB: | |
| required: false | |
| default: false | |
| type: boolean | |
| DUMP_PATH: | |
| required: false | |
| type: string | |
| default: '/opt/dspace-envs/8603/dump/dspace_dev5_dump_26.07.16.sql' | |
| ASSETSTORE_PATH: | |
| required: false | |
| type: string | |
| default: '/opt/dspace-envs/8603/assetstore' | |
| workflow_dispatch: | |
| inputs: | |
| INSTANCE: | |
| required: true | |
| default: '8603' | |
| type: choice | |
| options: | |
| - '8603' | |
| IMPORT: | |
| required: true | |
| default: true | |
| type: boolean | |
| ERASE_DB: | |
| required: false | |
| default: false | |
| type: boolean | |
| DEV_MACHINE: | |
| required: false | |
| type: string | |
| default: 'dev-6' | |
| DUMP_PATH: | |
| description: 'Absolute path of the SQL dump ON THE DEV MACHINE (used only when IMPORT is checked)' | |
| required: false | |
| type: string | |
| default: '/opt/dspace-envs/8603/dump/dspace_dev5_dump_26.07.16.sql' | |
| ASSETSTORE_PATH: | |
| description: 'Absolute path of the assetstore directory ON THE DEV MACHINE' | |
| required: false | |
| type: string | |
| default: '/opt/dspace-envs/8603/assetstore' | |
| PROBE_ONLY: | |
| description: 'Only report the state of the target machine; deploy and import nothing' | |
| required: false | |
| default: false | |
| type: boolean | |
| jobs: | |
| # Read-only report of the machine we are about to touch. Runs first so that a | |
| # misconfigured target is visible in the log even when the deploy then fails. | |
| probe: | |
| runs-on: dspace-${{inputs.DEV_MACHINE}}-dep-1 | |
| timeout-minutes: 10 | |
| env: | |
| INSTANCE: ${{inputs.INSTANCE}} | |
| CONFIG_PATH: /opt/dspace-envs/${{inputs.INSTANCE}} | |
| ENVFILE: /opt/dspace-envs/${{inputs.INSTANCE}}/.env | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: config dir + dump | |
| run: | | |
| echo "===== $CONFIG_PATH" | |
| ls -la "$CONFIG_PATH/" || echo "MISSING: $CONFIG_PATH" | |
| echo "===== $CONFIG_PATH/dump" | |
| ls -la "$CONFIG_PATH/dump/" || echo "MISSING: $CONFIG_PATH/dump" | |
| echo "===== dump referenced by DUMP_PATH input" | |
| ls -la "${{ inputs.DUMP_PATH }}" || echo "MISSING dump: ${{ inputs.DUMP_PATH }}" | |
| echo "===== port-ish keys from .env (values shown; these are ports, not secrets)" | |
| grep -E '^(PG_PORT|SOLR_PORT|DSPACE_PORT|DSPACE_FE_PORT|DSPACE_REST_PORT|DSPACE_DOCKER_REST_PORT|DSPACE_VER|DSPACE_HOST)=' "$ENVFILE" || echo "no port keys found" | |
| - name: who owns the host ports | |
| run: | | |
| echo "===== all dspace-ish containers and their published ports" | |
| docker ps -a --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' | grep -Ei 'dspace|postgres|solr' || echo "none" | |
| echo "===== anything publishing host port 5432" | |
| docker ps --format '{{.Names}}\t{{.Ports}}' | grep -E '(^|[^0-9])5432->' || echo "nothing publishes 5432" | |
| echo "===== host listeners on 5432" | |
| (ss -lntp 2>/dev/null || netstat -lntp 2>/dev/null) | grep -E ':5432' || echo "no listener on 5432 (or no permission to see it)" | |
| - name: resolved compose ports for this instance | |
| run: | | |
| echo "===== resolved ports for this instance (this is what deploy uses)" | |
| docker compose --env-file "$ENVFILE" -p dspace-$INSTANCE \ | |
| -f docker/docker-compose.yml -f docker/docker-compose-rest.yml \ | |
| -f "$CONFIG_PATH/docker-compose-rest.yml" -f "$CONFIG_PATH/docker-compose.yml" \ | |
| config 2>&1 | grep -nE '^ [a-z-]+:|container_name:|published:|^\s+ports:' | head -40 \ | |
| || echo "compose config failed" | |
| # The overlay must REPLACE the fixed ports the vanilla files publish, not add to | |
| # them: compose merges `ports` lists across -f files. If a fixed port survives the | |
| # merge above, the overlay is missing `ports: !override` and this instance will | |
| # fight whichever one already owns it on the host. | |
| echo "===== vanilla files alone, for comparison (the fixed ports the overlay must replace)" | |
| docker compose --env-file "$ENVFILE" -p dspace-$INSTANCE \ | |
| -f docker/docker-compose.yml -f docker/docker-compose-rest.yml \ | |
| config 2>&1 | grep -nE '^ [a-z-]+:|container_name:|published:|^\s+ports:' | head -30 \ | |
| || echo "compose config failed" | |
| echo "===== verdict" | |
| if docker compose --env-file "$ENVFILE" -p dspace-$INSTANCE \ | |
| -f docker/docker-compose.yml -f docker/docker-compose-rest.yml \ | |
| -f "$CONFIG_PATH/docker-compose-rest.yml" -f "$CONFIG_PATH/docker-compose.yml" \ | |
| config 2>/dev/null | grep -qE 'published: "(5432|8080|8983|4000|9876)"'; then | |
| echo " WARNING: overlay still inherits vanilla fixed ports." | |
| echo " Add 'ports: !override' to $CONFIG_PATH/docker-compose.yml and docker-compose-rest.yml," | |
| echo " otherwise the deploy will fail with 'port is already allocated'." | |
| else | |
| echo " OK: only instance-scoped ports are published" | |
| fi | |
| - name: postgres inside the instance db container | |
| run: | | |
| docker exec dspacedb$INSTANCE sh -c 'echo "PGPORT=$PGPORT"; pg_isready -U dspace; psql -U dspace -d postgres -tAc "SHOW port;" 2>/dev/null' || echo "could not probe dspacedb$INSTANCE" | |
| # Does the instance actually hold the imported data and serve it? A green import | |
| # job only proves the commands exited 0. | |
| - name: data + service health | |
| run: | | |
| # DSpace keeps Flyway's history in "schema_version", not Flyway's default | |
| # "flyway_schema_history" -- see DatabaseUtils.FLYWAY_TABLE. | |
| echo "===== last applied migrations (schema_version)" | |
| docker exec dspacedb$INSTANCE psql -U dspace -d dspace -tAc \ | |
| "select version, script, success from schema_version order by installed_rank desc limit 5;" \ | |
| 2>&1 || echo "could not read schema_version" | |
| # What `database repair` did to the imported CLARIN history. repair only rewrites | |
| # this bookkeeping table -- it never touches user tables or their rows. | |
| echo "===== CLARIN 7.6 entries / anything repair marked deleted" | |
| docker exec dspacedb$INSTANCE psql -U dspace -d dspace -tAc \ | |
| "select installed_rank||' | '||version||' | type='||type||' | success='||success||' | '||script from schema_version where version like '7.6%' or type='DELETE' order by installed_rank desc limit 12;" \ | |
| 2>&1 || echo "could not read schema_version" | |
| echo "===== CLARIN-specific tables still present after repair" | |
| docker exec dspacedb$INSTANCE psql -U dspace -d dspace -tAc \ | |
| "select 'public tables total='||count(*) from information_schema.tables where table_schema='public';" 2>&1 || true | |
| docker exec dspacedb$INSTANCE psql -U dspace -d dspace -tAc \ | |
| "select string_agg(table_name, ', ') from information_schema.tables where table_schema='public' and (table_name like '%clarin%' or table_name like '%license%');" 2>&1 || true | |
| echo "===== row counts" | |
| docker exec dspacedb$INSTANCE psql -U dspace -d dspace -tAc \ | |
| "select 'item='||(select count(*) from item) || ' bitstream='||(select count(*) from bitstream) || ' collection='||(select count(*) from collection) || ' community='||(select count(*) from community) || ' eperson='||(select count(*) from eperson);" \ | |
| 2>&1 || echo "could not count rows" | |
| echo "===== dspace version" | |
| docker exec dspace$INSTANCE /bin/bash -c "cd /dspace/bin && ./dspace version" 2>&1 | head -5 || echo "dspace version failed" | |
| echo "===== REST API on the instance" | |
| curl -s -o /dev/null -w " repository/server/api -> HTTP %{http_code}\n" "http://localhost:${DSPACE_DOCKER_REST_PORT:-6603}/repository/server/api" || true | |
| curl -s "http://localhost:${DSPACE_DOCKER_REST_PORT:-6603}/repository/server/api/core/items?size=1" 2>/dev/null | head -c 300 || true | |
| echo "" | |
| deploy-8603: | |
| if: (inputs.INSTANCE == '*' || inputs.INSTANCE == '8603') && !inputs.PROBE_ONLY | |
| needs: probe | |
| runs-on: dspace-${{inputs.DEV_MACHINE}}-dep-1 | |
| timeout-minutes: 60 | |
| env: | |
| INSTANCE: '8603' | |
| CONFIG_PATH: /opt/dspace-envs/${{inputs.INSTANCE}} | |
| ENVFILE: /opt/dspace-envs/${{inputs.INSTANCE}}/.env | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: ./.github/actions/erase-db | |
| if: inputs.ERASE_DB | |
| with: | |
| INSTANCE: ${{ env.INSTANCE }} | |
| NAME: dspace-${{ env.INSTANCE }} | |
| - name: Set start.sh as executable | |
| run: chmod +x build-scripts/run/start.sh | |
| - name: deploy to ${{inputs.DEV_MACHINE}} | |
| working-directory: build-scripts/run/ | |
| env: | |
| ADMIN_PASSWORD: ${{ secrets.DSPACE_ADMIN_PASSWORD }} | |
| USER_PASSWORD: ${{ secrets.DSPACE_USER_PASSWORD }} | |
| INSTANCE_OVERLAY: ${{ env.CONFIG_PATH }} | |
| run: | | |
| ./start.sh dspace-$INSTANCE | |
| cd ../.. | |
| docker compose --env-file $ENVFILE -p dspace-$INSTANCE -f docker/docker-compose.yml -f docker/docker-compose-rest.yml -f $CONFIG_PATH/docker-compose-rest.yml -f $CONFIG_PATH/docker-compose.yml up -d --no-build --remove-orphans | |
| import-8603: | |
| runs-on: dspace-${{inputs.DEV_MACHINE}}-dep-1 | |
| if: inputs.IMPORT | |
| timeout-minutes: 900 | |
| needs: deploy-8603 | |
| env: | |
| INSTANCE: ${{inputs.INSTANCE}} | |
| CONFIG_PATH: /opt/dspace-envs/${{inputs.INSTANCE}} | |
| DUMP_PATH: ${{inputs.DUMP_PATH}} | |
| ASSETSTORE_PATH: ${{inputs.ASSETSTORE_PATH}} | |
| ADMIN_PASSWORD: ${{ secrets.DSPACE_ADMIN_PASSWORD }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: vanilla import | |
| run: | | |
| export DNAME=dspace${INSTANCE} | |
| export DDBNAME=dspacedb${INSTANCE} | |
| if [ ! -f "$DUMP_PATH" ]; then | |
| echo "::error::Dump not found on ${{inputs.DEV_MACHINE}}: $DUMP_PATH" | |
| exit 1 | |
| fi | |
| echo "Importing dump: $DUMP_PATH ($(du -h "$DUMP_PATH" | cut -f1))" | |
| # Stop the app so it cannot hold connections to the database we are about to drop. | |
| docker stop $DNAME || true | |
| echo "=====" | |
| docker cp "$DUMP_PATH" $DDBNAME:/tmp/dump.sql | |
| # Recreate the database from scratch and load the vanilla dump. | |
| docker exec $DDBNAME /bin/bash -c " | |
| dropdb -U dspace dspace || true && | |
| createdb -U dspace --owner=dspace --encoding=UNICODE dspace && | |
| psql -U dspace -d dspace -f /tmp/dump.sql | |
| " | |
| echo "=====" | |
| docker exec $DDBNAME /bin/bash -c "rm -f /tmp/dump.sql" || true | |
| - name: run database migration | |
| run: | | |
| export DNAME=dspace${INSTANCE} | |
| export ENVFILE=$CONFIG_PATH/.env | |
| export OVERLAY=$CONFIG_PATH | |
| # --------------------------------------------------------------- | |
| # Why a one-off `docker compose run` instead of `docker exec $DNAME migrate`: | |
| # The dspace entrypoint auto-runs `dspace database migrate` (WITHOUT `ignored`) | |
| # and then starts the REST webapp. On a freshly imported dump the | |
| # "Ignored" XMLWorkflow migrations are NOT applied, so the webapp boots against | |
| # an un-migrated DB, crashes, and (with the restart policy) the container | |
| # restarts. That restart SIGKILLs any `docker exec` running inside it -> the | |
| # migration dies with exit 137. So we stop the app and run the migration as a | |
| # one-off compose container (overridden entrypoint = migrate only, no webapp, | |
| # no restart policy), using the same compose files/env as the deploy step. | |
| # --------------------------------------------------------------- | |
| echo "Stopping app container so its restart loop can't kill the migration..." | |
| docker stop $DNAME || true | |
| # `database repair` before `migrate`: the dump comes from a CLARIN 7.6 instance, | |
| # so its flyway_schema_history lists CLARIN-specific migrations that do not exist | |
| # in this vanilla v9 tree. Flyway validation then refuses to migrate at all: | |
| # "Detected applied migration not resolved locally: 7.6.2026.06.01." | |
| # repair marks those unresolvable entries as deleted, which is exactly what we want | |
| # when migrating an old CLARIN dump onto vanilla v9 -- their DDL is already in the | |
| # dump, we just do not carry the migration scripts any more. | |
| echo "Running one-off DB repair + migration via docker compose..." | |
| docker compose --env-file $ENVFILE -p dspace-${INSTANCE} \ | |
| -f docker/docker-compose.yml -f docker/docker-compose-rest.yml \ | |
| \ | |
| -f $OVERLAY/docker-compose-rest.yml -f $OVERLAY/docker-compose.yml \ | |
| run --rm --no-deps --entrypoint /bin/bash dspace \ | |
| -c "while (!</dev/tcp/dspacedb/5432) >/dev/null 2>&1; do sleep 1; done; cd /dspace/bin && ./dspace database repair && ./dspace database migrate ignored" | |
| echo "Migration done. Starting app container (entrypoint migrate is now a no-op)..." | |
| docker start $DNAME | |
| echo "Waiting for DSpace to be ready..." | |
| sleep 10 | |
| - name: restart dspace container | |
| run: | | |
| export DNAME=dspace${INSTANCE} | |
| echo "Restarting DSpace container..." | |
| docker restart $DNAME | |
| sleep 60 | |
| - name: copy assetstore | |
| run: | | |
| export DNAME=dspace${INSTANCE} | |
| # No `chown -R dspace:dspace` (unlike v7): the v9 runtime image has no `dspace` | |
| # user, the webapp runs as root, and that chown fails with "invalid user". | |
| echo "Preparing assetstore" | |
| if [ -z "$ASSETSTORE_PATH" ]; then | |
| echo "Assetstore path is empty. Not copying assetstore." | |
| else | |
| docker cp "$ASSETSTORE_PATH" $DNAME:/dspace/ | |
| fi | |
| - name: create administrator | |
| run: | | |
| export DNAME=dspace${INSTANCE} | |
| echo "Creating DSpace administrator..." | |
| docker exec $DNAME /bin/bash -c "cd /dspace/bin && ./dspace create-administrator -e dspace.admin.dev@dataquest.sk -f admin -l user -p \"${ADMIN_PASSWORD}\" -c en" | |
| - name: rebuild discovery index and run oai import | |
| run: | | |
| export DNAME=dspace${INSTANCE} | |
| echo "Rebuilding discovery index..." | |
| docker exec $DNAME /bin/bash -c "/dspace/bin/dspace index-discovery -b" | |
| echo "Running OAI import..." | |
| docker exec $DNAME /bin/bash -c "/dspace/bin/dspace oai import -c" | |
| playwright-after-deploy8603: | |
| needs: deploy-8603 | |
| if: '!inputs.IMPORT' | |
| uses: ./.github/workflows/playwright-tests.yml | |
| secrets: inherit | |
| rest-tests-after-deploy8603: | |
| runs-on: ubuntu-latest | |
| needs: playwright-after-deploy8603 | |
| if: '!inputs.IMPORT' | |
| timeout-minutes: 120 | |
| env: | |
| GH_TOKEN: ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }} | |
| REPO: dataquest-dev/dspace-rest-test | |
| WORKFLOW: run_unittests.yml | |
| steps: | |
| - name: run rest-tests | |
| run: | | |
| OUTPUT=$(gh workflow run "$WORKFLOW" \ | |
| --repo "$REPO" \ | |
| --ref master \ | |
| -f CUSTOMER="${{ github.ref_name }}" \ | |
| -f URL="http://dev-6.pc:8603/repository/server/api" 2>&1) | |
| RUN_ID=$(echo "$OUTPUT" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+') | |
| echo "Watching run: https://github.com/$REPO/actions/runs/$RUN_ID" | |
| gh run watch "$RUN_ID" --repo "$REPO" --interval 30 --exit-status | |
| playwright-after-import8603: | |
| needs: import-8603 | |
| if: inputs.IMPORT | |
| uses: ./.github/workflows/playwright-tests.yml | |
| secrets: inherit | |
| rest-tests-after-import8603: | |
| runs-on: ubuntu-latest | |
| needs: playwright-after-import8603 | |
| if: inputs.IMPORT | |
| timeout-minutes: 120 | |
| env: | |
| GH_TOKEN: ${{ secrets.DEPLOY_DEV5_GH_ACTION_DISPATCH }} | |
| REPO: dataquest-dev/dspace-rest-test | |
| WORKFLOW: run_unittests.yml | |
| steps: | |
| - name: run rest-tests | |
| run: | | |
| OUTPUT=$(gh workflow run "$WORKFLOW" \ | |
| --repo "$REPO" \ | |
| --ref master \ | |
| -f CUSTOMER="${{ github.ref_name }}" \ | |
| -f URL="http://dev-6.pc:8603/repository/server/api" 2>&1) | |
| RUN_ID=$(echo "$OUTPUT" | grep -oE '/runs/[0-9]+' | grep -oE '[0-9]+') | |
| echo "Watching run: https://github.com/$REPO/actions/runs/$RUN_ID" | |
| gh run watch "$RUN_ID" --repo "$REPO" --interval 30 --exit-status |