JCU/Docker update (#1345)

* Update docker for jcu

* Resolve the loadsql problem

* Keeping higher versions
---------

Co-authored-by: Matus Kasak <matus.kasak@dataquest.sk>

Author: Kasinhou

.dockerignore

@@ -4,6 +4,7 @@
 */target/
 dspace/modules/*/target/
 Dockerfile.*
-dspace/src/main/docker/dspace-postgres-loadsql
+dspace/src/main/docker/dspace-postgres-pgcrypto
+dspace/src/main/docker/dspace-postgres-pgcrypto-curl
 dspace/src/main/docker/README.md
 dspace/src/main/docker-compose/

.github/workflows/build.yml

@@ -3,8 +3,14 @@
 # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-java-with-maven
 name: Build
 
-# Run this Build for all pushes / PRs to current branch
-on: [push, pull_request]
+# Run this Build for pushes to our main and all PRs
+on:
+  push:
+    branches:
+      - dtq-dev
+      - customer/*
+  pull_request:
+  workflow_dispatch:
 
 permissions:
   contents: read  #  to fetch code (actions/checkout)
@@ -81,9 +87,6 @@ jobs:
   # Codecov upload is a separate job in order to allow us to restart this separate from the entire build/test
   # job above. This is necessary because Codecov uploads seem to randomly fail at times.
   # See https://community.codecov.com/t/upload-issues-unable-to-locate-build-via-github-actions-api/3954
-  # Codecov upload disabled for the JCU fork: the repository is not configured with a
-  # CODECOV_TOKEN and the project-level coverage check fails on customization commits.
-  # Coverage is still produced and uploaded as a build artifact by the 'tests' job above.
   #  codecov:
   #    # Must run after 'tests' job above
   #    needs: tests

.github/workflows/docker.yml

@@ -8,6 +8,14 @@ on:
   workflow_call:
     # Possible Inputs to this reusable job
     inputs:
+      python_version_script_dest:
+        required: false
+        default: version.txt
+        type: string
+      run_python_version_script:
+        required: false
+        default: false
+        type: boolean
       # Build name/id for this Docker build. Used for digest storage to avoid digest overlap between builds.
       build_id:
         required: true
@@ -58,9 +66,9 @@ env:
   #   e.g. PR against 'main' will use "latest". a PR against 'dspace-7_x' will use 'dspace-7_x'.
   IMAGE_TAGS: |
     type=raw,value=latest,enable=${{ github.ref_name == github.event.repository.default_branch }}
+    type=raw,value=${{ github.sha }}
     type=ref,event=branch,enable=${{ github.ref_name != github.event.repository.default_branch }}
     type=ref,event=tag
-    type=raw,value=${{ (github.event.pull_request.base.ref == github.event.repository.default_branch && 'latest') || github.event.pull_request.base.ref }},enable=${{ github.event_name == 'pull_request' }}
   # Define default tag "flavor" for docker/metadata-action per
   # https://github.com/docker/metadata-action#flavor-input
   # We manage the 'latest' tag ourselves to the 'main' branch (see settings above)
@@ -77,25 +85,27 @@ env:
   DEPLOY_ARCH: 'linux/amd64'
   # Registry used during building of Docker images. (All images are later copied to docker.io registry)
   # We use GitHub's Container Registry to avoid aggressive rate limits at DockerHub.
-  DOCKER_BUILD_REGISTRY: ghcr.io
+  DOCKER_BUILD_REGISTRY: docker.io
 
 jobs:
   docker-build:
 
     strategy:
       matrix:
         # Architectures / Platforms for which we will build Docker images
-        arch: [ 'linux/amd64', 'linux/arm64' ]
+        # arch: [ 'linux/amd64', 'linux/arm64' ]
+        arch: [ 'linux/amd64' ]
+        os: [ ubuntu-latest ]
         isPr:
           - ${{ github.event_name == 'pull_request' }}
-        # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work.
-        # The below exclude therefore ensures we do NOT build ARM64 for PRs.
-        exclude:
-          - isPr: true
-            arch: linux/arm64
+            # If this is a PR, we ONLY build for AMD64. For PRs we only do a sanity check test to ensure Docker builds work.
+            # The below exclude therefore ensures we do NOT build ARM64 for PRs.
+            # exclude:
+            # - isPr: true
+          # os: ubuntu-latest
+          # arch: linux/arm64
 
-    # If ARM64, then use the Ubuntu ARM64 runner. Otherwise, use the Ubuntu AMD64 runner
-    runs-on: ${{ matrix.arch == 'linux/arm64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
+    runs-on: ${{ matrix.os }}
 
     steps:
       # This step converts the slashes in the "arch" matrix values above into dashes & saves to env.ARCH_NAME
@@ -111,15 +121,21 @@ jobs:
       - name: Checkout codebase
         uses: actions/checkout@v6
 
+      - name: Add version
+        if: ${{ inputs.run_python_version_script }}
+        run: python scripts/sourceversion.py ${{ github.server_url }}/${{ github.repository }}/actions/runs/ ${{ github.run_id }} > ${{ inputs.python_version_script_dest }}
+
+
       # https://github.com/docker/login-action
       # NOTE: This login occurs for BOTH non-PRs or PRs. PRs *must* also login to access private images from GHCR
       # during the build process
       - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
+        # Only login if not a PR, as PRs only trigger a Docker build and not a push
+        if: ${{ ! matrix.isPr }}
         uses: docker/login-action@v4
         with:
-          registry: ${{ env.DOCKER_BUILD_REGISTRY }}
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
 
       # https://github.com/docker/setup-buildx-action
       - name: Setup Docker Buildx
@@ -164,193 +180,4 @@ jobs:
           # Use GitHub cache to load cached Docker images and cache the results of this build
           # This decreases the number of images we need to fetch from DockerHub
           cache-from: type=gha,scope=${{ inputs.build_id }}
-          cache-to: type=gha,scope=${{ inputs.build_id }},mode=min
-
-      # Export the digest of Docker build locally
-      - name: Export Docker build digest
-        if: ${{ ! matrix.isPr }}
-        run: |
-            mkdir -p /tmp/digests
-            digest="${{ steps.docker_build.outputs.digest }}"
-            touch "/tmp/digests/${digest#sha256:}"
-
-      # Upload digest to an artifact, so that it can be used in combined manifest below
-      # (The purpose of the combined manifest is to list both amd64 and arm64 builds under same tag)
-      - name: Upload Docker build digest to artifact
-        if: ${{ ! matrix.isPr }}
-        uses: actions/upload-artifact@v7
-        with:
-          name: digests-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
-          path: /tmp/digests/*
-          if-no-files-found: error
-          retention-days: 1
-
-      #------------------------------------------------------------------------------
-      # Second, we build the image again in order to store it in a local TAR file.
-      # This TAR of the image is cached/saved as an artifact, so that it can be used
-      # by later jobs to install the brand-new images for automated testing.
-      # This TAR build is performed BOTH for PRs and for branch commits (non-PRs).
-      #
-      # (This approach has the advantage of avoiding having to download the newly built
-      # image from DockerHub or GHCR during automated testing.)
-      #
-      # See the 'docker-deploy' job in docker.yml as an example of where this TAR is used.
-      #-------------------------------------------------------------------------------
-      # Build local image (again) and store in a TAR file in /tmp directory
-      # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
-      # NOTE: This step cannot be combined with the build above as it's a different type of output.
-      - name: Build and push image to local TAR file
-        if: ${{ matrix.arch == 'linux/amd64'}}
-        uses: docker/build-push-action@v7
-        with:
-          build-contexts: |
-            ${{ inputs.dockerfile_additional_contexts }}
-          context: ${{ inputs.dockerfile_context }}
-          file: ${{ inputs.dockerfile_path }}
-          # Tell DSpace's Docker files to use the build registry instead of DockerHub
-          build-args:
-            DOCKER_REGISTRY=${{ env.DOCKER_BUILD_REGISTRY }}
-          platforms: ${{ matrix.arch }}
-          tags: ${{ steps.meta_build.outputs.tags }}
-          labels: ${{ steps.meta_build.outputs.labels }}
-          # Use GitHub cache to load cached Docker images and cache the results of this build
-          # This decreases the number of images we need to fetch from DockerHub
-          cache-from: type=gha,scope=${{ inputs.build_id }}
-          cache-to: type=gha,scope=${{ inputs.build_id }},mode=min
-          # Export image to a local TAR file
-          outputs: type=docker,dest=/tmp/${{ inputs.build_id }}.tar
-
-      # Upload the local docker image (in TAR file) to a build Artifact
-      # This step is only done for AMD64, as that's the only image we use in our automated testing (at this time).
-      - name: Upload local image TAR to artifact
-        if: ${{ matrix.arch == 'linux/amd64'}}
-        uses: actions/upload-artifact@v7
-        with:
-          name: docker-image-${{ inputs.build_id }}-${{ env.ARCH_NAME }}
-          path: /tmp/${{ inputs.build_id }}.tar
-          if-no-files-found: error
-          retention-days: 1
-
-  ##########################################################################################
-  # Merge Docker digests (from various architectures) into a single manifest.
-  # This runs after all Docker builds complete above. The purpose is to include all builds
-  # under a single manifest for this tag.
-  # (e.g. both linux/amd64 and linux/arm64 should be listed under the same tagged Docker image)
-  ##########################################################################################
-  docker-build_manifest:
-    # Only run if this is NOT a PR
-    if: ${{ github.event_name != 'pull_request' }}
-    runs-on: ubuntu-latest
-    needs:
-      - docker-build
-    steps:
-      - name: Download Docker build digests
-        uses: actions/download-artifact@v8
-        with:
-          path: /tmp/digests
-          # Download digests for both AMD64 and ARM64 into same directory
-          pattern: digests-${{ inputs.build_id }}-*
-          merge-multiple: true
-
-      - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
-        uses: docker/login-action@v4
-        with:
-          registry: ${{ env.DOCKER_BUILD_REGISTRY }}
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v4
-
-      - name: Add Docker metadata for image
-        id: meta
-        uses: docker/metadata-action@v6
-        with:
-          images: ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: ${{ env.IMAGE_TAGS }}
-          flavor: ${{ env.TAGS_FLAVOR }}
-
-      - name: Create manifest list from digests and push to ${{ env.DOCKER_BUILD_REGISTRY }}
-        working-directory: /tmp/digests
-        run: |
-          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
-          $(printf '${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
-
-      - name: Inspect manifest in ${{ env.DOCKER_BUILD_REGISTRY }}
-        run: |
-          docker buildx imagetools inspect ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
-
-  ##########################################################################################
-  # Copy images / manifest to DockerHub.
-  # This MUST run after *both* images (AMD64 and ARM64) are built and uploaded to GitHub
-  # Container Registry (GHCR). Attempting to run this in parallel to GHCR builds can result
-  # in a race condition...i.e. the copy to DockerHub may fail if GHCR image has been updated
-  # at the moment when the copy occurs.
-  ##########################################################################################
-  docker-copy_to_dockerhub:
-    # Only run if this is NOT a PR
-    if: ${{ github.event_name != 'pull_request' }}
-    runs-on: ubuntu-latest
-    needs:
-      - docker-build_manifest
-
-    steps:
-      # 'regctl' is used to more easily copy the image to DockerHub and obtain the digest from DockerHub
-      # See https://github.com/regclient/regclient/blob/main/docs/regctl.md
-      - name: Install regctl for Docker registry tools
-        run: |
-          export REGCTL_VERSION=v0.9.2
-          mkdir -p bin
-          curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
-          chmod a+x bin/regctl
-          echo "$(pwd)/bin" >> $GITHUB_PATH
-
-      # This recreates Docker tags for DockerHub
-      - name: Add Docker metadata for image
-        id: meta_dockerhub
-        uses: docker/metadata-action@v6
-        with:
-          images: ${{ env.IMAGE_NAME }}
-          tags: ${{ env.IMAGE_TAGS }}
-          flavor: ${{ env.TAGS_FLAVOR }}
-
-      # Login to source registry first, as this is where we are copying *from*
-      - name: Login to ${{ env.DOCKER_BUILD_REGISTRY }}
-        uses: docker/login-action@v4
-        with:
-          registry: ${{ env.DOCKER_BUILD_REGISTRY }}
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Login to DockerHub, since this is where we are copying *to*
-      - name: Login to DockerHub
-        uses: docker/login-action@v4
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
-
-      # Copy the image from source to DockerHub
-      - name: Copy image from ${{ env.DOCKER_BUILD_REGISTRY }} to docker.io
-        run: |
-          regctl image copy ${{ env.DOCKER_BUILD_REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }} docker.io/${{ env.IMAGE_NAME }}:${{ steps.meta_dockerhub.outputs.version }}
-
-      #--------------------------------------------------------------------
-      # Finally, check whether demo.dspace.org or sandbox.dspace.org need
-      # to be redeployed based on these new DockerHub images.
-      #--------------------------------------------------------------------
-      # If this build is for the branch that Sandbox uses and passed in a REDEPLOY_SANDBOX_URL secret,
-      # Then redeploy https://sandbox.dspace.org
-      - name: Redeploy sandbox.dspace.org (based on main branch)
-        if: |
-          env.REDEPLOY_SANDBOX_URL != '' &&
-          github.ref_name == env.DEPLOY_SANDBOX_BRANCH
-        run: |
-          curl -X POST $REDEPLOY_SANDBOX_URL
-      # If this build is for the branch that Demo uses and passed in a REDEPLOY_DEMO_URL secret,
-      # Then redeploy https://demo.dspace.org
-      - name: Redeploy demo.dspace.org (based on maintenance branch)
-        if: |
-          env.REDEPLOY_DEMO_URL != '' &&
-          github.ref_name == env.DEPLOY_DEMO_BRANCH
-        run: |
-          curl -X POST $REDEPLOY_DEMO_URL
+          cache-to: type=gha,scope=${{ inputs.build_id }},mode=max

.github/workflows/reusable-docker-build.yml

@@ -0,0 +1,46 @@
+name: Release
+
+on:
+  push:
+    tags:      
+      - '**'
+  workflow_dispatch:
+
+
+env:
+  IMAGE_BASE_NAME: dataquest/dspace
+
+jobs:
+  retag-BE-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Login to DockerHub
+        uses: docker/login-action@v4
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_ACCESS_TOKEN }}
+
+      - name: retag image
+        run: |
+          docker pull ${{ env.IMAGE_BASE_NAME }}:${{ github.sha }}
+          docker tag ${{ env.IMAGE_BASE_NAME }}:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}:${{ github.ref_name }}
+
+          docker pull ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.sha }}
+          docker tag ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.ref_name }}
+
+          docker pull ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.sha }}
+          docker tag ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.ref_name }}
+          
+          docker pull ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.sha }}
+          docker tag ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.ref_name }}
+
+          docker pull ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.sha }}
+          docker tag ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.sha }} ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.ref_name }}
+
+      - name: push image
+        run: |
+          docker push ${{ env.IMAGE_BASE_NAME }}:${{ github.ref_name }}
+          docker push ${{ env.IMAGE_BASE_NAME }}-cli:${{ github.ref_name }}
+          docker push ${{ env.IMAGE_BASE_NAME }}-solr:${{ github.ref_name }}
+          docker push ${{ env.IMAGE_BASE_NAME }}-postgres-pgcrypto:${{ github.ref_name }}
+          docker push ${{ env.IMAGE_BASE_NAME }}-dependencies:${{ github.ref_name }}

.github/workflows/tag-release.yml

@@ -67,5 +67,12 @@ RUN apt-get update \
 EXPOSE 8080 8000
 # Give java extra memory (2GB)
 ENV JAVA_OPTS=-Xmx2000m
+
+# We create a 'dspace' user to run DSpace instead of running as root. An explicit UID is required 
+# because Kubernetes deployment accepts only numeric user IDs when specifying the container user.
+RUN useradd -u 1100 -m -s /bin/bash dspace \
+    && chown -Rv dspace: /dspace
+USER dspace
+
 # On startup, run DSpace Runnable JAR (uses the "dspace.dir" setting defined in "dspace__P__dir" env variable)
 ENTRYPOINT ["java", "-jar", "webapps/server-boot.jar"]

Dockerfile

@@ -8,10 +8,10 @@
 
 services:
   dspacedb:
-    image: "${DOCKER_REGISTRY:-docker.io}/${DOCKER_OWNER:-dspace}/dspace-postgres-loadsql:${DSPACE_VER:-dspace-9_x}"
+    image: dataquest/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-9_x}-loadsql
     environment:
       # This SQL is available from https://github.com/DSpace-Labs/AIP-Files/releases/tag/demo-entities-data
-      - LOADSQL=${LOADSQL:-https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql}
+      - LOADSQL=https://github.com/DSpace-Labs/AIP-Files/releases/download/demo-entities-data/dspace7-entities-data.sql
   dspace:
     ### OVERRIDE default 'entrypoint' in 'docker-compose.yml ####
     # Ensure that the database is ready BEFORE starting tomcat
@@ -24,4 +24,4 @@ services:
       - |
         while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
         /dspace/bin/dspace database migrate ignored
-        java -jar /dspace/webapps/server-boot.jar
+        java -jar /dspace/webapps/server-boot.jar --dspace.dir=/dspace