Add container healthchecks, wait-for-healthy dependencies and automatic restart

Dockerfile.test

@@ -57,9 +57,13 @@ ENV DSPACE_INSTALL=/dspace
 ENV TOMCAT_INSTALL=/usr/local/tomcat
 # Copy the /dspace directory from 'ant_build' containger to /dspace in this container
 COPY --from=ant_build /dspace $DSPACE_INSTALL
-# Need host command for "[dspace]/bin/make-handle-config"
+# host: needed by "[dspace]/bin/make-handle-config"
+# curl: used by the container healthcheck in docker-compose.yml. The tomcat base image
+#       ships it today, but the probe must not silently depend on that staying true: if
+#       curl went away the container would never report healthy, service_healthy would
+#       block dspace-angular, and autoheal would restart a container that is actually fine.
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends host \
+    && apt-get install -y --no-install-recommends host curl \
     && apt-get purge -y --auto-remove \
     && rm -rf /var/lib/apt/lists/*
 # Enable the AJP connector in Tomcat's server.xml

docker-compose.yml

@@ -1,4 +1,24 @@
 version: '3.7'
+# IMAGE VERSION - build it, do not pull it
+#
+# The dspace image MUST be built from this repository:
+#
+#     docker compose build dspace
+#
+# This branch carries customisations that no published image contains. For example
+# dspace/config/ehcache.xml references org.dspace.external.provider.orcid.xml.CacheLogger,
+# which exists in dspace-api here but in neither upstream 7.6.5 nor 7.6.8. Since this
+# compose file also mounts ./dspace/config over the image config, a pulled image gives
+# you a newer/other webapp reading this tree config, and the DSpace kernel dies with
+# "Error parsing XML configuration at file:/dspace/config/ehcache.xml".
+#
+# `docker compose up` builds a missing image on its own, so a clean machine is fine.
+# The trap is a STALE image left over from an earlier pull: it shadows the build and
+# the failure looks unrelated to images. `docker compose build` is the fix.
+#
+# DSPACE_VER is pinned to the pom.xml version rather than the floating dspace-7_x tag
+# so the local build cannot be silently shadowed by a newer upstream one. Bump it
+# together with pom.xml.
 networks:
   dspacenet:
     ipam:
@@ -10,6 +30,11 @@ services:
   # DSpace (backend) webapp container
   dspace:
     container_name: dspace
+    # Required for the ExitOnOutOfMemoryError in JAVA_OPTS below to be useful: it makes Docker
+    # bring the container back after the JVM kills itself. Note this policy reacts
+    # ONLY to the process exiting - an unhealthy-but-running container is the
+    # autoheal sidecar's job, not this.
+    restart: unless-stopped
     environment:
       # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables.
       # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml
@@ -29,12 +54,27 @@ services:
       # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above.
       proxies__P__trusted__P__ipranges: '172.23.0'
       LOGGING_CONFIG: /dspace/config/log4j2-container.xml
-    image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}"
+      # JVM options. -Xmx2000m is the image default and is kept as-is.
+      #
+      # ExitOnOutOfMemoryError is the piece that gives OOM coverage. A healthcheck is
+      # the wrong tool for OOM: after an OutOfMemoryError the JVM is often still able
+      # to answer a trivial health request while being unable to serve real work, so a
+      # probe can report healthy on a process that is effectively dead. Letting the JVM
+      # exit turns an invisible internal failure into a container exit, which the
+      # restart policy below then handles - no sidecar involved.
+      #
+      # HeapDumpOnOutOfMemoryError writes the dump to the mounted log volume so the
+      # cause survives the restart. Without it the restart destroys the evidence.
+      JAVA_OPTS: '-Xmx2000m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dspace/log'
+    image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7.6.5}-test"
     build:
       context: .
       dockerfile: Dockerfile.test
     depends_on:
-    - dspacedb
+      dspacedb:
+        condition: service_healthy
+      dspacesolr:
+        condition: service_healthy
     networks:
       - dspacenet
     ports:
@@ -49,25 +89,84 @@ services:
     volumes:
     # Keep DSpace assetstore directory between reboots
     - assetstore:/dspace/assetstore
+    # Keep the log directory between container recreates. Without this, the heap dump
+    # written by HeapDumpOnOutOfMemoryError (see JAVA_OPTS) lives only in the writable
+    # layer and is destroyed by `docker compose down` - i.e. exactly when someone is
+    # trying to find out why the container died. A named volume is used on purpose: it
+    # inherits ownership of /dspace/log from the image, whereas a bind mount to a fresh
+    # host directory would be owned by root and silently unwritable.
+    - dspacelogs:/dspace/log
     # Mount local [src]/dspace/config/ to container. This syncs your local configs with container
     # NOTE: Environment variables specified above will OVERRIDE any configs in local.cfg or dspace.cfg
     - ./dspace/config:/dspace/config
-    # Ensure that the database is ready BEFORE starting tomcat
-    # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep
-    # 2. Then, run database migration to init database tables
-    # 3. Finally, start Tomcat
+    # The database is guaranteed to be ready by depends_on: condition: service_healthy
+    # above, so this no longer waits for it. The TCP poll that used to live here was
+    # weaker anyway: the port accepts connections before Postgres accepts queries.
+    # 1. Run database migration to init database tables
+    # 2. Start Tomcat
     entrypoint:
     - /bin/bash
     - '-c'
     - |
-      while (!</dev/tcp/dspacedb/5432) > /dev/null 2>&1; do sleep 1; done;
       /dspace/bin/dspace database migrate
       catalina.sh run
+    # This probe targets the liveness GROUP, not the aggregated /actuator/health.
+    # The aggregate is DOWN even on a clean install (SEOHealthIndicator reports a
+    # missing robots.txt/sitemap/SSR) and a restart cannot fix that, so wiring a
+    # healthcheck to it would restart-loop forever. The group also maps DOWN to a
+    # real 503, so a plain `curl -f` suffices - no fragile JSON string matching.
+    #
+    # What this probe covers, and what it deliberately does not:
+    #
+    #   DB unreachable / dead connection pool -> the `db` component in the liveness
+    #     group goes DOWN, the group maps that to 503, curl -f fails.
+    #
+    #   Tomcat thread pool exhausted, or the JVM wedged in a GC spiral -> actuator is
+    #     served by the SAME Tomcat connector and thread pool as the REST API, so a
+    #     wedged server cannot answer this request either. --max-time 5 turns that into
+    #     a failure instead of a hang. (Measured: with the database frozen, the health
+    #     endpoint returned nothing for 25s+.) This is why the timeout matters as much
+    #     as the endpoint choice.
+    #
+    #   OutOfMemoryError -> NOT covered here on purpose. After an OOM the JVM can often
+    #     still answer a trivial health request while being unable to do real work, so a
+    #     probe would report healthy on a dead process. That case is handled by
+    #     -XX:+ExitOnOutOfMemoryError in JAVA_OPTS plus `restart: unless-stopped`.
+    #
+    #   Disk full, missing robots.txt, misconfiguration -> deliberately NOT covered.
+    #     A restart cannot fix any of them, so making them fail this probe would only
+    #     produce a restart loop.
+    healthcheck:
+      # The leading chaos-flag check is a deliberate test hook: it lets you prove the
+      # whole unhealthy -> restart chain end to end without taking down Postgres or
+      # Solr, and without waiting for a real outage.
+      #   docker exec dspace touch /tmp/chaos-fail   -> report a fault artificially
+      #   docker exec dspace rm    /tmp/chaos-fail   -> clear it
+      # NOTE: the flag survives a restart (it lives in the container's writable
+      # layer, which `docker restart` does not reset), so remove it once the restart
+      # has been observed or autoheal will keep restarting the container.
+      test: ["CMD-SHELL", "[ -f /tmp/chaos-fail ] && exit 1; curl -fsS --max-time 5 http://localhost:8080/server/actuator/health/liveness > /dev/null || exit 1"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      # DSpace needs several minutes to boot; too short a start_period restart-loops.
+      start_period: 300s
+    labels:
+      # Consumed by the optional autoheal sidecar, see docker-compose-autoheal.yml
+      autoheal: "true"
   # DSpace PostgreSQL database container
   dspacedb:
     container_name: dspacedb
+    # Two independent recovery paths, both needed:
+    #   restart policy -> the postmaster died and the container exited
+    #   autoheal label -> the container still runs but pg_isready keeps failing
+    # Restarting a database is not free, which is why the sidecar enforces a restart
+    # budget (3 per hour by default) and then stops and asks for a human.
+    restart: unless-stopped
+    labels:
+      autoheal: "true"
     # Uses a custom Postgres image with pgcrypto installed
-    image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}"
+    image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7.6.5}"
     build:
       # Must build out of subdirectory to have access to install script for pgcrypto
       context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/
@@ -84,12 +183,39 @@ services:
     volumes:
     # Keep Postgres data directory between reboots
     - pgdata:/pgdata
+    # pg_isready is the purpose-built readiness probe: unlike a TCP port check it
+    # only succeeds once the server actually accepts queries (not during initdb
+    # or crash recovery, when the port is already listening).
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U dspace -d dspace"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 30s
   # DSpace Solr container
   dspacesolr:
     container_name: dspacesolr
-    image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}"
+    # Two independent recovery paths, both needed:
+    #   restart policy -> the Solr process died and the container exited
+    #   autoheal label -> the container still runs but the search core is gone
+    #
+    # Note what a restart does NOT fix: precreate-core only checks whether the core
+    # DIRECTORY exists, so once a core has been unloaded (its core.properties is gone
+    # but the directory stays) a restart logs "Core search already exists" and moves on,
+    # leaving the core unregistered. That is intentional on the sidecar side - it burns
+    # its restart budget, gives up, and asks for a human, which beats looping forever.
+    restart: unless-stopped
+    labels:
+      autoheal: "true"
+    image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7.6.5}"
     build:
-      context: ./dspace/src/main/docker/dspace-solr/
+      # Context is the repository root, matching .github/workflows/docker.yml. The
+      # Dockerfile does `COPY scripts/log4j2.solr.xml`, which lives at the repo root,
+      # so the narrower ./dspace/src/main/docker/dspace-solr/ context used before made
+      # `docker compose build dspacesolr` fail with "/scripts/log4j2.solr.xml: not found"
+      # while CI built the same image fine.
+      context: .
+      dockerfile: ./dspace/src/main/docker/dspace-solr/Dockerfile
       # Provide path to Solr configs necessary to build Docker image
       additional_contexts:
         solrconfigs: ./dspace/solr/
@@ -124,7 +250,18 @@ services:
       precreate-core statistics /opt/solr/server/solr/configsets/statistics
       cp -r /opt/solr/server/solr/configsets/statistics/* statistics
       exec solr -f
+    # A port check is not enough: precreate-core runs AFTER Solr starts listening,
+    # so a port probe goes green while DSpace would still fail every query.
+    # Asking for a specific core discriminates - a missing core returns
+    # {"status":{"search":{}}} with no "name", so the grep fails.
+    healthcheck:
+      test: ["CMD-SHELL", "curl -fsS 'http://localhost:8983/solr/admin/cores?action=STATUS&core=search' | grep -q '\"name\":\"search\"'"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+      start_period: 60s
 volumes:
   assetstore:
+  dspacelogs:
   pgdata:
   solr_data:

dspace/config/modules/actuator.cfg

@@ -56,3 +56,34 @@ info.app.mail.alert-recipient = ${alert.recipient}
 info.app.cors.allowed-origins = ${rest.cors.allowed-origins}
 
 info.app.ui.url = ${dspace.ui.url}
+
+#---------------------------------------------------------------#
+#--------------------HEALTH GROUPS------------------------------#
+#---------------------------------------------------------------#
+
+# Why groups exist: the aggregated /actuator/health is DOWN on a perfectly
+# healthy, freshly installed repository. SEOHealthIndicator calls down() when
+# robots.txt / sitemap / SSR are missing, and status.order ranks "down" highest,
+# so that single indicator drags the whole aggregate to DOWN. Restarting the
+# container never fixes a missing robots.txt, so a container healthcheck wired
+# to the aggregate would restart-loop forever.
+# Groups let a probe ask a narrower question. Both groups map DOWN to a real
+# 503 (the global mapping must stay 200 - the Angular /health page relies on it)
+# so probes can be a plain `curl -f` with no body parsing.
+
+# LIVENESS = "should this process be restarted?"
+# Only components a restart can actually fix. Solr is deliberately NOT here:
+# restarting Tomcat does not bring Solr back, it would only cause a restart loop
+# and take down requests that do not need Solr at all.
+management.endpoint.health.group.liveness.include = db
+management.endpoint.health.group.liveness.show-details = never
+management.endpoint.health.group.liveness.status.http-mapping.down = 503
+management.endpoint.health.group.liveness.status.http-mapping.out-of-service = 503
+
+# READINESS = "should the load balancer send traffic here?"
+# Wider - includes Solr. A Solr outage takes the instance out of rotation
+# without restarting it.
+management.endpoint.health.group.readiness.include = db,solrSearchCore,solrStatisticsCore
+management.endpoint.health.group.readiness.show-details = never
+management.endpoint.health.group.readiness.status.http-mapping.down = 503
+management.endpoint.health.group.readiness.status.http-mapping.out-of-service = 503

dspace/src/main/docker-compose/docker-compose-angular.yml

@@ -15,8 +15,10 @@ networks:
 services:
   dspace-angular:
     container_name: dspace-angular
+    restart: unless-stopped
     depends_on:
-    - dspace
+      dspace:
+        condition: service_healthy
     environment:
       DSPACE_UI_SSL: 'false'
       DSPACE_UI_HOST: dspace-angular
@@ -26,11 +28,39 @@ services:
       DSPACE_REST_HOST: localhost
       DSPACE_REST_PORT: 8080
       DSPACE_REST_NAMESPACE: /server
-    image: dspace/dspace-angular:dspace-7_x
+    image: "${DOCKER_OWNER:-dspace}/dspace-angular:${DSPACE_VER:-dspace-7.6.5}"
     ports:
     - published: 4000
       target: 4000
     - published: 9876
       target: 9876
     stdin_open: true
     tty: true
+    # Probe choice, and why it is not the obvious one:
+    #
+    # NOT /app/health - that endpoint is a pure proxy to the backend actuator (see
+    # server.ts healthCheck()), so it reports the BACKEND's health, not this
+    # container's. It would mark the UI unhealthy whenever the backend is down, and
+    # restarting the UI cannot fix the backend, so autoheal would loop. It also
+    # forwards the backend's HTTP 200-on-DOWN.
+    #
+    # NOT /robots.txt either - that express route lives in the SSR server (server.ts)
+    # which only runs in the *dist* image. This overlay uses dspace/dspace-angular,
+    # the DEVELOPMENT image: it runs `ng serve`, which returns 404 for /robots.txt.
+    # Measured on this image: / -> 200 in 35ms, /robots.txt -> 404.
+    # If you switch this overlay to dspace-angular-dist, /robots.txt becomes the
+    # better probe - it is served from a template with no backend call.
+    #
+    # start_period is large on purpose: `ng serve` compiles the bundles at container
+    # start and does not answer the first request for many minutes. A short value here
+    # means autoheal kills the build and the container never finishes booting - we hit
+    # exactly that during testing.
+    healthcheck:
+      test: ["CMD-SHELL", "node -e \"const r=require('http').get({host:'127.0.0.1',port:4000,path:'/'},s=>process.exit(s.statusCode===200?0:1));r.setTimeout(8000,()=>{r.destroy();process.exit(1)});r.on('error',()=>process.exit(1))\""]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 900s
+    labels:
+      # Consumed by the optional autoheal sidecar, see docker-compose-autoheal.yml
+      autoheal: "true"