diff --git a/Dockerfile.test b/Dockerfile.test index 65e9cb956a44..3c87cfb7b29e 100644 --- a/Dockerfile.test +++ b/Dockerfile.test @@ -57,9 +57,13 @@ ENV DSPACE_INSTALL=/dspace ENV TOMCAT_INSTALL=/usr/local/tomcat # Copy the /dspace directory from 'ant_build' containger to /dspace in this container COPY --from=ant_build /dspace $DSPACE_INSTALL -# Need host command for "[dspace]/bin/make-handle-config" +# host: needed by "[dspace]/bin/make-handle-config" +# curl: used by the container healthcheck in docker-compose.yml. The tomcat base image +# ships it today, but the probe must not silently depend on that staying true: if +# curl went away the container would never report healthy, service_healthy would +# block dspace-angular, and autoheal would restart a container that is actually fine. RUN apt-get update \ - && apt-get install -y --no-install-recommends host \ + && apt-get install -y --no-install-recommends host curl \ && apt-get purge -y --auto-remove \ && rm -rf /var/lib/apt/lists/* # Enable the AJP connector in Tomcat's server.xml diff --git a/docker-compose.yml b/docker-compose.yml index a7894135c6ab..2a02c4987bdd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,24 @@ version: '3.7' +# IMAGE VERSION - build it, do not pull it +# +# The dspace image MUST be built from this repository: +# +# docker compose build dspace +# +# This branch carries customisations that no published image contains. For example +# dspace/config/ehcache.xml references org.dspace.external.provider.orcid.xml.CacheLogger, +# which exists in dspace-api here but in neither upstream 7.6.5 nor 7.6.8. Since this +# compose file also mounts ./dspace/config over the image config, a pulled image gives +# you a newer/other webapp reading this tree config, and the DSpace kernel dies with +# "Error parsing XML configuration at file:/dspace/config/ehcache.xml". +# +# `docker compose up` builds a missing image on its own, so a clean machine is fine. +# The trap is a STALE image left over from an earlier pull: it shadows the build and +# the failure looks unrelated to images. `docker compose build` is the fix. +# +# DSPACE_VER is pinned to the pom.xml version rather than the floating dspace-7_x tag +# so the local build cannot be silently shadowed by a newer upstream one. Bump it +# together with pom.xml. networks: dspacenet: ipam: @@ -10,6 +30,11 @@ services: # DSpace (backend) webapp container dspace: container_name: dspace + # Required for the ExitOnOutOfMemoryError in JAVA_OPTS below to be useful: it makes Docker + # bring the container back after the JVM kills itself. Note this policy reacts + # ONLY to the process exiting - an unhealthy-but-running container is the + # autoheal sidecar's job, not this. + restart: unless-stopped environment: # Below syntax may look odd, but it is how to override dspace.cfg settings via env variables. # See https://github.com/DSpace/DSpace/blob/main/dspace/config/config-definition.xml @@ -29,12 +54,27 @@ services: # from the host machine. This IP range MUST correspond to the 'dspacenet' subnet defined above. proxies__P__trusted__P__ipranges: '172.23.0' LOGGING_CONFIG: /dspace/config/log4j2-container.xml - image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7_x-test}" + # JVM options. -Xmx2000m is the image default and is kept as-is. + # + # ExitOnOutOfMemoryError is the piece that gives OOM coverage. A healthcheck is + # the wrong tool for OOM: after an OutOfMemoryError the JVM is often still able + # to answer a trivial health request while being unable to serve real work, so a + # probe can report healthy on a process that is effectively dead. Letting the JVM + # exit turns an invisible internal failure into a container exit, which the + # restart policy below then handles - no sidecar involved. + # + # HeapDumpOnOutOfMemoryError writes the dump to the mounted log volume so the + # cause survives the restart. Without it the restart destroys the evidence. + JAVA_OPTS: '-Xmx2000m -XX:+ExitOnOutOfMemoryError -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/dspace/log' + image: "${DOCKER_OWNER:-dspace}/dspace:${DSPACE_VER:-dspace-7.6.5}-test" build: context: . dockerfile: Dockerfile.test depends_on: - - dspacedb + dspacedb: + condition: service_healthy + dspacesolr: + condition: service_healthy networks: - dspacenet ports: @@ -49,25 +89,84 @@ services: volumes: # Keep DSpace assetstore directory between reboots - assetstore:/dspace/assetstore + # Keep the log directory between container recreates. Without this, the heap dump + # written by HeapDumpOnOutOfMemoryError (see JAVA_OPTS) lives only in the writable + # layer and is destroyed by `docker compose down` - i.e. exactly when someone is + # trying to find out why the container died. A named volume is used on purpose: it + # inherits ownership of /dspace/log from the image, whereas a bind mount to a fresh + # host directory would be owned by root and silently unwritable. + - dspacelogs:/dspace/log # Mount local [src]/dspace/config/ to container. This syncs your local configs with container # NOTE: Environment variables specified above will OVERRIDE any configs in local.cfg or dspace.cfg - ./dspace/config:/dspace/config - # Ensure that the database is ready BEFORE starting tomcat - # 1. While a TCP connection to dspacedb port 5432 is not available, continue to sleep - # 2. Then, run database migration to init database tables - # 3. Finally, start Tomcat + # The database is guaranteed to be ready by depends_on: condition: service_healthy + # above, so this no longer waits for it. The TCP poll that used to live here was + # weaker anyway: the port accepts connections before Postgres accepts queries. + # 1. Run database migration to init database tables + # 2. Start Tomcat entrypoint: - /bin/bash - '-c' - | - while (! /dev/null 2>&1; do sleep 1; done; /dspace/bin/dspace database migrate catalina.sh run + # This probe targets the liveness GROUP, not the aggregated /actuator/health. + # The aggregate is DOWN even on a clean install (SEOHealthIndicator reports a + # missing robots.txt/sitemap/SSR) and a restart cannot fix that, so wiring a + # healthcheck to it would restart-loop forever. The group also maps DOWN to a + # real 503, so a plain `curl -f` suffices - no fragile JSON string matching. + # + # What this probe covers, and what it deliberately does not: + # + # DB unreachable / dead connection pool -> the `db` component in the liveness + # group goes DOWN, the group maps that to 503, curl -f fails. + # + # Tomcat thread pool exhausted, or the JVM wedged in a GC spiral -> actuator is + # served by the SAME Tomcat connector and thread pool as the REST API, so a + # wedged server cannot answer this request either. --max-time 5 turns that into + # a failure instead of a hang. (Measured: with the database frozen, the health + # endpoint returned nothing for 25s+.) This is why the timeout matters as much + # as the endpoint choice. + # + # OutOfMemoryError -> NOT covered here on purpose. After an OOM the JVM can often + # still answer a trivial health request while being unable to do real work, so a + # probe would report healthy on a dead process. That case is handled by + # -XX:+ExitOnOutOfMemoryError in JAVA_OPTS plus `restart: unless-stopped`. + # + # Disk full, missing robots.txt, misconfiguration -> deliberately NOT covered. + # A restart cannot fix any of them, so making them fail this probe would only + # produce a restart loop. + healthcheck: + # The leading chaos-flag check is a deliberate test hook: it lets you prove the + # whole unhealthy -> restart chain end to end without taking down Postgres or + # Solr, and without waiting for a real outage. + # docker exec dspace touch /tmp/chaos-fail -> report a fault artificially + # docker exec dspace rm /tmp/chaos-fail -> clear it + # NOTE: the flag survives a restart (it lives in the container's writable + # layer, which `docker restart` does not reset), so remove it once the restart + # has been observed or autoheal will keep restarting the container. + test: ["CMD-SHELL", "[ -f /tmp/chaos-fail ] && exit 1; curl -fsS --max-time 5 http://localhost:8080/server/actuator/health/liveness > /dev/null || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + # DSpace needs several minutes to boot; too short a start_period restart-loops. + start_period: 300s + labels: + # Consumed by the optional autoheal sidecar, see docker-compose-autoheal.yml + autoheal: "true" # DSpace PostgreSQL database container dspacedb: container_name: dspacedb + # Two independent recovery paths, both needed: + # restart policy -> the postmaster died and the container exited + # autoheal label -> the container still runs but pg_isready keeps failing + # Restarting a database is not free, which is why the sidecar enforces a restart + # budget (3 per hour by default) and then stops and asks for a human. + restart: unless-stopped + labels: + autoheal: "true" # Uses a custom Postgres image with pgcrypto installed - image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7_x}" + image: "${DOCKER_OWNER:-dspace}/dspace-postgres-pgcrypto:${DSPACE_VER:-dspace-7.6.5}" build: # Must build out of subdirectory to have access to install script for pgcrypto context: ./dspace/src/main/docker/dspace-postgres-pgcrypto/ @@ -84,12 +183,39 @@ services: volumes: # Keep Postgres data directory between reboots - pgdata:/pgdata + # pg_isready is the purpose-built readiness probe: unlike a TCP port check it + # only succeeds once the server actually accepts queries (not during initdb + # or crash recovery, when the port is already listening). + healthcheck: + test: ["CMD-SHELL", "pg_isready -U dspace -d dspace"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s # DSpace Solr container dspacesolr: container_name: dspacesolr - image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7_x}" + # Two independent recovery paths, both needed: + # restart policy -> the Solr process died and the container exited + # autoheal label -> the container still runs but the search core is gone + # + # Note what a restart does NOT fix: precreate-core only checks whether the core + # DIRECTORY exists, so once a core has been unloaded (its core.properties is gone + # but the directory stays) a restart logs "Core search already exists" and moves on, + # leaving the core unregistered. That is intentional on the sidecar side - it burns + # its restart budget, gives up, and asks for a human, which beats looping forever. + restart: unless-stopped + labels: + autoheal: "true" + image: "${DOCKER_OWNER:-dspace}/dspace-solr:${DSPACE_VER:-dspace-7.6.5}" build: - context: ./dspace/src/main/docker/dspace-solr/ + # Context is the repository root, matching .github/workflows/docker.yml. The + # Dockerfile does `COPY scripts/log4j2.solr.xml`, which lives at the repo root, + # so the narrower ./dspace/src/main/docker/dspace-solr/ context used before made + # `docker compose build dspacesolr` fail with "/scripts/log4j2.solr.xml: not found" + # while CI built the same image fine. + context: . + dockerfile: ./dspace/src/main/docker/dspace-solr/Dockerfile # Provide path to Solr configs necessary to build Docker image additional_contexts: solrconfigs: ./dspace/solr/ @@ -124,7 +250,23 @@ services: precreate-core statistics /opt/solr/server/solr/configsets/statistics cp -r /opt/solr/server/solr/configsets/statistics/* statistics exec solr -f + # Asks the core's own ping handler rather than the CoreAdmin list. A registered core + # answers 200; an unregistered one is simply not a route, so Solr answers 404 and + # curl -f fails. That is worth more than it looks: + # - no pipe and no grep, so the probe depends on nothing but curl + # - no matching on Solr's JSON, which could be reformatted by a Solr upgrade + # - ping runs a real query against the core, so it checks the core actually + # answers rather than merely appearing in a list + # A port check would be useless here: precreate-core runs AFTER Solr starts + # listening, so the port is up while every DSpace query would still fail. + healthcheck: + test: ["CMD-SHELL", "curl -fsS -o /dev/null http://localhost:8983/solr/search/admin/ping"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 60s volumes: assetstore: + dspacelogs: pgdata: solr_data: diff --git a/dspace/config/modules/actuator.cfg b/dspace/config/modules/actuator.cfg index b23ccc3424b0..bb0e5a36d5ef 100644 --- a/dspace/config/modules/actuator.cfg +++ b/dspace/config/modules/actuator.cfg @@ -56,3 +56,34 @@ info.app.mail.alert-recipient = ${alert.recipient} info.app.cors.allowed-origins = ${rest.cors.allowed-origins} info.app.ui.url = ${dspace.ui.url} + +#---------------------------------------------------------------# +#--------------------HEALTH GROUPS------------------------------# +#---------------------------------------------------------------# + +# Why groups exist: the aggregated /actuator/health is DOWN on a perfectly +# healthy, freshly installed repository. SEOHealthIndicator calls down() when +# robots.txt / sitemap / SSR are missing, and status.order ranks "down" highest, +# so that single indicator drags the whole aggregate to DOWN. Restarting the +# container never fixes a missing robots.txt, so a container healthcheck wired +# to the aggregate would restart-loop forever. +# Groups let a probe ask a narrower question. Both groups map DOWN to a real +# 503 (the global mapping must stay 200 - the Angular /health page relies on it) +# so probes can be a plain `curl -f` with no body parsing. + +# LIVENESS = "should this process be restarted?" +# Only components a restart can actually fix. Solr is deliberately NOT here: +# restarting Tomcat does not bring Solr back, it would only cause a restart loop +# and take down requests that do not need Solr at all. +management.endpoint.health.group.liveness.include = db +management.endpoint.health.group.liveness.show-details = never +management.endpoint.health.group.liveness.status.http-mapping.down = 503 +management.endpoint.health.group.liveness.status.http-mapping.out-of-service = 503 + +# READINESS = "should the load balancer send traffic here?" +# Wider - includes Solr. A Solr outage takes the instance out of rotation +# without restarting it. +management.endpoint.health.group.readiness.include = db,solrSearchCore,solrStatisticsCore +management.endpoint.health.group.readiness.show-details = never +management.endpoint.health.group.readiness.status.http-mapping.down = 503 +management.endpoint.health.group.readiness.status.http-mapping.out-of-service = 503 diff --git a/dspace/src/main/docker-compose/docker-compose-angular.yml b/dspace/src/main/docker-compose/docker-compose-angular.yml index c9b87c904f17..626f247d00a9 100644 --- a/dspace/src/main/docker-compose/docker-compose-angular.yml +++ b/dspace/src/main/docker-compose/docker-compose-angular.yml @@ -15,8 +15,10 @@ networks: services: dspace-angular: container_name: dspace-angular + restart: unless-stopped depends_on: - - dspace + dspace: + condition: service_healthy environment: DSPACE_UI_SSL: 'false' DSPACE_UI_HOST: dspace-angular @@ -26,7 +28,7 @@ services: DSPACE_REST_HOST: localhost DSPACE_REST_PORT: 8080 DSPACE_REST_NAMESPACE: /server - image: dspace/dspace-angular:dspace-7_x + image: "${DOCKER_OWNER:-dspace}/dspace-angular:${DSPACE_VER:-dspace-7.6.5}" ports: - published: 4000 target: 4000 @@ -34,3 +36,31 @@ services: target: 9876 stdin_open: true tty: true + # Probe choice, and why it is not the obvious one: + # + # NOT /app/health - that endpoint is a pure proxy to the backend actuator (see + # server.ts healthCheck()), so it reports the BACKEND's health, not this + # container's. It would mark the UI unhealthy whenever the backend is down, and + # restarting the UI cannot fix the backend, so autoheal would loop. It also + # forwards the backend's HTTP 200-on-DOWN. + # + # NOT /robots.txt either - that express route lives in the SSR server (server.ts) + # which only runs in the *dist* image. This overlay uses dspace/dspace-angular, + # the DEVELOPMENT image: it runs `ng serve`, which returns 404 for /robots.txt. + # Measured on this image: / -> 200 in 35ms, /robots.txt -> 404. + # If you switch this overlay to dspace-angular-dist, /robots.txt becomes the + # better probe - it is served from a template with no backend call. + # + # start_period is large on purpose: `ng serve` compiles the bundles at container + # start and does not answer the first request for many minutes. A short value here + # means autoheal kills the build and the container never finishes booting - we hit + # exactly that during testing. + healthcheck: + test: ["CMD-SHELL", "node -e \"const r=require('http').get({host:'127.0.0.1',port:4000,path:'/'},s=>process.exit(s.statusCode===200?0:1));r.setTimeout(8000,()=>{r.destroy();process.exit(1)});r.on('error',()=>process.exit(1))\""] + interval: 30s + timeout: 10s + retries: 3 + start_period: 900s + labels: + # Consumed by the optional autoheal sidecar, see docker-compose-autoheal.yml + autoheal: "true" diff --git a/dspace/src/main/docker-compose/docker-compose-autoheal.yml b/dspace/src/main/docker-compose/docker-compose-autoheal.yml new file mode 100644 index 000000000000..0101b39c8ddc --- /dev/null +++ b/dspace/src/main/docker-compose/docker-compose-autoheal.yml @@ -0,0 +1,111 @@ +# +# The contents of this file are subject to the license and copyright +# detailed in the LICENSE and NOTICE files at the root of the source +# tree and available online at +# +# http://www.dspace.org/license/ +# + +# Optional sidecar that restarts unhealthy containers - with a restart budget. +# +# WHY THIS EXISTS +# Docker does NOT restart a container just because its healthcheck says "unhealthy". +# The `restart` policy only reacts to the main process exiting. A container can sit +# unhealthy indefinitely while `docker ps` still shows it running - exactly the +# "nothing crashed but something inside was broken" failure mode. +# +# WHY NOT willfarrell/autoheal +# That image has no restart budget: if a container can never become healthy, it +# restarts it forever. We hit this three times while building this setup - once on +# the backend (a config/version mismatch meant Spring never booted) and twice on the +# frontend (`ng serve` needs ~15 min to build; the healthcheck's start_period was too +# short, so the build was killed mid-flight and could never finish). In both cases the +# restarts made the problem HARDER to diagnose, because the container kept dying +# under the person reading its logs. +# +# WHAT THIS DOES INSTEAD +# Restarts an unhealthy container at most MAX_RESTARTS times within WINDOW_SECONDS. +# Past that it gives up, logs a loud one-off message, and leaves the container +# unhealthy so a human can look at it. The budget resets once the container reports +# healthy again. +# +# SECURITY: mounting /var/run/docker.sock grants this container effective root on the +# host. Acceptable for local development; for production either accept it deliberately, +# put a restricted socket proxy in front of it, or move to an orchestrator with +# built-in restart semantics (Swarm, Kubernetes livenessProbe + backoff). +# +# Usage: +# docker compose -p d7 -f docker-compose.yml \ +# -f dspace/src/main/docker-compose/docker-compose-autoheal.yml up -d + +networks: + # Default to using network named 'dspacenet' from docker-compose.yml. + default: + name: ${COMPOSE_PROJECT_NAME}_dspacenet + external: true +services: + autoheal: + container_name: autoheal + image: docker:27-cli + restart: always + environment: + # Only containers carrying this label are eligible + AUTOHEAL_LABEL: ${AUTOHEAL_LABEL:-autoheal} + # Seconds between polls of the Docker API + AUTOHEAL_INTERVAL: ${AUTOHEAL_INTERVAL:-10} + # Restart budget: at most MAX_RESTARTS restarts per WINDOW_SECONDS, per container + AUTOHEAL_MAX_RESTARTS: ${AUTOHEAL_MAX_RESTARTS:-3} + AUTOHEAL_WINDOW_SECONDS: ${AUTOHEAL_WINDOW_SECONDS:-3600} + volumes: + - /var/run/docker.sock:/var/run/docker.sock + entrypoint: + - /bin/sh + - '-c' + - | + set -eu + STATE=/tmp/autoheal + mkdir -p "$$STATE" + log() { echo "$$(date '+%Y-%m-%d %H:%M:%S') $$*"; } + log "watching label $$AUTOHEAL_LABEL=true; budget $$AUTOHEAL_MAX_RESTARTS restarts / $${AUTOHEAL_WINDOW_SECONDS}s" + while true; do + now=$$(date +%s) + + # Reset the budget for anything that recovered. + for cid in $$(docker ps -q --filter "label=$$AUTOHEAL_LABEL=true" --filter "health=healthy"); do + name=$$(docker inspect -f '{{.Name}}' "$$cid" | tr -d '/') + if [ -f "$$STATE/$$name.gaveup" ]; then log "$$name recovered - restart budget reset"; fi + rm -f "$$STATE/$$name" "$$STATE/$$name.gaveup" + done + + for cid in $$(docker ps -q --filter "label=$$AUTOHEAL_LABEL=true" --filter "health=unhealthy"); do + name=$$(docker inspect -f '{{.Name}}' "$$cid" | tr -d '/') + f="$$STATE/$$name" + + # Drop restart timestamps that fell out of the window. + if [ -f "$$f" ]; then + awk -v now="$$now" -v w="$$AUTOHEAL_WINDOW_SECONDS" 'now-$$1 < w' "$$f" > "$$f.tmp" || true + mv "$$f.tmp" "$$f" + else + : > "$$f" + fi + count=$$(wc -l < "$$f" | tr -d ' ') + + if [ "$$count" -ge "$$AUTOHEAL_MAX_RESTARTS" ]; then + # Loud once, then quiet - so the log stays readable while someone debugs. + if [ ! -f "$$f.gaveup" ]; then + touch "$$f.gaveup" + log "GIVING UP on $$name: $$count restarts in the last $${AUTOHEAL_WINDOW_SECONDS}s did not help." + log "GIVING UP on $$name: leaving it unhealthy for investigation. Restarting is not fixing this - check its logs." + fi + continue + fi + + log "$$name is unhealthy - restarting ($$((count+1))/$$AUTOHEAL_MAX_RESTARTS in window)" + if docker restart -t 10 "$$cid" >/dev/null 2>&1; then + echo "$$now" >> "$$f" + else + log "$$name restart FAILED" + fi + done + sleep "$$AUTOHEAL_INTERVAL" + done diff --git a/dspace/src/main/docker/dspace-solr/Dockerfile b/dspace/src/main/docker/dspace-solr/Dockerfile index ad3998fb9964..c7bf0a9479c4 100644 --- a/dspace/src/main/docker/dspace-solr/Dockerfile +++ b/dspace/src/main/docker/dspace-solr/Dockerfile @@ -37,4 +37,12 @@ RUN chown -R solr:solr /opt/solr/server/solr/configsets COPY scripts/log4j2.solr.xml /var/solr/log4j2.xml + +# curl: used by the container healthcheck in docker-compose.yml, which asks the CoreAdmin +# API whether the "search" core is registered. The solr:*-slim base ships curl today, but +# the probe must not silently depend on that: without it Solr would never report healthy, +# service_healthy would block the backend, and autoheal would restart a healthy container. +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* USER solr