Merge branch 'master' into marc/debian-release-tests

Author: Marketen

iso/scripts/common_iso_generation.sh

@@ -4,16 +4,34 @@ WORKDIR="/usr/src/app"
 ISO_BUILD_PATH="${WORKDIR}/dappnode-iso"
 DAPPNODE_ISO_PREFIX="Dappnode-"
 
+# Usage: download_iso <path> <name> <url> [fallback_url...]
+# URLs are tried in order, which covers mirrors that move older releases elsewhere.
 download_iso() {
     local iso_path=$1
     local iso_name=$2
-    local iso_url=$3
+    shift 2
 
     echo "[INFO] Downloading base ISO image: ${iso_name}..."
-    if [ ! -f "${iso_path}" ]; then
-        wget "${iso_url}" -O "${iso_path}"
+    if [ -f "${iso_path}" ]; then
+        echo "[INFO] Download complete!"
+        return 0
     fi
-    echo "[INFO] Download complete!"
+
+    local iso_url
+    for iso_url in "$@"; do
+        # Unlike busybox wget, curl moves on to the next address when a mirror
+        # host resolves to several and one of them is down.
+        if curl -fsSL --retry 3 --retry-connrefused --connect-timeout 30 -o "${iso_path}" "${iso_url}"; then
+            echo "[INFO] Download complete!"
+            return 0
+        fi
+        # A failed download leaves a partial file behind, which would be reused on the next run
+        rm -f "${iso_path}"
+        echo "[WARN] Could not download ${iso_url}"
+    done
+
+    echo "[ERROR] Could not download ${iso_name} from any known location"
+    exit 1
 }
 
 verify_download() {

iso/scripts/generate_dappnode_iso_debian.sh

@@ -9,7 +9,10 @@ BASE_ISO_NAME="debian-13.5.0-amd64-netinst.iso"
 BASE_ISO_VERSION="${BASE_ISO_NAME#debian-}"
 BASE_ISO_VERSION="${BASE_ISO_VERSION%-amd64-netinst.iso}"
 BASE_ISO_PATH="/images/${BASE_ISO_NAME}"
-BASE_ISO_URL="https://cdimage.debian.org/mirror/cdimage/archive/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}"
+# Debian serves the newest point release from debian-cd/ and only moves it to the
+# archive once it is superseded, so try both.
+BASE_ISO_URL="https://cdimage.debian.org/debian-cd/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}"
+BASE_ISO_ARCHIVE_URL="https://cdimage.debian.org/mirror/cdimage/archive/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}"
 BASE_ISO_SHASUM="95838884f5ea6c82421dfe6baaa5a639dbbe6756c1e380f9fe7a7cb0c1949d2a  ${BASE_ISO_PATH}"
 
 DAPPNODE_ISO_NAME="${DAPPNODE_ISO_PREFIX}${BASE_ISO_NAME}"
@@ -87,7 +90,7 @@ generate_debian_iso() {
         -isohybrid-gpt-basdat -o "${iso_output_path}" ${iso_build_path}
 }
 
-download_iso "${BASE_ISO_PATH}" "${BASE_ISO_NAME}" "${BASE_ISO_URL}"
+download_iso "${BASE_ISO_PATH}" "${BASE_ISO_NAME}" "${BASE_ISO_URL}" "${BASE_ISO_ARCHIVE_URL}"
 verify_download "${BASE_ISO_PATH}" "${BASE_ISO_SHASUM}"
 clean_old_files "${ISO_BUILD_PATH}" "${DAPPNODE_ISO_PREFIX}"
 extract_iso "${BASE_ISO_PATH}" "${ISO_BUILD_PATH}"

scripts/dappnode_install.sh

@@ -428,7 +428,17 @@ normalize_ipfs_version_ref() {
         local cid_path="$ref"
         local manifest_url="${IPFS_ENDPOINT%/}${cid_path}/dappnode_package.json"
         local manifest
-        manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)"
+        # On the first boot after an ISO install this runs from rc.local, which
+        # does not wait for DHCP (Debian's allow-hotplug interfaces are not part
+        # of network-online.target), so give the network up to ~2 minutes.
+        local attempt
+        for attempt in $(seq 1 24); do
+            manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)"
+            [[ -n "$manifest" ]] && break
+            # stdout is this function's return value, so report on stderr
+            [[ "$attempt" -eq 1 ]] && warn "Could not fetch IPFS manifest for ${comp} yet, retrying..." >&2
+            [[ "$attempt" -lt 24 ]] && sleep 5
+        done
         if [[ -z "$manifest" ]]; then
             error "Could not fetch IPFS manifest for ${comp} from: $manifest_url"
             error "Provide ${comp}_VERSION as /ipfs/<cid>:<version> (example: /ipfs/Qm...:0.2.11)"

scripts/dappnode_install_pre.sh

@@ -107,8 +107,21 @@ install_iptables() {
     fi
 }
 
+# The Debian installer comments out the install media's cdrom: APT source only
+# after the late-commands have run. Until then apt-get update fails on it, so
+# disable it now as the installer would (apt-setup/disable-cdrom-entries).
+disable_iso_cdrom_sources() {
+    [ "$ISO_INSTALLATION" = "true" ] || return 0
+    local sources_file
+    for sources_file in /etc/apt/sources.list /etc/apt/sources.list.d/*.list; do
+        [ -f "$sources_file" ] || continue
+        sed -i -E 's/^(deb(-src)?[[:space:]]+(\[[^]]*\][[:space:]]+)?cdrom:)/# \1/' "$sources_file"
+    done
+}
+
 # HOST UPDATE
 host_update() {
+    disable_iso_cdrom_sources
     # Process substitution keeps apt-get's exit status while still logging its
     # output. A regular pipeline would return tee's status and hide dpkg errors.
     apt-get update > >(tee -a "$LOG_FILE") 2>&1 || return $?