Add compressed firmware updates installed into an execute-in-place slot

Author: dgarske

.github/workflows/test-compressed-simulator.yml

@@ -0,0 +1,145 @@
+name: Compressed images - test with simulator target
+
+on:
+  push:
+    branches: [ 'master', 'main', 'release/**' ]
+  pull_request:
+    branches: [ '*' ]
+
+jobs:
+  compressed_update_simulator_tests:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/wolfssl/wolfboot-ci-sim:v1.0
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        # The sink decoder is a separate output path in src/gzip.c, and
+        # GZIP_SMALL selects a different Huffman decoder behind it, so both
+        # have to be exercised and not merely built. The window sizes the
+        # decoder's buffer and bounds back-references, so the smallest and
+        # the default are both run.
+        sign: [ ED25519, ECC256 ]
+        window: [ 4096, 16384 ]
+        # The container on internal flash is read in place; on external
+        # flash it is pulled in through the read callback, which is a
+        # different path through the decoder's input.
+        config: [ sim-compressed-update, sim-compressed-update-ext,
+                  sim-compressed-update-ext-enc ]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Trust workspace
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: make clean
+        run: |
+          make distclean
+
+      - name: Install zlib headers
+        run: |
+          apt-get update && apt-get install -y zlib1g-dev
+
+      - name: Select config
+        run: |
+          cp config/examples/${{ matrix.config }}.config .config
+          sed -i "s/^SIGN?=.*/SIGN?=${{ matrix.sign }}/" .config
+          sed -i "s/^WOLFBOOT_COMPRESSED_WINDOW=.*//" .config
+          echo "WOLFBOOT_COMPRESSED_WINDOW=${{ matrix.window }}" >> .config
+
+      - name: Build tools
+        run: |
+          make -C tools/keytools && make -C tools/bin-assemble
+
+      # Not a bare `make`: that also assembles internal_flash.dd with the
+      # update partition inside it, which is wrong when the partition lives
+      # on the external device. The test target below builds both images.
+      - name: Build wolfboot.elf and test-app/image.elf
+        run: |
+          make clean && make wolfboot.elf test-app/image.elf
+
+      - name: Run the compressed update simulator test
+        run: |
+          ./tools/scripts/sim-compressed-update.sh
+
+  compressed_update_cross_build:
+    runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/wolfssl/wolfboot-ci-arm:v1.0
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        config: [ stm32h7-compressed-update, stm32f439zi-compressed-update ]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Trust workspace
+        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
+
+      - name: Install zlib headers
+        run: |
+          apt-get update && apt-get install -y zlib1g-dev
+
+      - name: Select config
+        run: |
+          cp config/examples/${{ matrix.config }}.config .config
+
+      - name: Build
+        run: |
+          make clean && make keysclean && make
+
+      # The installer walks the boot partition a sector at a time and the
+      # trailer lives in its last sector, so a partition that is not a whole
+      # number of sectors would have its tail erased by the trailer write.
+      - name: Check the partition geometry
+        run: |
+          BOOT=$(grep -E '^WOLFBOOT_PARTITION_SIZE\??=' .config | tail -1 | cut -d= -f2)
+          UPD=$(grep -E '^WOLFBOOT_PARTITION_UPDATE_SIZE\??=' .config | tail -1 | cut -d= -f2)
+          SEC=$(grep -E '^WOLFBOOT_SECTOR_SIZE\??=' .config | tail -1 | cut -d= -f2)
+          echo "boot $BOOT, update $UPD, sector $SEC"
+          [ $(($BOOT % $SEC)) -eq 0 ] || { echo "boot not sector aligned"; exit 1; }
+          [ $(($UPD % $SEC)) -eq 0 ] || { echo "update not sector aligned"; exit 1; }
+          # The point of the mode: the update partition is the smaller one.
+          [ $(($UPD)) -lt $(($BOOT)) ] || { echo "update is not smaller than boot"; exit 1; }
+          echo "ok: update is $(( 100 * $UPD / $BOOT ))% of boot"
+
+      # The window knob has to reach the decoder's buffer, which is the RAM
+      # the mode actually costs.
+      - name: Check the decoder window buffer was sized from the config
+        run: |
+          WIN=$(grep -E '^WOLFBOOT_COMPRESSED_WINDOW\??=' .config | tail -1 | cut -d= -f2)
+          SZ=$(arm-none-eabi-nm -S wolfboot.elf | awk '$4=="compressed_window" {print $2}')
+          [ -n "$SZ" ] || { echo "compressed_window not linked in"; exit 1; }
+          WANT=$((2 * $WIN))
+          [ $((0x$SZ)) -eq $WANT ] || \
+            { echo "compressed_window is $((0x$SZ)) bytes, expected $WANT"; exit 1; }
+          echo "ok: compressed_window is $((0x$SZ)) bytes for a $WIN window"
+
+      # Signing has to refuse a container the update partition cannot hold,
+      # so a release that compresses worse than the split fails here rather
+      # than on the device.
+      - name: Check a container is produced and fits
+        run: |
+          UPD=$(grep -E '^WOLFBOOT_PARTITION_UPDATE_SIZE\??=' .config | tail -1 | cut -d= -f2)
+          SEC=$(grep -E '^WOLFBOOT_SECTOR_SIZE\??=' .config | tail -1 | cut -d= -f2)
+          SIGN=$(grep -E '^SIGN\??=' .config | tail -1 | cut -d= -f2 | tr 'A-Z' 'a-z')
+          HASH=$(grep -E '^HASH\??=' .config | tail -1 | cut -d= -f2 | tr 'A-Z' 'a-z')
+          WIN=$(grep -E '^WOLFBOOT_COMPRESSED_WINDOW\??=' .config | tail -1 | cut -d= -f2)
+          WOLFBOOT_PARTITION_SIZE=$(grep -E '^WOLFBOOT_PARTITION_SIZE\??=' .config | tail -1 | cut -d= -f2) \
+          WOLFBOOT_PARTITION_UPDATE_SIZE=$UPD WOLFBOOT_SECTOR_SIZE=$SEC \
+          WOLFBOOT_COMPRESSED_WINDOW=$WIN \
+            ./tools/keytools/sign --compress --$SIGN --$HASH \
+              test-app/image.elf wolfboot_signing_private_key.der 2
+          SZ=$(stat -c%s test-app/image_v2_signed_compressed.bin)
+          LIMIT=$(( $(($UPD)) - $(($SEC)) ))
+          echo "container $SZ bytes, limit $LIMIT"
+          [ "$SZ" -le "$LIMIT" ] || { echo "container does not fit"; exit 1; }
+          echo "ok: container fits the update partition"

Makefile

@@ -362,6 +362,7 @@ endif
 
 # Environment variables for sign tool
 SIGN_ENV=IMAGE_HEADER_SIZE=$(IMAGE_HEADER_SIZE) \
+		 WOLFBOOT_COMPRESSED_WINDOW=$(WOLFBOOT_COMPRESSED_WINDOW) \
 		 WOLFBOOT_PARTITION_SIZE=$(WOLFBOOT_PARTITION_SIZE) \
 		 WOLFBOOT_PARTITION_UPDATE_SIZE=$(WOLFBOOT_PARTITION_UPDATE_SIZE) \
 		 WOLFBOOT_SECTOR_SIZE=$(WOLFBOOT_SECTOR_SIZE) \

config/examples/sim-compressed-update-ext-enc.config

@@ -0,0 +1,35 @@
+ARCH=sim
+TARGET=sim
+SIGN?=ED25519
+HASH?=SHA256
+WOLFBOOT_SMALL_STACK?=0
+SPI_FLASH=0
+EXT_FLASH=1
+ENCRYPT=1
+ENCRYPT_WITH_AES128=1
+DEBUG=1
+
+# Compressed update partition on external flash, encrypted at rest, boot
+# partition executed in place from internal flash. ENCRYPT=1 hooks the
+# external flash read, which is the same seam the installer and the image
+# hasher pull the container through, so it is decrypted on its way in.
+COMPRESSED_UPDATE=1
+GZIP=1
+
+# LZ77 window the image was compressed with. The decoder needs twice this in
+# RAM and refuses an image compressed with a larger one.
+WOLFBOOT_COMPRESSED_WINDOW=16384
+
+# The update partition holds only the container, so it is sized for that.
+# sizes should be multiple of system page size
+WOLFBOOT_PARTITION_SIZE=0x50000
+WOLFBOOT_PARTITION_UPDATE_SIZE=0x2C000
+WOLFBOOT_SECTOR_SIZE=0x1000
+WOLFBOOT_PARTITION_BOOT_ADDRESS=0x80000
+# on external flash, so it starts at the base of that device
+WOLFBOOT_PARTITION_UPDATE_ADDRESS=0x00000
+# Unused in this mode: there is no swap step.
+WOLFBOOT_PARTITION_SWAP_ADDRESS=0x180000
+
+# required for keytools
+WOLFBOOT_FIXED_PARTITIONS=1

config/examples/sim-compressed-update-ext.config

@@ -0,0 +1,34 @@
+ARCH=sim
+TARGET=sim
+SIGN?=ED25519
+HASH?=SHA256
+WOLFBOOT_SMALL_STACK?=0
+SPI_FLASH=0
+EXT_FLASH=1
+DEBUG=1
+
+# Compressed update partition on external flash, boot partition executed in
+# place from internal flash. This is the common microcontroller layout: a
+# small internal flash for the application and a cheap serial flash for the
+# incoming update. The container is not addressable, so wolfBoot pulls it in
+# through the external flash reader as it inflates.
+COMPRESSED_UPDATE=1
+GZIP=1
+
+# LZ77 window the image was compressed with. The decoder needs twice this in
+# RAM and refuses an image compressed with a larger one.
+WOLFBOOT_COMPRESSED_WINDOW=16384
+
+# The update partition holds only the container, so it is sized for that.
+# sizes should be multiple of system page size
+WOLFBOOT_PARTITION_SIZE=0x50000
+WOLFBOOT_PARTITION_UPDATE_SIZE=0x2C000
+WOLFBOOT_SECTOR_SIZE=0x1000
+WOLFBOOT_PARTITION_BOOT_ADDRESS=0x80000
+# on external flash, so it starts at the base of that device
+WOLFBOOT_PARTITION_UPDATE_ADDRESS=0x00000
+# Unused in this mode: there is no swap step.
+WOLFBOOT_PARTITION_SWAP_ADDRESS=0x180000
+
+# required for keytools
+WOLFBOOT_FIXED_PARTITIONS=1

config/examples/sim-compressed-update.config

@@ -0,0 +1,38 @@
+ARCH=sim
+TARGET=sim
+SIGN?=ED25519
+HASH?=SHA256
+WOLFBOOT_SMALL_STACK?=0
+SPI_FLASH=0
+DEBUG=1
+
+# Compressed update partition, execute in place. UPDATE holds a signed gzip
+# stream whose payload is a complete signed image; wolfBoot verifies the
+# compressed bytes, inflates them into BOOT a chunk at a time, and BOOT is
+# then an ordinary partition that the normal boot path verifies.
+COMPRESSED_UPDATE=1
+GZIP=1
+
+# LZ77 window the image was compressed with. The decoder needs twice this in
+# RAM and refuses an image compressed with a larger one.
+WOLFBOOT_COMPRESSED_WINDOW=16384
+
+# UPDATE only ever holds the compressed image, so it is sized for that and
+# the flash a second full copy would have cost goes to BOOT instead. The
+# signing tool refuses to emit an artifact that does not fit.
+# sizes should be multiple of system page size
+# Sized for a realistic ratio: the test fixture compresses to about 63%,
+# so the container fills roughly 90% of the update partition and the fit
+# check is exercised near its real margin rather than with slack.
+WOLFBOOT_PARTITION_SIZE=0x50000
+WOLFBOOT_PARTITION_UPDATE_SIZE=0x2C000
+WOLFBOOT_SECTOR_SIZE=0x1000
+WOLFBOOT_PARTITION_BOOT_ADDRESS=0x80000
+# if on external flash, it should be multiple of system page size
+WOLFBOOT_PARTITION_UPDATE_ADDRESS=0x100000
+# Unused in this mode: there is no swap step. Still required by the keytools
+# and the partition trailer layout, so it stays reserved.
+WOLFBOOT_PARTITION_SWAP_ADDRESS=0x180000
+
+# required for keytools
+WOLFBOOT_FIXED_PARTITIONS=1

config/examples/stm32f439zi-compressed-update.config

@@ -0,0 +1,61 @@
+# NUCLEO-F439ZI with a compressed update partition, executing in place.
+#
+# The update partition only ever holds the compressed container, so it is
+# sized for that and the flash a second full copy would have cost goes to
+# the application: 512KB of executable instead of the 448KB an equal split
+# of the same space would allow.
+#
+# Everything stays inside the first 1MB because hal/stm32f4.c describes only
+# that much flash. The F439ZI has 2MB, but the second bank's sector numbers
+# are not contiguous with the first's, so this HAL cannot erase there; the
+# HAL now refuses such a layout at build time rather than failing silently.
+#
+# The split has to cover the worst ratio the product will ever ship. 512/384
+# leaves room for a container up to 75% of the image; measured embedded
+# images land near 65-70%. The signing tool refuses an artifact that does
+# not fit, so a release that compresses worse than the split fails at build
+# time, not on the device.
+#
+# Console is USART3 on PD8/PD9, the ST-LINK virtual COM port on the
+# NUCLEO-144 boards, via hal/uart/uart_drv_stm32f4.c.
+ARCH?=ARM
+TARGET?=stm32f4
+SIGN?=ED25519
+HASH?=SHA256
+DEBUG?=1
+DEBUG_UART?=1
+CFLAGS_EXTRA+=-DUSE_UART3
+VTOR?=1
+NVM_FLASH_WRITEONCE?=0
+SPMATH?=1
+
+# 168MHz from an 8MHz HSE, the NUCLEO-144 default.
+CLOCK_SPEED?=168000000
+STM32_PLLM?=8
+STM32_PLLN?=336
+STM32_PLLP?=2
+STM32_PLLQ?=7
+
+# Compressed update partition, inflated into BOOT, which keeps executing in
+# place. Implies DISABLE_BACKUP: there is no swap and no backup of the
+# outgoing image. An interrupted install restarts from an intact source,
+# because the update partition is only ever read.
+COMPRESSED_UPDATE?=1
+GZIP?=1
+GZIP_SMALL?=0
+
+# LZ77 window the image was compressed with. The decoder needs twice this
+# in RAM, so 16KB costs 32KB of wolfBoot bss on a part with 192KB of SRAM.
+# Sign with 'sign --compress=14' to match.
+WOLFBOOT_COMPRESSED_WINDOW?=16384
+
+WOLFBOOT_SECTOR_SIZE?=0x20000
+WOLFBOOT_PARTITION_SIZE?=0x80000
+WOLFBOOT_PARTITION_UPDATE_SIZE?=0x60000
+WOLFBOOT_PARTITION_BOOT_ADDRESS?=0x08020000
+WOLFBOOT_PARTITION_UPDATE_ADDRESS?=0x080A0000
+# No swap step in this mode, so no swap partition is reserved.
+WOLFBOOT_PARTITION_SWAP_ADDRESS?=0
+
+# required for keytools
+WOLFBOOT_FIXED_PARTITIONS=1

config/examples/stm32h7-compressed-update.config

@@ -0,0 +1,54 @@
+# STM32H7 with a compressed update partition, executing in place.
+#
+# The H743 has 2MB of flash in 128KB sectors. The update partition only ever
+# holds the compressed container, so it is sized for that and the flash a
+# second full copy would have cost goes to the application: 1MB of
+# executable against the 896KB an equal split of the same 15 sectors would
+# allow, and against the 832KB the stock stm32h7.config configures.
+#
+# The split has to cover the worst ratio the product will ever ship. 8/7
+# leaves room for a container up to 87.5% of the image, which no real
+# firmware approaches; measured embedded images land near 65-70%. The
+# signing tool refuses an artifact that does not fit, so a release that
+# compresses worse than the split fails at build time, not on the device.
+ARCH?=ARM
+TARGET?=stm32h7
+SIGN?=ECC256
+HASH?=SHA256
+DEBUG?=0
+DEBUG_UART?=0
+VTOR?=1
+NO_ASM?=0
+EXT_FLASH?=0
+SPI_FLASH?=0
+ALLOW_DOWNGRADE?=0
+NVM_FLASH_WRITEONCE?=0
+WOLFBOOT_VERSION?=1
+V?=0
+SPMATH?=1
+RAM_CODE?=0
+DUALBANK_SWAP?=0
+
+# Compressed update partition, inflated into BOOT, which keeps executing in
+# place. Implies DISABLE_BACKUP: there is no swap and no backup of the
+# outgoing image. An interrupted install restarts from an intact source,
+# because the update partition is only ever read.
+COMPRESSED_UPDATE?=1
+GZIP?=1
+GZIP_SMALL?=0
+
+# LZ77 window the image was compressed with. The decoder needs twice this
+# in RAM; wolfBoot runs from the 128KB DTCM here, so 32KB of buffer plus the
+# decoder's tables fits comfortably. Sign with 'sign --compress=14'.
+WOLFBOOT_COMPRESSED_WINDOW?=16384
+
+WOLFBOOT_SECTOR_SIZE?=0x20000
+WOLFBOOT_PARTITION_SIZE?=0x100000
+WOLFBOOT_PARTITION_UPDATE_SIZE?=0xE0000
+WOLFBOOT_PARTITION_BOOT_ADDRESS?=0x8020000
+WOLFBOOT_PARTITION_UPDATE_ADDRESS?=0x8120000
+# No swap step in this mode, so no swap partition is reserved.
+WOLFBOOT_PARTITION_SWAP_ADDRESS?=0
+
+# required for keytools
+WOLFBOOT_FIXED_PARTITIONS=1