@@ -673,7 +673,8 @@ jobs:
673673 timeout-minutes : 30
674674 env :
675675 ENGINE : ${{ matrix.engine }}
676- # Read by the apt plan step; see the note above it.
676+ # Read by util/apt_packages.py's plan and install; see the note above
677+ # the plan step.
677678 APT_PACKAGES : ${{ matrix.engine == 'PyShark' && 'tshark' || matrix.engine == 'PCAP_CT' && 'libpcap0.8' || matrix.engine == 'PyPCAP' && 'build-essential libpcap-dev' || '' }}
678679 APT_SET : ${{ matrix.engine }}
679680 APT_CACHE_SAVE : ' true'
@@ -768,34 +769,19 @@ jobs:
768769 # on a hit, so a real install failure still fails; a cache-service error
769770 # is only a warning from actions/cache. The 180s bound is unchanged: with
770771 # the above it binds only on a cold miss, and TestAptRetry in
771- # tests/project/test_unit_tests_queue.py pins it. APT_PACKAGES ( the job's
772- # env, read by the plan step) must name what the install step installs,
773- # or the key describes the wrong set -- a lost cache, not a wrong install .
772+ # tests/project/test_unit_tests_queue.py pins it. APT_PACKAGES, the job's
773+ # env, is what both the plan and the install read, so the key always
774+ # describes the set installed .
774775 #
775- # `pypcap-parity` runs the same steps; every body but the install's is
776- # byte-identical to these.
776+ # The plan, seed, install and collect steps run util/apt_packages.py
777+ # (#1967), which runs the commands above under `sudo` and writes the same
778+ # annotations; its docstring says what each subcommand does. `pypcap-parity`
779+ # runs the same steps; every body but the install's is identical to these.
777780 - name : Plan this engine's system packages
778781 id : apt-plan
779782 if : env.APT_PACKAGES != ''
780783 timeout-minutes : 4
781- run : |
782- set -eu
783- for attempt in 1 2; do
784- if sudo timeout 60 apt-get update; then
785- break
786- fi
787- echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound."
788- [ "$attempt" = 1 ] || exit 1
789- done
790- # shellcheck disable=SC2086
791- sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES)
792- plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true)
793- printf 'To download:\n%s\n' "${plan:-(nothing)}"
794- if [ -n "$plan" ]; then
795- prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-"
796- echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT"
797- echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
798- fi
784+ run : /usr/bin/python3 util/apt_packages.py plan
799785
800786 - name : Restore this engine's cached .deb files
801787 id : apt-cache
@@ -808,40 +794,11 @@ jobs:
808794
809795 - name : Seed apt's archive with this engine's cached .deb files
810796 if : steps.apt-cache.outputs.cache-matched-key != ''
811- run : |
812- set -eu
813- sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/
814- echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files"
797+ run : /usr/bin/python3 util/apt_packages.py seed
815798
816799 - name : Install this engine's system packages
817800 timeout-minutes : 10
818- run : |
819- set -eu
820- mirrors=/etc/apt/apt-mirrors.txt
821- apt_install() {
822- for attempt in 1 2; do
823- if sudo timeout 60 apt-get update \
824- && sudo DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y --no-install-recommends "$@"; then
825- return 0
826- fi
827- echo "::warning title=apt attempt $attempt of 2 failed (${ENGINE})::apt-get update or install failed or exceeded its per-attempt bound; retrying once."
828- sudo dpkg --configure -a || true
829- if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \
830- && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \
831- && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then
832- echo "::warning title=apt retry on a fallback mirror (${ENGINE})::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download."
833- fi
834- done
835- return 1
836- }
837- if [ "$ENGINE" = "PyShark" ]; then
838- echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections
839- apt_install tshark
840- elif [ "$ENGINE" = "PCAP_CT" ]; then
841- apt_install libpcap0.8
842- elif [ "$ENGINE" = "PyPCAP" ]; then
843- apt_install build-essential libpcap-dev
844- fi
801+ run : /usr/bin/python3 util/apt_packages.py install --name "$ENGINE"
845802
846803 # #2019: from Ubuntu 25.04, apparmor 4.1 ships an enforcing profile for
847804 # /usr/bin/tshark (/etc/apparmor.d/tshark) whose own file rules read only
@@ -867,22 +824,7 @@ jobs:
867824 if : >-
868825 env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != ''
869826 && steps.apt-cache.outputs.cache-hit != 'true'
870- run : |
871- set -eu
872- out="$RUNNER_TEMP/apt-debs"
873- rm -rf "$out"
874- mkdir -p "$out"
875- for deb in /var/cache/apt/archives/*.deb; do
876- [ -e "$deb" ] || continue
877- pkg=$(dpkg-deb -f "$deb" Package)
878- ver=$(dpkg-deb -f "$deb" Version)
879- arch=$(dpkg-deb -f "$deb" Architecture)
880- # shellcheck disable=SC2016
881- if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then
882- cp "$deb" "$out/"
883- fi
884- done
885- echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files"
827+ run : /usr/bin/python3 util/apt_packages.py collect
886828
887829 - name : Save this engine's .deb files to the cache
888830 if : >-
@@ -1117,7 +1059,8 @@ jobs:
11171059 runs-on : ubuntu-latest
11181060 timeout-minutes : 45
11191061 env :
1120- # Read by the apt plan step; see `engine-tests`'s note on it.
1062+ # Read by util/apt_packages.py's plan and install; see `engine-tests`'s
1063+ # note on it.
11211064 APT_PACKAGES : build-essential libpcap-dev tshark
11221065 APT_SET : pypcap-parity
11231066 # Every cell installs the same set under the same key; only one saves it,
@@ -1155,11 +1098,10 @@ jobs:
11551098 # *_runtime.py wholesale. Without tshark, PyShark.unsupported_reason()
11561099 # declines the engine, the extractor silently falls back to the default
11571100 # parser, and the test's assertions used to pass on that fallback's
1158- # output -- a vacuous pass, invisible in skip counts. The debconf
1159- # pre-seed and DEBIAN_FRONTEND mirror `engine-tests`'s "Install this
1160- # engine's system packages" step exactly, for the same reason: tshark's
1161- # postinst otherwise blocks on the "allow non-superusers to capture
1162- # packets" prompt.
1101+ # output -- a vacuous pass, invisible in skip counts. The install
1102+ # pre-seeds debconf and sets DEBIAN_FRONTEND as `engine-tests`'s does,
1103+ # for the same reason: tshark's postinst otherwise blocks on the "allow
1104+ # non-superusers to capture packets" prompt.
11631105 #
11641106 # The apt steps -- plan, restore, seed, install, collect, save -- are
11651107 # `engine-tests`'s, for the reasons in the note above its plan step:
@@ -1174,24 +1116,7 @@ jobs:
11741116 if : env.APT_PACKAGES != ''
11751117 continue-on-error : ${{ matrix.ktlo == true }}
11761118 timeout-minutes : 4
1177- run : |
1178- set -eu
1179- for attempt in 1 2; do
1180- if sudo timeout 60 apt-get update; then
1181- break
1182- fi
1183- echo "::warning title=apt-get update attempt $attempt of 2 failed (${APT_SET})::apt-get update failed or exceeded its 60s bound."
1184- [ "$attempt" = 1 ] || exit 1
1185- done
1186- # shellcheck disable=SC2086
1187- sim=$(apt-get install -s --no-install-recommends $APT_PACKAGES)
1188- plan=$(printf '%s\n' "$sim" | grep '^Inst ' | sort || true)
1189- printf 'To download:\n%s\n' "${plan:-(nothing)}"
1190- if [ -n "$plan" ]; then
1191- prefix="apt-debs-${ImageOS:-unknown}-${RUNNER_ARCH}-${APT_SET}-"
1192- echo "restore-prefix=$prefix" >> "$GITHUB_OUTPUT"
1193- echo "key=${prefix}$(printf '%s\n' "$plan" | sha256sum | cut -c1-16)" >> "$GITHUB_OUTPUT"
1194- fi
1119+ run : /usr/bin/python3 util/apt_packages.py plan
11951120
11961121 - name : Restore the cached .deb files
11971122 id : apt-cache
@@ -1207,33 +1132,13 @@ jobs:
12071132 id : apt-seed
12081133 if : steps.apt-cache.outputs.cache-matched-key != ''
12091134 continue-on-error : ${{ matrix.ktlo == true }}
1210- run : |
1211- set -eu
1212- sudo cp "$RUNNER_TEMP"/apt-debs/*.deb /var/cache/apt/archives/
1213- echo "Seeded $(find "$RUNNER_TEMP/apt-debs" -name '*.deb' | wc -l) .deb files"
1135+ run : /usr/bin/python3 util/apt_packages.py seed
12141136
12151137 - name : Install libpcap headers, a C toolchain, and tshark
12161138 id : apt-install
12171139 continue-on-error : ${{ matrix.ktlo == true }}
12181140 timeout-minutes : 10
1219- run : |
1220- set -eu
1221- echo "wireshark-common wireshark-common/install-setuid boolean false" | sudo debconf-set-selections
1222- mirrors=/etc/apt/apt-mirrors.txt
1223- for attempt in 1 2; do
1224- if sudo timeout 60 apt-get update \
1225- && sudo DEBIAN_FRONTEND=noninteractive timeout 180 apt-get install -y --no-install-recommends build-essential libpcap-dev tshark; then
1226- exit 0
1227- fi
1228- echo "::warning title=apt attempt $attempt of 2 failed::apt-get update or install failed or exceeded its per-attempt bound; retrying once."
1229- sudo dpkg --configure -a || true
1230- if [ "$attempt" = 1 ] && [ -f "$mirrors" ] \
1231- && grep -v 'azure\.archive\.ubuntu\.com' "$mirrors" | grep -q '^http' \
1232- && sudo sed -i '/azure\.archive\.ubuntu\.com/d' "$mirrors"; then
1233- echo "::warning title=apt retry on a fallback mirror::Retrying against $(head -n 1 "$mirrors" | cut -f 1) instead of azure.archive.ubuntu.com, resuming the partial download."
1234- fi
1235- done
1236- exit 1
1141+ run : /usr/bin/python3 util/apt_packages.py install
12371142
12381143 # #2019: tshark's AppArmor profile on Ubuntu 25.04 and later; see
12391144 # `engine-tests`'s step of the same name, whose body this is.
@@ -1252,22 +1157,7 @@ jobs:
12521157 if : >-
12531158 env.APT_CACHE_SAVE == 'true' && steps.apt-plan.outputs.key != ''
12541159 && steps.apt-cache.outputs.cache-hit != 'true'
1255- run : |
1256- set -eu
1257- out="$RUNNER_TEMP/apt-debs"
1258- rm -rf "$out"
1259- mkdir -p "$out"
1260- for deb in /var/cache/apt/archives/*.deb; do
1261- [ -e "$deb" ] || continue
1262- pkg=$(dpkg-deb -f "$deb" Package)
1263- ver=$(dpkg-deb -f "$deb" Version)
1264- arch=$(dpkg-deb -f "$deb" Architecture)
1265- # shellcheck disable=SC2016
1266- if [ "$(dpkg-query -W -f '${Version}' "$pkg:$arch" 2>/dev/null || true)" = "$ver" ]; then
1267- cp "$deb" "$out/"
1268- fi
1269- done
1270- echo "Collected $(find "$out" -name '*.deb' | wc -l) .deb files"
1160+ run : /usr/bin/python3 util/apt_packages.py collect
12711161
12721162 - name : Save the .deb files to the cache
12731163 if : >-
0 commit comments