Skip to content

Commit fd0d2e7

Browse files
authored
ci(release): run the tag checks and the release upload from util/ scripts (#1967) (#2065)
Step 2 of #1967, first batch for create-release.yml: the shell its jobs repeated moves into two stdlib-only scripts, with each step's env:, if:, shell: and continue-on-error unchanged. * util/release_tags.py check: "Check the release tags name this run's commit" in github, pypi and conda; release_tags.py owner: "Check which commit the version's tag names" in version_check. Both resolve the tag through one function. * util/release_upload.py: "Upload package to GitHub Release" in pypi and conda. * workflows.rst lists both scripts, and the interpreter each runs on. tests/project/test_release_scripts_1967.py runs both against stand-in git and gh, and runs the former shell on the same cases, requiring the same status, output, $GITHUB_OUTPUT and calls. The #1634 tests now run the steps' commands, and PUBLISHING_MARKERS matches the upload script.
1 parent 27e6af1 commit fd0d2e7

8 files changed

Lines changed: 876 additions & 134 deletions

File tree

‎.github/workflows/create-release.yml‎

Lines changed: 6 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -402,20 +402,7 @@ jobs:
402402
env:
403403
PCAPKIT_TAG: v${{ steps.get_version.outputs.PCAPKIT_VERSION }}
404404
PCAPKIT_RUN_SHA: ${{ github.sha }}
405-
run: |
406-
set -euo pipefail
407-
408-
run_commit="$(git rev-parse --verify "${PCAPKIT_RUN_SHA}^{commit}")"
409-
git fetch --quiet --force origin "refs/tags/${PCAPKIT_TAG}:refs/tags/${PCAPKIT_TAG}" || true
410-
tagged="$(git rev-parse -q --verify "refs/tags/${PCAPKIT_TAG}^{commit}" || true)"
411-
elsewhere=false
412-
if [ -n "$tagged" ] && [ "$tagged" != "$run_commit" ]; then
413-
elsewhere=true
414-
fi
415-
416-
echo "${PCAPKIT_TAG} -> ${tagged:-no commit}; this run's commit is ${run_commit}; released elsewhere: ${elsewhere}"
417-
echo "PCAPKIT_RELEASED_AT=${tagged}" >> "$GITHUB_OUTPUT"
418-
echo "PCAPKIT_RELEASED_ELSEWHERE=${elsewhere}" >> "$GITHUB_OUTPUT"
405+
run: python3 util/release_tags.py owner
419406

420407
outputs:
421408
PCAPKIT_VERSION: ${{ steps.get_version.outputs.PCAPKIT_VERSION }}
@@ -563,19 +550,7 @@ jobs:
563550
env:
564551
PCAPKIT_TAGS: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}
565552
PCAPKIT_RUN_SHA: ${{ github.sha }}
566-
run: |
567-
set -euo pipefail
568-
569-
run_commit="$(git rev-parse --verify "${PCAPKIT_RUN_SHA}^{commit}")"
570-
for tag in ${PCAPKIT_TAGS}; do
571-
git fetch --quiet --force origin "refs/tags/${tag}:refs/tags/${tag}" || true
572-
tagged="$(git rev-parse -q --verify "refs/tags/${tag}^{commit}" || true)"
573-
if [ "$tagged" != "$run_commit" ]; then
574-
echo "::error title=${tag} is not this run's commit (#1634)::${tag} resolves to '${tagged:-no commit}', but this run was created for ${run_commit}. Refusing to publish this tree under it: every artefact of a release comes from its tag's commit, so finish that release by re-running the run created for that commit."
575-
exit 1
576-
fi
577-
echo "${tag} -> ${run_commit}"
578-
done
553+
run: python3 util/release_tags.py check
579554

580555
# #1634: a refusal is most likely the workflow-file rule quoted at "Mint
581556
# the release App's token", which GitHub's 2023-11-02 changelog words more
@@ -793,19 +768,7 @@ jobs:
793768
env:
794769
PCAPKIT_TAGS: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}
795770
PCAPKIT_RUN_SHA: ${{ github.sha }}
796-
run: |
797-
set -euo pipefail
798-
799-
run_commit="$(git rev-parse --verify "${PCAPKIT_RUN_SHA}^{commit}")"
800-
for tag in ${PCAPKIT_TAGS}; do
801-
git fetch --quiet --force origin "refs/tags/${tag}:refs/tags/${tag}" || true
802-
tagged="$(git rev-parse -q --verify "refs/tags/${tag}^{commit}" || true)"
803-
if [ "$tagged" != "$run_commit" ]; then
804-
echo "::error title=${tag} is not this run's commit (#1634)::${tag} resolves to '${tagged:-no commit}', but this run was created for ${run_commit}. Refusing to publish this tree under it: every artefact of a release comes from its tag's commit, so finish that release by re-running the run created for that commit."
805-
exit 1
806-
fi
807-
echo "${tag} -> ${run_commit}"
808-
done
771+
run: python3 util/release_tags.py check
809772

810773
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
811774
id: setup
@@ -881,35 +844,7 @@ jobs:
881844
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
882845
PCAPKIT_RELEASE_TAG: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}
883846
PCAPKIT_ASSETS: dist/*
884-
run: |
885-
set -euo pipefail
886-
887-
published="$(gh release view "$PCAPKIT_RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --json assets --jq '.assets[] | [.name, .state] | @tsv')"
888-
while IFS= read -r pattern; do
889-
[ -n "$pattern" ] || continue
890-
for file in $pattern; do
891-
if [ ! -f "$file" ]; then
892-
echo "::error title=Nothing to upload::'${pattern}' matches no file, so ${PCAPKIT_RELEASE_TAG} would go without it."
893-
exit 1
894-
fi
895-
name="${file##*/}"
896-
state=""
897-
while IFS=$'\t' read -r asset asset_state; do
898-
[ "$asset" != "$name" ] || state="$asset_state"
899-
done <<< "$published"
900-
if [ "$state" = uploaded ]; then
901-
echo "::notice title=${name} already on ${PCAPKIT_RELEASE_TAG}::Left as published (#1640): this run's rebuild is not byte-identical to it."
902-
continue
903-
fi
904-
if [ -n "$state" ]; then
905-
echo "::warning title=${name} uploaded again::${PCAPKIT_RELEASE_TAG} holds it in state '${state}', not 'uploaded' -- an upload that failed partway -- so it is deleted and this file uploaded in its place."
906-
gh release upload "$PCAPKIT_RELEASE_TAG" "$file" --repo "$GITHUB_REPOSITORY" --clobber
907-
else
908-
gh release upload "$PCAPKIT_RELEASE_TAG" "$file" --repo "$GITHUB_REPOSITORY"
909-
fi
910-
echo "${name} -> ${PCAPKIT_RELEASE_TAG}"
911-
done
912-
done <<< "$PCAPKIT_ASSETS"
847+
run: python3 util/release_upload.py
913848

914849
# #1633: this KTLO leg's outcome despite ``continue-on-error``, for
915850
# ``release_status``. ``steps.*.outcome`` covers every step with an ``id``:
@@ -1000,19 +935,7 @@ jobs:
1000935
env:
1001936
PCAPKIT_TAGS: v${{ needs.version_check.outputs.PCAPKIT_VERSION }} conda-${{ needs.version_check.outputs.PCAPKIT_VERSION }}+0
1002937
PCAPKIT_RUN_SHA: ${{ github.sha }}
1003-
run: |
1004-
set -euo pipefail
1005-
1006-
run_commit="$(git rev-parse --verify "${PCAPKIT_RUN_SHA}^{commit}")"
1007-
for tag in ${PCAPKIT_TAGS}; do
1008-
git fetch --quiet --force origin "refs/tags/${tag}:refs/tags/${tag}" || true
1009-
tagged="$(git rev-parse -q --verify "refs/tags/${tag}^{commit}" || true)"
1010-
if [ "$tagged" != "$run_commit" ]; then
1011-
echo "::error title=${tag} is not this run's commit (#1634)::${tag} resolves to '${tagged:-no commit}', but this run was created for ${run_commit}. Refusing to publish this tree under it: every artefact of a release comes from its tag's commit, so finish that release by re-running the run created for that commit."
1012-
exit 1
1013-
fi
1014-
echo "${tag} -> ${run_commit}"
1015-
done
938+
run: python3 util/release_tags.py check
1016939

1017940
# #888: the per-leg correctness fix. Uncached, per ``version_check``'s
1018941
# Anaconda check, so this reads the live state.
@@ -1267,35 +1190,7 @@ jobs:
12671190
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
12681191
PCAPKIT_RELEASE_TAG: v${{ needs.version_check.outputs.PCAPKIT_VERSION }}
12691192
PCAPKIT_ASSETS: ${{ steps.conda_build.outputs.packages }}
1270-
run: |
1271-
set -euo pipefail
1272-
1273-
published="$(gh release view "$PCAPKIT_RELEASE_TAG" --repo "$GITHUB_REPOSITORY" --json assets --jq '.assets[] | [.name, .state] | @tsv')"
1274-
while IFS= read -r pattern; do
1275-
[ -n "$pattern" ] || continue
1276-
for file in $pattern; do
1277-
if [ ! -f "$file" ]; then
1278-
echo "::error title=Nothing to upload::'${pattern}' matches no file, so ${PCAPKIT_RELEASE_TAG} would go without it."
1279-
exit 1
1280-
fi
1281-
name="${file##*/}"
1282-
state=""
1283-
while IFS=$'\t' read -r asset asset_state; do
1284-
[ "$asset" != "$name" ] || state="$asset_state"
1285-
done <<< "$published"
1286-
if [ "$state" = uploaded ]; then
1287-
echo "::notice title=${name} already on ${PCAPKIT_RELEASE_TAG}::Left as published (#1640): this run's rebuild is not byte-identical to it."
1288-
continue
1289-
fi
1290-
if [ -n "$state" ]; then
1291-
echo "::warning title=${name} uploaded again::${PCAPKIT_RELEASE_TAG} holds it in state '${state}', not 'uploaded' -- an upload that failed partway -- so it is deleted and this file uploaded in its place."
1292-
gh release upload "$PCAPKIT_RELEASE_TAG" "$file" --repo "$GITHUB_REPOSITORY" --clobber
1293-
else
1294-
gh release upload "$PCAPKIT_RELEASE_TAG" "$file" --repo "$GITHUB_REPOSITORY"
1295-
fi
1296-
echo "${name} -> ${PCAPKIT_RELEASE_TAG}"
1297-
done
1298-
done <<< "$PCAPKIT_ASSETS"
1193+
run: python3 util/release_upload.py
12991194

13001195
# #888: a run that released nothing says why, in one place. ``version_check``
13011196
# can skip before any evidence exists, and ``unit-tests`` can fail or skip

‎docs/source/contributing/workflows.rst‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1234,6 +1234,16 @@ does; each has tests under :file:`tests/project/` that run it.
12341234
* - :file:`util/codeql_report.py`
12351235
- ``Write the report`` in ``codeql-analysis.yml``'s ``discussion`` (see `CI
12361236
Reports`_).
1237+
* - :file:`util/release_tags.py`
1238+
- ``Check the release tags name this run's commit`` in ``create-release.yml``'s
1239+
``github``, ``pypi`` and ``conda``, as ``release_tags.py check``, and ``Check
1240+
which commit the version's tag names`` in its ``version_check``, as
1241+
``release_tags.py owner`` (`#1634
1242+
<https://github.com/JarryShaw/PyPCAPKit/issues/1634>`__, `#1748
1243+
<https://github.com/JarryShaw/PyPCAPKit/issues/1748>`__).
1244+
* - :file:`util/release_upload.py`
1245+
- ``Upload package to GitHub Release`` in ``create-release.yml``'s ``pypi`` and
1246+
``conda`` (`#1640 <https://github.com/JarryShaw/PyPCAPKit/issues/1640>`__).
12371247

12381248
The summaries run on the image's ``/usr/bin/python3`` rather than the leg's
12391249
interpreter, which may have failed to set up or be too broken to run anything, so
@@ -1257,6 +1267,12 @@ could not signal. Both ``changes`` jobs check out on every event, since the scri
12571267
tree. ``create-release.yml``'s ``pypi`` job keeps its own copy of
12581268
the KTLO summary's shell until its shell moves to scripts too; a project pin runs
12591269
that copy and the script on the same steps and requires the same output.
1270+
The release scripts run as ``python3``, from a checkout of the whole tree, and are
1271+
standard library only: :file:`util/release_tags.py` on the image's
1272+
own, since ``github`` has no setup-python and ``pypi`` and ``conda`` run the check
1273+
before theirs, and :file:`util/release_upload.py` on the leg's, after it. A project
1274+
pin runs each on the cases the shell it replaced handled and requires the same
1275+
status, output and calls.
12601276
:file:`util/_actions.py` is standard library only, and keeps to 3.7's syntax,
12611277
since the scripts on ``/usr/bin/python3`` and on every leg's interpreter import it.
12621278

‎tests/project/test_actions_helper_2047.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
HELPER = ROOT / 'util' / '_actions.py'
4242
#: The scripts that import the helper.
4343
SCRIPTS = ('apt_packages', 'ci_timing', 'classify_changes', 'codeql_report', 'coverage_report', 'leg_summary',
44-
'pip_install', 'regenerate_samples', 'vendor_code_changed', 'vendor_test_gate')
44+
'pip_install', 'regenerate_samples', 'release_tags', 'release_upload', 'vendor_code_changed',
45+
'vendor_test_gate')
4546
#: The top-level modules the helper may import, each in Python 3.7's standard
4647
#: library: the running interpreter's :data:`sys.stdlib_module_names` would
4748
#: pass ``tomllib``, which 3.7 lacks.
@@ -56,6 +57,8 @@
5657
'leg_summary': (['ktlo'], {'LEG': 'Python 3.8 (KTLO, non-blocking)', 'STEPS': '{}'}),
5758
'pip_install': (['--help'], {}),
5859
'regenerate_samples': (['--help'], {}),
60+
'release_tags': (['--help'], {}),
61+
'release_upload': (['--help'], {}),
5962
'vendor_code_changed': (['--help'], {}),
6063
'vendor_test_gate': (['--help'], {}),
6164
}

‎tests/project/test_release_gates.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,9 @@
278278
# anaconda-client binary by path; a bare ``anaconda`` on ``PATH`` is not matched.
279279
'/bin/anaconda"': 'uploads a package to anaconda.org with the anaconda-client CLI',
280280
'softprops/action-gh-release': 'creates a public GitHub Release, and the tag it names',
281-
# ``pypi`` and ``conda`` attach their files this way since #1634.
282-
'gh release upload': 'attaches a file to a public GitHub Release',
281+
# ``pypi`` and ``conda`` attach their files with ``gh release upload`` since
282+
# #1634, which util/release_upload.py runs since #1967.
283+
'util/release_upload.py': 'attaches a file to a public GitHub Release',
283284
'git push': 'pushes a commit or a tag to the repository',
284285
}
285286

0 commit comments

Comments
 (0)