Skip to content

Commit cd13187

Browse files
susnuxsusnux
authored
Merge pull request #15743 from nextcloud/feature/squash-gh-pages-history
ci: squash gh-pages history on every release
2 parents 2de9481 + 706ca9e commit cd13187

3 files changed

Lines changed: 455 additions & 0 deletions

File tree

‎.github/workflows/squash-gh-pages.yml‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
2+
# SPDX-License-Identifier: AGPL-3.0-or-later
3+
#
4+
# Squash the gh-pages history on every release.
5+
#
6+
# Every deploy commits a full set of rebuilt PDF and ePub files, so gh-pages
7+
# grows by ~18 MiB a day. build/squash-gh-pages.sh rebuilds the branch as one
8+
# snapshot commit per release day, keeps the deploys made since the last
9+
# release, and stores out of support version folders only once. The published
10+
# tree is unchanged; see the script for details.
11+
#
12+
# Required setup (repository admin):
13+
# GH_PAGES_DEPLOY_KEY – private half of a deploy key with write access, added
14+
# as a bypass actor to the "Pages" ruleset, which
15+
# otherwise blocks force pushes to gh-pages.
16+
name: "Squash gh-pages history"
17+
18+
on:
19+
push:
20+
tags:
21+
- 'v*'
22+
pull_request:
23+
paths:
24+
- '.github/workflows/squash-gh-pages.yml'
25+
- 'build/squash-gh-pages.sh'
26+
- 'build/tests/**'
27+
workflow_dispatch:
28+
inputs:
29+
dry_run:
30+
description: "Compute the rewrite without pushing it"
31+
type: boolean
32+
default: true
33+
34+
permissions:
35+
contents: read
36+
37+
concurrency:
38+
group: squash-gh-pages
39+
cancel-in-progress: false
40+
41+
jobs:
42+
test:
43+
name: Test squash script
44+
runs-on: ubuntu-latest
45+
46+
steps:
47+
- name: Checkout repository
48+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
49+
with:
50+
persist-credentials: false
51+
52+
- name: Run unit tests
53+
run: bash build/tests/test-squash-gh-pages.sh
54+
55+
squash:
56+
name: Squash gh-pages
57+
needs: test
58+
# Final releases only: rc, beta and alpha tags do not get a snapshot.
59+
if: >-
60+
github.event_name == 'workflow_dispatch' ||
61+
(github.event_name == 'push' && !contains(github.ref_name, 'rc') &&
62+
!contains(github.ref_name, 'beta') && !contains(github.ref_name, 'alpha'))
63+
runs-on: ubuntu-latest
64+
65+
permissions:
66+
contents: read
67+
68+
steps:
69+
# Full history: the release dates are read from the tags.
70+
- name: Checkout repository
71+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
72+
with:
73+
ref: ${{ github.event.repository.default_branch }}
74+
fetch-depth: 0
75+
persist-credentials: false
76+
77+
- name: Setup PHP
78+
uses: shivammathur/setup-php@f3e473d116dcccaddc5834248c87452386958240 # v2.37.2
79+
with:
80+
php-version: '8.3'
81+
82+
- name: Detect lowest supported version
83+
id: versions
84+
env:
85+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
86+
run: |
87+
mapfile -t nums < <(git ls-remote --heads origin "heads/stable[0-9][0-9]" \
88+
| sed -n 's?.*refs/heads/stable\([0-9]\{2\}\)$?\1?p' | sort -n)
89+
eval "$(php build/detect-versions.php "${nums[@]}")"
90+
echo "Folders below server/${lowest_stable} are frozen"
91+
echo "lowest_stable=${lowest_stable}" >> "$GITHUB_OUTPUT"
92+
93+
- name: Fetch gh-pages history
94+
run: git fetch --no-tags origin +refs/heads/gh-pages:refs/heads/gh-pages
95+
96+
- name: Rewrite gh-pages
97+
id: rewrite
98+
run: |
99+
echo "old=$(git rev-parse gh-pages)" >> "$GITHUB_OUTPUT"
100+
build/squash-gh-pages.sh gh-pages "${{ steps.versions.outputs.lowest_stable }}" \
101+
refs/heads/gh-pages-squashed | tee -a "$GITHUB_OUTPUT"
102+
103+
# Deploy PRs are based on the old history: merging one after the rewrite
104+
# would bring every squashed commit back. The next nightly deploy opens a
105+
# fresh one on top of the new history.
106+
- name: Close open deploy pull requests
107+
if: steps.rewrite.outputs.changed == 'true' && inputs.dry_run != true
108+
env:
109+
GH_TOKEN: ${{ secrets.COMMAND_BOT_PAT }}
110+
run: |
111+
gh pr list --repo "${{ github.repository }}" --base gh-pages --state open \
112+
--json number,headRefName \
113+
--jq '.[] | select(.headRefName | startswith("automated/deploy/")) | .number' \
114+
| while read -r number; do
115+
gh pr close "${number}" --repo "${{ github.repository }}" --delete-branch \
116+
--comment "Closed because gh-pages was squashed. The next scheduled deploy reopens it on the new history."
117+
done
118+
119+
# The lease makes the push fail if anything landed on gh-pages since the
120+
# fetch, rather than dropping it. Re-run the workflow in that case.
121+
- name: Push rewritten gh-pages
122+
if: steps.rewrite.outputs.changed == 'true' && inputs.dry_run != true
123+
env:
124+
DEPLOY_KEY: ${{ secrets.GH_PAGES_DEPLOY_KEY }}
125+
run: |
126+
key="${RUNNER_TEMP}/gh-pages-deploy-key"
127+
hosts="${RUNNER_TEMP}/github-known-hosts"
128+
install -m 600 /dev/null "${key}"
129+
printf '%s\n' "${DEPLOY_KEY}" > "${key}"
130+
# GitHub's published host keys, fetched over TLS rather than trusted on first use.
131+
curl -fsSL https://api.github.com/meta | jq -r '.ssh_keys[] | "github.com " + .' > "${hosts}"
132+
GIT_SSH_COMMAND="ssh -i ${key} -o IdentitiesOnly=yes -o UserKnownHostsFile=${hosts} -o StrictHostKeyChecking=yes" git push \
133+
--force-with-lease="refs/heads/gh-pages:${{ steps.rewrite.outputs.old }}" \
134+
"git@github.com:${{ github.repository }}.git" \
135+
refs/heads/gh-pages-squashed:refs/heads/gh-pages
136+
rm -f "${key}"

‎build/squash-gh-pages.sh‎

Lines changed: 141 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
#!/usr/bin/env bash
2+
# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
3+
# SPDX-License-Identifier: AGPL-3.0-or-later
4+
#
5+
# Rewrite the gh-pages history to keep it from growing without bounds.
6+
#
7+
# Every deploy commits a full set of rebuilt PDF and ePub files, so the branch
8+
# grows by tens of MiB per deploy even when the docs barely change. The built
9+
# output has no value as history, so this script rebuilds the branch as:
10+
#
11+
# - one snapshot commit per release day (final vX.Y.Z tags only), holding the
12+
# tree of the last deploy made before the end of that day (UTC),
13+
# - every deploy made after the last release, unchanged,
14+
# - version folders below <frozen-below> (out of support, no more releases)
15+
# replaced by their current content in every commit, so they are stored once.
16+
#
17+
# The tip tree is never modified, so the published site is identical. The
18+
# rewrite is deterministic: running it again on its own output yields the same
19+
# commits, which makes it safe to run on every release tag.
20+
#
21+
# Usage: squash-gh-pages.sh <branch> <frozen-below> <output-ref>
22+
#
23+
# Prints changed=true|false and tip=<sha> for $GITHUB_OUTPUT, and a commit
24+
# count summary on stderr.
25+
26+
set -euo pipefail
27+
28+
# Group final release tags by UTC day.
29+
# stdin: "<tag> <unix timestamp>" lines
30+
# stdout: "<end of day timestamp> <tag>[,<tag>...]" lines, oldest day first
31+
release_days() {
32+
grep -E '^v[0-9]+\.[0-9]+\.[0-9]+ ' \
33+
| awk '{ print int($2 / 86400) * 86400 + 86399, $1 }' \
34+
| LC_ALL=C sort -k1,1n -k2,2 \
35+
| awk '$1 == day { tags = tags "," $2; next }
36+
day { print day, tags }
37+
{ day = $1; tags = $2 }
38+
END { if (day) print day, tags }'
39+
}
40+
41+
# Pick the commits to keep.
42+
# $1: file of "<sha> <committer timestamp>" lines, oldest first
43+
# $2: file of release_days output
44+
# stdout: "<sha> <tags>" for a snapshot, "<sha> -" for an unreleased commit
45+
plan() {
46+
awk 'NR == FNR { sha[NR] = $1; ts[NR] = $2; n = NR; next }
47+
{
48+
last = 0
49+
for (i = 1; i <= n; i++) if (ts[i] <= $1) last = i
50+
if (!last) next
51+
if (last in tags) tags[last] = tags[last] "," $2
52+
else { order[++m] = last; tags[last] = $2 }
53+
}
54+
END {
55+
for (k = 1; k <= m; k++) print sha[order[k]], tags[order[k]]
56+
for (i = (m ? order[m] + 1 : 1); i <= n; i++) print sha[i], "-"
57+
}' "$1" "$2"
58+
}
59+
60+
# ls-tree lines of server/<N> folders below $2 in tree-ish $1 (or the others with $3 = keep)
61+
version_entries() {
62+
git ls-tree "$1" 2>/dev/null | awk -F '\t' -v below="$2" -v keep="${3:-}" '
63+
{ frozen = $2 ~ /^[0-9]+$/ && $2 + 0 < below }
64+
frozen != (keep == "keep")'
65+
}
66+
67+
# Tree of commit $1 with every frozen server/<N> folder replaced by its tip content.
68+
frozen_tree() {
69+
local server
70+
server=$( { version_entries "$1:server" "$FROZEN_BELOW" keep; printf '%s' "$FROZEN_ENTRIES"; } \
71+
| awk NF | git mktree)
72+
{
73+
git ls-tree "$1^{tree}" | awk -F '\t' '$2 != "server"'
74+
if [ "$server" != "$(git mktree </dev/null)" ]; then
75+
printf '040000 tree %s\tserver\n' "$server"
76+
fi
77+
} | git mktree
78+
}
79+
80+
# Commit tree $2 on parent $3 (may be empty) with the metadata of commit $1.
81+
# stdin: commit message
82+
commit_like() {
83+
local header author committer author_email committer_email
84+
header=$(git cat-file commit "$1" | sed '/^$/q')
85+
# "Name <email> <timestamp> <tz>"
86+
author=$(sed -n 's/^author //p' <<<"$header")
87+
committer=$(sed -n 's/^committer //p' <<<"$header")
88+
author_email=${author##*<}
89+
committer_email=${committer##*<}
90+
GIT_AUTHOR_NAME=${author% <*} GIT_AUTHOR_EMAIL=${author_email%%>*} GIT_AUTHOR_DATE=${author##*> } \
91+
GIT_COMMITTER_NAME=${committer% <*} GIT_COMMITTER_EMAIL=${committer_email%%>*} GIT_COMMITTER_DATE=${committer##*> } \
92+
git commit-tree "$2" ${3:+-p "$3"}
93+
}
94+
95+
# Rebuild branch $1 with folders below $2 frozen, print the new tip (the branch is not moved).
96+
rewrite() {
97+
local branch=$1 tmp parent='' sha tags tree
98+
FROZEN_BELOW=$2
99+
FROZEN_ENTRIES=$(version_entries "$branch:server" "$FROZEN_BELOW")
100+
tmp=$(mktemp -d)
101+
102+
git log --first-parent --reverse --format='%H %ct' "$branch" >"$tmp/commits"
103+
git for-each-ref --format='%(refname:short) %(creatordate:unix)' refs/tags \
104+
| { release_days || true; } >"$tmp/days"
105+
106+
while read -r sha tags; do
107+
tree=$(frozen_tree "$sha")
108+
if [ "$tags" = - ]; then
109+
parent=$(git cat-file commit "$sha" | sed '1,/^$/d' | commit_like "$sha" "$tree" "$parent")
110+
else
111+
parent=$(printf 'chore: documentation snapshot for %s\n\nSquashed gh-pages history up to this release.\n' \
112+
"${tags//,/, }" | commit_like "$sha" "$tree" "$parent")
113+
fi
114+
done < <(plan "$tmp/commits" "$tmp/days")
115+
116+
rm -rf "$tmp"
117+
echo "$parent"
118+
}
119+
120+
main() {
121+
if [ $# -ne 3 ]; then
122+
echo "Usage: $0 <branch> <frozen-below> <output-ref>" >&2
123+
exit 1
124+
fi
125+
local branch=$1 tip
126+
tip=$(rewrite "$branch" "$2")
127+
128+
if [ "$(git rev-parse "$tip^{tree}")" != "$(git rev-parse "$branch^{tree}")" ]; then
129+
echo "Rewritten tip tree differs from the current branch, aborting" >&2
130+
exit 1
131+
fi
132+
git update-ref "$3" "$tip"
133+
134+
echo "$branch: $(git rev-list --count --first-parent "$branch") commits -> $(git rev-list --count "$tip") commits" >&2
135+
if [ "$tip" = "$(git rev-parse "$branch")" ]; then echo changed=false; else echo changed=true; fi
136+
echo "tip=$tip"
137+
}
138+
139+
if [ "${BASH_SOURCE[0]}" = "$0" ]; then
140+
main "$@"
141+
fi

0 commit comments

Comments
 (0)