Skip to content

Commit d162719

Browse files
authored
fix: retry git fetches that stall (#49)
* test(e2e): speed up Windows CI by running natively with Docker on local disks * fix: retry git fetches that stall
1 parent 34a363a commit d162719

21 files changed

Lines changed: 575 additions & 158 deletions

‎.github/workflows/e2e.yml‎

Lines changed: 90 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,18 @@ concurrency:
2626
group: e2e-${{ github.ref }}
2727
cancel-in-progress: true
2828

29-
# The Linux job runs the suite natively, the way the dev container runs it
30-
# locally: Go at the dev container's version (.devcontainer/Dockerfile) plus
31-
# the runner's stock Docker, with the same TESTCONTAINERS_RYUK_DISABLED knob.
32-
# The Windows jobs keep the compose runner image
33-
# (tests/e2e/windows/Dockerfile-runner), which carries its own Go toolchain,
34-
# so a Windows host needs only Docker. See docs/e2e.md.
29+
# Both jobs run the suite natively: Go at the dev container's version
30+
# (.devcontainer/Dockerfile) plus the runner's Docker. The Linux job runs it
31+
# the way the dev container runs it locally, with the same
32+
# TESTCONTAINERS_RYUK_DISABLED knob. The Windows jobs skip the compose
33+
# runner image local Windows hosts use (tests/e2e/windows/Dockerfile-runner):
34+
# baking Go into a Windows image costs minutes per run. See docs/e2e.md.
35+
36+
env:
37+
GO_VERSION: '1.26.8'
38+
# Runners are discarded with whatever still runs on them, so the suites
39+
# tear containers down in the background (common.AsyncCleanup).
40+
E2E_ASYNC_CLEANUP: 'true'
3541

3642
jobs:
3743
e2e-linux:
@@ -53,7 +59,7 @@ jobs:
5359
- uses: actions/checkout@v7
5460
- uses: actions/setup-go@v7
5561
with:
56-
go-version: '1.26.8'
62+
go-version: ${{ env.GO_VERSION }}
5763
cache: false
5864
- run: go test -v -timeout 55m -run TestE2E ./linux
5965
working-directory: tests/e2e
@@ -66,27 +72,90 @@ jobs:
6672
matrix:
6773
# TODO See https://github.com/actions/runner-images/issues/13729
6874
runs-on: [windows-latest, windows-11-arm]
69-
include:
70-
- runs-on: windows-latest
71-
compose: tests/e2e/windows/compose.amd64.yaml
72-
- runs-on: windows-11-arm
73-
compose: tests/e2e/windows/compose.arm64.yaml
7475
runs-on: ${{ matrix.runs-on }}
7576
timeout-minutes: 60
77+
defaults:
78+
run:
79+
shell: pwsh
80+
env:
81+
# windows-latest runs six combos, slowest first (common.SlowestFirst):
82+
# with four at a time the short ones left over finish within the first
83+
# wave. windows-11-arm runs two.
84+
E2E_MAX_JOBS: '4'
7685
steps:
7786
- uses: actions/checkout@v7
78-
# TODO For compiling moby only. Because https://github.com/actions/runner-images/blob/win11-arm64/20260920.174/images/windows/Windows11-Arm64-Readme.md is still using `Go 1.24.13`.
87+
# The engine's data-root takes every image pull, layer commit and
88+
# in-container Flutter install, and the runners' C: OS disk is slow
89+
# remote storage. windows-latest keeps the workspace on a local D:
90+
# disk: move the preinstalled engine's data-root next to it.
91+
- if: matrix.runs-on == 'windows-latest'
92+
name: Move the Docker data-root to the workspace disk
93+
run: |
94+
$dataRoot = (Split-Path -Qualifier $env:GITHUB_WORKSPACE) + '\docker'
95+
$config = Join-Path $env:ProgramData 'docker\config\daemon.json'
96+
$settings = if (Test-Path $config) { Get-Content -Raw $config | ConvertFrom-Json } else { [pscustomobject]@{} }
97+
$settings | Add-Member -NotePropertyName 'data-root' -NotePropertyValue $dataRoot -Force
98+
New-Item -ItemType Directory -Force -Path (Split-Path $config) | Out-Null
99+
$settings | ConvertTo-Json | Set-Content -Path $config -Encoding ascii
100+
Restart-Service docker
101+
for ($attempt = 1; $attempt -le 30; $attempt++) {
102+
$current = docker info --format '{{.DockerRootDir}}' 2>$null
103+
if ($LASTEXITCODE -eq 0) { break }
104+
Start-Sleep -Seconds 2
105+
}
106+
if ($current -ne $dataRoot) { throw "FAIL the engine runs off '$current', want '$dataRoot'" }
107+
Write-Output "PASS the Docker data-root is $current"
108+
# windows-11-arm has no D:, but carries an unformatted local NVMe disk
109+
# (actions/runner-images#14088): format it for the engine's data-root.
79110
- if: matrix.runs-on == 'windows-11-arm'
111+
id: disk
112+
name: Format the local NVMe disk
113+
run: |
114+
$disk = Get-Disk | Where-Object PartitionStyle -eq 'RAW' | Select-Object -First 1
115+
if ($disk) {
116+
$volume = $disk | Initialize-Disk -PartitionStyle GPT -PassThru |
117+
New-Partition -AssignDriveLetter -UseMaximumSize |
118+
Format-Volume -FileSystem NTFS -Confirm:$false
119+
"data-root=$($volume.DriveLetter):\vfox-docker\data" >> $env:GITHUB_OUTPUT
120+
Write-Output "PASS formatted raw disk $($disk.Number) as $($volume.DriveLetter):"
121+
}
122+
else {
123+
Write-Output '::warning::no raw local disk; the Docker data-root stays on C:'
124+
}
125+
# Building the engine from source takes minutes; the script reuses the
126+
# binaries restored here.
127+
- if: matrix.runs-on == 'windows-11-arm'
128+
id: moby
129+
uses: actions/cache@v6
130+
with:
131+
path: C:\vfox-docker\bin
132+
key: moby-windows-arm64-go${{ env.GO_VERSION }}-${{ hashFiles('tests/e2e/windows/install-docker-arm64.ps1') }}
133+
# Go is needed this early only to build the engine; otherwise it is set
134+
# up below, while the base image pulls.
135+
- if: matrix.runs-on == 'windows-11-arm' && steps.moby.outputs.cache-hit != 'true'
80136
uses: actions/setup-go@v7
81137
with:
82-
go-version: 1.26.8
138+
go-version: ${{ env.GO_VERSION }}
83139
cache: false
84140
- if: matrix.runs-on == 'windows-11-arm'
85-
run: pwsh -NoProfile -File tests/e2e/windows/install-docker-arm64.ps1
86-
- if: matrix.runs-on == 'windows-11-arm'
87-
name: Install Compose CLI plugin
141+
run: pwsh -NoProfile -File tests/e2e/windows/install-docker-arm64.ps1 -DataRoot '${{ steps.disk.outputs.data-root }}'
142+
# Unpacking servercore is the longest setup step and needs nothing but
143+
# the engine: start it now so that setting up Go and compiling the
144+
# suite overlap it. The suite's own pull of the image joins this one.
145+
# No output redirection: Start-Process then goes through ShellExecute,
146+
# so the pull does not inherit this step's output pipes, and the runner
147+
# only kills what still holds them when the step ends.
148+
- name: Pull the base image in the background
88149
run: |
89-
New-Item -ItemType Directory -Force -Path "$env:USERPROFILE\.docker\cli-plugins" | Out-Null
90-
curl.exe -fsSL "https://github.com/docker/compose/releases/download/v5.5.1/docker-compose-windows-aarch64.exe" -o "$env:USERPROFILE\.docker\cli-plugins\docker-compose.exe"
91-
docker compose version
92-
- run: docker compose -f ${{ matrix.compose }} run --rm --build e2e
150+
$arch = if ($env:PROCESSOR_ARCHITECTURE -eq 'ARM64') { 'arm64' } else { 'amd64' }
151+
$match = Select-String -Path tests/e2e/windows/Dockerfile -Pattern '^ARG BASE_IMAGE=(.+)-amd64$'
152+
if (-not $match) { throw 'FAIL tests/e2e/windows/Dockerfile has no ARG BASE_IMAGE=...-amd64 line' }
153+
$image = $match.Matches[0].Groups[1].Value + "-$arch"
154+
Start-Process docker -ArgumentList 'pull', $image
155+
Write-Output "Pulling $image in the background"
156+
- uses: actions/setup-go@v7
157+
with:
158+
go-version: ${{ env.GO_VERSION }}
159+
cache: false
160+
- run: go test -v -timeout 55m -run TestE2E ./windows
161+
working-directory: tests/e2e

‎docs/arm64.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,9 @@ run on Windows ARM64.
4747
## Linux ARM64 and Windows ARM64
4848

4949
- **`git` is required.** The checkout runs `git init` / `fetch` / `checkout`
50-
against `https://github.com/flutter/flutter.git`.
50+
against `https://github.com/flutter/flutter.git`. A fetch that stalls (under
51+
1 KB/s for a minute) is abandoned and retried, up to three attempts;
52+
`GIT_HTTP_LOW_SPEED_LIMIT` / `GIT_HTTP_LOW_SPEED_TIME` change the threshold.
5153
- **The first `flutter` run is slow.** It downloads the native Dart SDK and
5254
builds the Flutter tool, so it needs a network connection.
5355
- **No checksum is verified.** Git history is the integrity check.

‎docs/e2e.md‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,10 @@
33
The offline Lua hook tests (`tests/hooks_test.lua`) run in the **Test Plugin** workflow.
44
The end-to-end suite runs in containers against real vfox on Linux and Windows;
55
each `vfox` x `flavor` x `mirror` combination gets its own throwaway
6-
container. On Linux, CI runs the suite natively on the stock runners (Docker
7-
plus Go from `actions/setup-go`), the way the repository dev container
8-
reproduces it locally. On Windows it runs through a Docker Compose runner
9-
image, since dev containers cannot host Windows container images (see below).
6+
container. CI runs both suites natively on the stock runners (Docker plus Go
7+
from `actions/setup-go`). Locally, the repository dev container reproduces the
8+
Linux run; Windows hosts use a Docker Compose runner image instead, since dev
9+
containers cannot host Windows container images (see below).
1010

1111
## Dev container
1212

@@ -40,7 +40,10 @@ devcontainer exec --workspace-folder . bash
4040
host daemon and waits for the suite to dial it back, which a
4141
Docker-out-of-Docker container cannot do. The suites terminate every
4242
container explicitly via `t.Cleanup` instead; leftovers from killed runs
43-
share the `vfox-flutter-e2e-` name prefix for pruning. Rootless Docker users
43+
share the `vfox-flutter-e2e-` name prefix for pruning. CI sets
44+
`E2E_ASYNC_CLEANUP=true` so that teardown runs in the background instead of
45+
holding up each combination, since its runners are discarded anyway; the
46+
last containers may then outlive the suite. Rootless Docker users
4447
export `DOCKER_SOCK="$XDG_RUNTIME_DIR/docker.sock"` on the host before
4548
opening the container. On Windows hosts, keeping the checkout inside WSL 2
4649
(e.g. under `\\wsl$`) keeps file and bind-mount access fast; a checkout on
@@ -89,6 +92,28 @@ named pipe; the host requirement is Docker in Windows-container mode plus the
8992
Compose v5 plugin (installed once per machine below). Go never touches the
9093
host, so it cannot conflict with a vfox-managed toolchain.
9194

95+
CI skips that runner image, which costs minutes per run to build, and runs
96+
`go test` on the runner with Go from `actions/setup-go`, like the Linux job.
97+
It also moves the engine's data-root off the runners' slow C: OS disk,
98+
where image pulls, layer commits and the in-container Flutter installs would
99+
otherwise crawl: next to the workspace on `windows-latest`'s local D: disk,
100+
and onto the local NVMe disk `windows-11-arm` ships unformatted. With that,
101+
`E2E_MAX_JOBS` is 4 there. Both suites start the combinations slowest first
102+
(the CN mirror, then the official SDK), so the last ones to start are short.
103+
Unpacking the servercore base image remains the longest setup step, so CI
104+
starts pulling it in the background before it sets up Go and compiles the
105+
suite.
106+
107+
The target image (`tests/e2e/windows/Dockerfile`) unpacks PowerShell 7 and
108+
MinGit from their portable zips instead of running their installers: Git
109+
for Windows' installer is an x86 program, which arm64 containers only run
110+
under emulation. MinGit is the same `git.exe` without bash, Perl and the
111+
credential manager, none of which vfox or Flutter calls.
112+
113+
The `main` combinations share one `vfox.exe`: the suite builds it from vfox's
114+
main branch with its own Go toolchain while the image builds, and copies it
115+
into each `main` container before the container starts.
116+
92117
The `windows-11-arm` runner has no Windows Hypervisor Platform feature,
93118
so Hyper-V isolation is unavailable; process isolation needs only the Containers feature.
94119

@@ -165,7 +190,11 @@ docker compose -f tests/e2e/windows/compose.arm64.yaml run --rm --build e2e
165190

166191
On arm64, Go is also what builds the engine, so `install-docker-arm64.ps1`
167192
runs before the suite; the compose runner image then carries its own pinned
168-
Go for the tests, and the host Go is not used again. Scope the run down with
193+
Go for the tests, and the host Go is not used again. The script keeps the
194+
engine binaries in `C:\vfox-docker\bin` and reuses them on later runs while
195+
they match its pinned version (CI caches that directory); `-DataRoot` moves
196+
the engine's images and containers off the default `C:\vfox-docker\data`.
197+
Scope the run down with
169198
`$env:VFOX_VERSION = 'latest'; $env:FLAVOR = 'official'` in front of the
170199
compose command, as on the other platforms.
171200

‎docs/ohos.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,9 @@ vfox search flutter
4141
- **Mirrors don't apply.** `FLUTTER_STORAGE_BASE_URL` and `VFOX_FLUTTER_GITHUB_MIRROR` (see `mirror.md`) only affect
4242
official SDK downloads; OpenHarmony builds are cloned from gitcode, so both variables
4343
are ignored.
44+
- **Stalled clones are retried.** A fetch from gitcode that stalls (under 1 KB/s for a
45+
minute) is abandoned and retried, up to three attempts; `GIT_HTTP_LOW_SPEED_LIMIT` /
46+
`GIT_HTTP_LOW_SPEED_TIME` change the threshold.
4447

4548
### Building OpenHarmony applications
4649

‎lib/git.lua‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,11 @@ local VFOX_DIR = ".vfox"
1818
local TMP_DIR = "tmp"
1919
local ENGINE_PIN = "bin/internal/engine.version"
2020
local FETCH_ATTEMPTS = 3
21+
-- A stalled connection never fails on its own, which would leave the retries
22+
-- above unused: abort a fetch that moves under 1 KB/s for a minute, so the next
23+
-- attempt reconnects. GIT_HTTP_LOW_SPEED_LIMIT and GIT_HTTP_LOW_SPEED_TIME
24+
-- still override both values.
25+
local FETCH_STALL_GUARD = "-c http.lowSpeedLimit=1000 -c http.lowSpeedTime=60"
2126
local GITHUB_MIRROR_ENV = "VFOX_FLUTTER_GITHUB_MIRROR"
2227
local GH_BASE = "https://github.com/"
2328

@@ -137,7 +142,7 @@ end
137142

138143
function M.fetch(root, remote, ref)
139144
for _ = 1, FETCH_ATTEMPTS do
140-
if git(root, "fetch -q --depth 1 " .. remote .. " " .. ref) then
145+
if git(root, FETCH_STALL_GUARD .. " fetch -q --depth 1 " .. remote .. " " .. ref) then
141146
return true
142147
end
143148
end

‎tests/e2e/common/common.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,8 +28,10 @@ import (
2828
"fmt"
2929
"os"
3030
"runtime"
31+
"slices"
3132
"strconv"
3233
"strings"
34+
"sync"
3335
"testing"
3436
"time"
3537
)
@@ -180,6 +182,51 @@ func MatrixForArch(arch string) ([]Combo, error) {
180182
return combos, nil
181183
}
182184

185+
// SlowestFirst returns combos ordered by how long they usually take, the
186+
// slowest first, so a bounded run (RunCombos) ends on short combos instead
187+
// of starting a long one last: non-default mirrors (the CN mirror is far
188+
// from the CI runners) before the default one, official before ohos. The
189+
// order is otherwise kept.
190+
func SlowestFirst(combos []Combo) []Combo {
191+
slowness := func(c Combo) int {
192+
w := 0
193+
if c.Mirror != "default" {
194+
w += 2
195+
}
196+
if c.Flavor == "official" {
197+
w++
198+
}
199+
return w
200+
}
201+
sorted := slices.Clone(combos)
202+
slices.SortStableFunc(sorted, func(a, b Combo) int { return slowness(b) - slowness(a) })
203+
return sorted
204+
}
205+
206+
// RunCombos runs run as one subtest per combo, at most jobs at a time,
207+
// starting them in the given order: unlike t.Parallel behind a semaphore,
208+
// whose start order is up to the scheduler, so callers can put the slowest
209+
// combos first.
210+
func RunCombos(t *testing.T, combos []Combo, jobs int, run func(t *testing.T, c Combo)) {
211+
t.Helper()
212+
queue := make(chan Combo, len(combos))
213+
for _, c := range combos {
214+
queue <- c
215+
}
216+
close(queue)
217+
var wg sync.WaitGroup
218+
for range min(max(jobs, 1), len(combos)) {
219+
wg.Add(1)
220+
go func() {
221+
defer wg.Done()
222+
for c := range queue {
223+
t.Run(c.Slug(), func(t *testing.T) { run(t, c) })
224+
}
225+
}()
226+
}
227+
wg.Wait()
228+
}
229+
183230
// MaxJobs bounds parallel containers; honours $E2E_MAX_JOBS and falls back
184231
// to def when unset or invalid. Linux passes 3, Windows passes 2 (matching
185232
// the old e2e.ps1).
@@ -202,6 +249,29 @@ func RetryAttempts() int {
202249
return 3
203250
}
204251

252+
// AsyncCleanup reports whether containers are torn down in the background
253+
// instead of holding up the test that ran them; honours $E2E_ASYNC_CLEANUP.
254+
// CI sets it, since the runner is discarded with whatever still runs on
255+
// it; local runs keep waiting so no container outlives the suite.
256+
func AsyncCleanup() bool {
257+
v, _ := strconv.ParseBool(strings.TrimSpace(os.Getenv("E2E_ASYNC_CLEANUP")))
258+
return v
259+
}
260+
261+
// Cleanup registers terminate to run once t finishes, in the background
262+
// under AsyncCleanup (the suite may then exit before the last containers
263+
// are gone).
264+
func Cleanup(t *testing.T, terminate func()) {
265+
t.Helper()
266+
t.Cleanup(func() {
267+
if AsyncCleanup() {
268+
go terminate()
269+
return
270+
}
271+
terminate()
272+
})
273+
}
274+
205275
// ResolveFlutterVersion honours $FLUTTER_VERSION / $OHOS_VERSION, else the
206276
// defaults above.
207277
func ResolveFlutterVersion(t *testing.T, flavor string) string {

0 commit comments

Comments
 (0)