Skip to content

Commit 1aeaeb6

Browse files
authored
Add resume for download function (pytorch#15948)
It seems like it fails when it's a bit above 1Gb, try to add a resume so we can continue from the last failing point instead of download all again
1 parent 94254d4 commit 1aeaeb6

3 files changed

Lines changed: 84 additions & 36 deletions

File tree

‎.github/workflows/pull.yml‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -13,32 +13,32 @@ concurrency:
1313
cancel-in-progress: true
1414

1515
jobs:
16-
# test-qnn-wheel-packages-linux:
17-
# name: test-qnn-wheel-packages-linux
18-
# uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
19-
# permissions:
20-
# id-token: write
21-
# contents: read
22-
# strategy:
23-
# fail-fast: false
24-
# matrix:
25-
# python-version: [ "3.10", "3.11", "3.12" ]
26-
# with:
27-
# runner: linux.2xlarge
28-
# docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
29-
# submodules: 'recursive'
30-
# ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
31-
# timeout: 180
32-
# script: |
33-
# # The generic Linux job chooses to use base env, not the one setup by the image
34-
# CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
35-
# conda activate "${CONDA_ENV}"
16+
test-qnn-wheel-packages-linux:
17+
name: test-qnn-wheel-packages-linux
18+
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
19+
permissions:
20+
id-token: write
21+
contents: read
22+
strategy:
23+
fail-fast: false
24+
matrix:
25+
python-version: [ "3.10", "3.11", "3.12" ]
26+
with:
27+
runner: linux.2xlarge
28+
docker-image: ci-image:executorch-ubuntu-22.04-qnn-sdk
29+
submodules: 'recursive'
30+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
31+
timeout: 180
32+
script: |
33+
# The generic Linux job chooses to use base env, not the one setup by the image
34+
CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
35+
conda activate "${CONDA_ENV}"
3636
37-
# # Create a clean env for each python version
38-
# conda create -y -n test_env_${{ matrix.python-version }} python=${{ matrix.python-version }}
39-
# conda activate test_env_${{ matrix.python-version }}
37+
# Create a clean env for each python version
38+
conda create -y -n test_env_${{ matrix.python-version }} python=${{ matrix.python-version }}
39+
conda activate test_env_${{ matrix.python-version }}
4040
41-
# PYTHON_EXECUTABLE=python bash .ci/scripts/test_wheel_package_qnn.sh "${{ matrix.python-version }}"
41+
PYTHON_EXECUTABLE=python bash .ci/scripts/test_wheel_package_qnn.sh "${{ matrix.python-version }}"
4242
4343
test-setup-linux-gcc:
4444
name: test-setup-linux-gcc

‎backends/qualcomm/scripts/download_qnn_sdk.py‎

Lines changed: 59 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ def _atomic_download(url: str, dest: pathlib.Path):
117117

118118

119119
def _download_archive(url: str, archive_path: pathlib.Path) -> bool:
120-
"""Robust streaming download with retries."""
120+
"""Streaming download with retry + resume support."""
121121

122122
logger.debug("Archive will be saved to: %s", archive_path)
123123

@@ -130,32 +130,80 @@ def _download_archive(url: str, archive_path: pathlib.Path) -> bool:
130130
)
131131
session.mount("https://", HTTPAdapter(max_retries=retries))
132132

133+
# ------------------------------------------------------------
134+
# 1. Detect total file size (HEAD is broken on Qualcomm)
135+
# ------------------------------------------------------------
133136
try:
134-
with session.get(url, stream=True) as r:
137+
# NOTE:
138+
# Qualcomm's download endpoint does not return accurate metadata on HEAD requests.
139+
# Many Qualcomm URLs first redirect to an HTML "wrapper" page (typically ~134 bytes),
140+
# and the HEAD request reflects *that wrapper* rather than the actual ZIP archive.
141+
#
142+
# Example:
143+
# HEAD -> Content-Length: 134, Content-Type: text/html
144+
# GET -> Content-Length: 1354151797, Content-Type: application/zip
145+
#
146+
# Because Content-Length from HEAD is frequently incorrect, we fall back to issuing
147+
# a GET request with stream=True to obtain the real Content-Length without downloading
148+
# the full file. This ensures correct resume logic and size validation.
149+
r_head = session.get(url, stream=True)
150+
r_head.raise_for_status()
151+
152+
if "content-length" not in r_head.headers:
153+
logger.error("Server did not return content-length!")
154+
return False
155+
156+
total_size = int(r_head.headers["content-length"])
157+
except Exception as e:
158+
logger.exception("Failed to determine file size: %s", e)
159+
return False
160+
161+
# ------------------------------------------------------------
162+
# 2. If partial file exists, resume
163+
# ------------------------------------------------------------
164+
downloaded = archive_path.stat().st_size if archive_path.exists() else 0
165+
if downloaded > total_size:
166+
logger.warning("Existing file is larger than expected. Removing.")
167+
archive_path.unlink()
168+
downloaded = 0
169+
170+
logger.info("Resuming download from %d / %d bytes", downloaded, total_size)
171+
172+
headers = {}
173+
if downloaded > 0:
174+
headers["Range"] = f"bytes={downloaded}-"
175+
176+
try:
177+
# resume GET
178+
with session.get(url, stream=True, headers=headers) as r:
135179
r.raise_for_status()
136180

137-
downloaded = 0
138181
chunk_size = 1024 * 1024 # 1MB
182+
mode = "ab" if downloaded > 0 else "wb"
139183

140-
with open(archive_path, "wb") as f:
184+
with open(archive_path, mode) as f:
141185
for chunk in r.iter_content(chunk_size):
142186
if chunk:
143187
f.write(chunk)
144188
downloaded += len(chunk)
145189

146-
logger.info("Download completed!")
147-
148190
except Exception as e:
149191
logger.exception("Error during download: %s", e)
150192
return False
151193

152-
if archive_path.exists() and archive_path.stat().st_size == 0:
153-
logger.warning("Downloaded file is empty!")
154-
return False
155-
elif not archive_path.exists():
156-
logger.error("File was not downloaded!")
194+
# ------------------------------------------------------------
195+
# 3. Validate final size
196+
# ------------------------------------------------------------
197+
final_size = archive_path.stat().st_size
198+
if final_size != total_size:
199+
logger.error(
200+
"Download incomplete: expected %d, got %d",
201+
total_size,
202+
final_size,
203+
)
157204
return False
158205

206+
logger.info("Download completed successfully!")
159207
return True
160208

161209

‎tools/cmake/preset/pybind.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux")
3737
set_overridable_option(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER ON)
3838
set_overridable_option(EXECUTORCH_BUILD_EXTENSION_LLM ON)
3939
if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|amd64|i.86)$")
40-
set_overridable_option(EXECUTORCH_BUILD_QNN OFF)
40+
set_overridable_option(EXECUTORCH_BUILD_QNN ON)
4141
endif()
4242
elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows" OR CMAKE_SYSTEM_NAME STREQUAL
4343
"WIN32"

0 commit comments

Comments
 (0)