Skip to content

Commit 673af1a

Browse files
authored
feat(download_lt): add hash verification for downloaded ZIPs (jxmorris12#164)
1 parent a95d3da commit 673af1a

5 files changed

Lines changed: 255 additions & 3 deletions

File tree

‎README.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -312,12 +312,24 @@ Exit codes:
312312
- default: `https://languagetool.org/download/`
313313
- `LTP_DOWNLOAD_HOST_ARCHIVE`: override archive download host.
314314
- default: `https://languagetool.org/download/archive/`
315+
- `LTP_DOWNLOAD_SHA256_<VERSION>`: version-specific expected SHA-256 for the downloaded LanguageTool archive, for example `LTP_DOWNLOAD_SHA256_6_9_SNAPSHOT`.
316+
- `LTP_DOWNLOAD_SHA256`: fallback expected SHA-256 for the downloaded LanguageTool archive.
317+
- `LTP_BYPASS_VERIFIED_DOWNLOADS`: set to `true` to skip SHA-256 verification.
318+
319+
Downloaded zips are verified with SHA-256 when a checksum is available. Checksums are resolved in this order:
320+
1. `LTP_DOWNLOAD_SHA256_<VERSION>`, where non-alphanumeric characters in the version are replaced with `_` and the name is uppercased.
321+
2. `LTP_DOWNLOAD_SHA256`.
322+
3. The bundled `language_tool_python/integrity.toml` manifest.
323+
324+
The bundled manifest covers release/archive downloads. Snapshots are not stable, so provide `LTP_DOWNLOAD_SHA256_<VERSION>` or `LTP_DOWNLOAD_SHA256` if you want to verify a snapshot. If no checksum is available, the download proceeds without SHA-256 verification.
315325

316326
Example:
317327

318328
```bash
319329
export LTP_PATH=/path/to/cache
320330
export LTP_JAR_DIR_PATH=/path/to/LanguageTool-6.9-SNAPSHOT
331+
export LTP_DOWNLOAD_SHA256_6_9_SNAPSHOT=<sha256>
332+
# export LTP_BYPASS_VERIFIED_DOWNLOADS=true
321333
```
322334

323335
## Resource Management

‎language_tool_python/download_lt.py‎

Lines changed: 60 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
"""LanguageTool download module."""
22

33
import contextlib
4+
import hashlib
5+
import importlib.resources
46
import logging
57
import os
68
import re
@@ -14,8 +16,10 @@
1416
from shutil import which
1517
from typing import IO, Dict, List, Optional, Tuple, Union
1618
from urllib.parse import urljoin
19+
from warnings import warn
1720

1821
import requests
22+
import toml
1923
import tqdm
2024
from packaging import version
2125
from packaging.version import Version
@@ -49,6 +53,16 @@
4953

5054
LTP_DOWNLOAD_VERSION = "latest"
5155
LT_SNAPSHOT_CURRENT_VERSION = "6.9-SNAPSHOT"
56+
LTP_DOWNLOAD_SHA256_ENV_VAR = "LTP_DOWNLOAD_SHA256"
57+
LTP_BYPASS_VERIFIED_DOWNLOADS_ENV_VAR = "LTP_BYPASS_VERIFIED_DOWNLOADS"
58+
59+
with (
60+
importlib.resources.as_file(
61+
importlib.resources.files("language_tool_python").joinpath("integrity.toml")
62+
) as hashes_path,
63+
open(hashes_path, "rb") as f,
64+
):
65+
EXPECTED_DOWNLOAD_SHA256 = toml.loads(f.read().decode("utf-8"))
5266

5367
JAVA_VERSION_REGEX = re.compile(
5468
r'^(?:java|openjdk) version "(?P<major1>\d+)(|\.(?P<major2>\d+)\.[^"]+)"',
@@ -62,6 +76,39 @@
6276
)
6377

6478

79+
def _get_zip_hash(version_name: str) -> Optional[str]:
80+
"""Get the expected SHA-256 hash for a given version of LanguageTool.
81+
This function checks for environment variables that may specify the expected hash for the given version. It normalizes the version name to construct the environment variable name. If no specific environment variable is found for the version, it falls back to a general environment variable or a manifest lookup. If the bypass environment variable is set, it will skip verification and return None.
82+
83+
:param version_name: The version name of LanguageTool (e.g., '6.0', '20240101', or 'latest').
84+
:type version_name: str
85+
:return: The expected SHA-256 hash for the given version, or None if verification is bypassed or no hash is configured.
86+
:rtype: Optional[str]
87+
"""
88+
if os.environ.get(LTP_BYPASS_VERIFIED_DOWNLOADS_ENV_VAR, "").lower() == "true":
89+
err = (
90+
f"Verified downloads are bypassed. No SHA-256 checksum will be used for "
91+
f"LanguageTool {version_name}. Set {LTP_BYPASS_VERIFIED_DOWNLOADS_ENV_VAR}=false to re-enable verification."
92+
)
93+
warn(err, RuntimeWarning, stacklevel=2)
94+
return None
95+
suffix = re.sub(r"[^A-Za-z0-9]+", "_", version_name).strip("_").upper()
96+
version_env_var = f"LTP_DOWNLOAD_SHA256_{suffix}"
97+
configured = (
98+
(os.environ.get(version_env_var), version_env_var),
99+
(os.environ.get(LTP_DOWNLOAD_SHA256_ENV_VAR), LTP_DOWNLOAD_SHA256_ENV_VAR),
100+
(EXPECTED_DOWNLOAD_SHA256.get(version_name), f"manifest:{version_name}"),
101+
)
102+
for checksum, source in configured:
103+
if checksum:
104+
normalized = checksum.strip().lower()
105+
if not re.fullmatch(r"[0-9a-f]{64}", normalized):
106+
err = f"Invalid SHA-256 checksum configured by {source}."
107+
raise PathError(err)
108+
return normalized
109+
return None
110+
111+
65112
def parse_java_version(version_text: str) -> Tuple[int, int]:
66113
"""
67114
Parse the Java version from a given version text.
@@ -360,9 +407,11 @@ def _get_remote_zip(
360407
:return: A ZipFile object of the downloaded archive.
361408
:rtype: zipfile.ZipFile
362409
:raises TimeoutError: If the download request times out.
363-
:raises PathError: If the download fails due to HTTP errors (404, 403, etc.).
410+
:raises PathError: If the download fails due to HTTP errors (404, 403, etc.) or if the checksum does not match.
364411
"""
365412
logger.info("Starting download from %s", self.download_url)
413+
expected_sha256 = _get_zip_hash(self.version_name)
414+
sha256 = hashlib.sha256()
366415
try:
367416
req = requests.get(
368417
self.download_url, stream=True, proxies=proxies, timeout=60
@@ -389,9 +438,19 @@ def _get_remote_zip(
389438
)
390439
for chunk in req.iter_content(chunk_size=1024):
391440
if chunk: # filter out keep-alive new chunks
441+
sha256.update(chunk)
392442
progress.update(len(chunk))
393443
downloaded_file.write(chunk)
394444
progress.close()
445+
actual_sha256 = sha256.hexdigest()
446+
logger.debug("Download completed. SHA-256: %s", actual_sha256)
447+
if expected_sha256 is not None and actual_sha256 != expected_sha256:
448+
err = (
449+
f"Downloaded LanguageTool archive checksum mismatch. "
450+
f"Expected {expected_sha256}, got {actual_sha256}."
451+
)
452+
raise PathError(err)
453+
downloaded_file.seek(0)
395454
return zipfile.ZipFile(downloaded_file)
396455

397456
@classmethod
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# Hashes of languagetool zip files that are available at
2+
# https://languagetool.org/download/ and https://languagetool.org/download/archive/
3+
# The hashes are calculated with sha256 and are used to check the integrity of the downloaded zip files.
4+
# This manifest only contains hashes for archives and releases and not for snapshots because
5+
# they are not stable.
6+
7+
"4.0"="0e8c40f3731213fdd3260cc1b8637401ac5f25bf1ddba756d129225f5eeaa1c5"
8+
"4.1"="350ae3584d72dfc769f63adbf46a438e3a66ad86c9cdd4dd0c3e3c44ed38b31e"
9+
"4.2"="2621801fb4c62f78cc813c17d67c70d39229b8982040b927325754eb7e90135b"
10+
"4.3"="65a9f222329390497c3db7944ad5f3a7ee2d54957a956ccf15b4a62a1a561aa4"
11+
"4.4"="8362ecd238d33866919516f69f440d12411a0ea509e0d54b1ff08aef3ebea5db"
12+
"4.5"="61e239148309580d02f1729c3b5a7da61eaa2cf36898a5be6daaac95263f1af9"
13+
"4.6"="5fd82f2cae2254c682e603a8969a2af292c72c576b2db52dd680f0ec06abce3d"
14+
"4.7"="4f00f71930871600cb8fe53a8c4d4c9c60f05711c979fd313be1c6c674efde3c"
15+
"4.8"="b4775e88fd90d6389f86023600c4c0e16a9cc00cfe8f093bf1f0c2bbf2a1904d"
16+
"4.9"="2e6998563a3ee4b142ffe91eb20d3ae3e15ade5f507964b9dc2ec19373329b35"
17+
"5.0"="212eb275cf9df8b6f49d8ce2bb386246081be7f01d907577fbd09a9f27055628"
18+
"5.1"="c91ff43d54c863a9389a81c5fa1a6723041b982435f220e9c2c4db843b66d1b6"
19+
"5.2"="e7776143c76a88449d451897cedc9f3ce698450e25bce25d4ed52457fa2d0cde"
20+
"5.3"="3303c97a331605d8ffa496bceb08120835543ccd4408e8067178fb9f16476dc2"
21+
"5.4"="bb02957a2967c82424cc1c6657e704dbc3dfe54078f3d8134646741471b2bc2b"
22+
"5.5"="18d16d103f1e59b125a1f48ff9d3c23051f21a80b2ca64a1da305d576245f90d"
23+
"5.6"="6cabbf546cf7fab2f4b8493827657d528a78add565b219cb2e27001612925b7e"
24+
"5.7"="0c7ca3f7fa94d298c4ffb9d342b33e31cf6e93015602227b94924e3237b5ce79"
25+
"5.8"="20913a50eb4568fdc727066eee8aab833f52b626710fc9e0213c3f1098bfdee3"
26+
"5.9"="0d06c2e8b016c105d24a242aa4967329a4cc8f7ffc95fb0e11113bd9db48c4ac"
27+
"6.0"="d1a2ab7f59606003db139c65dd76d347d27d298cc465132a970fd167fdacde4f"
28+
"6.1"="ea4d8bb3a92d109e63ab6d0ef6a1ef1b5b73f4dd16cdd7b2a7a02a0e74c7651d"
29+
"6.2"="1e8a10ac8e6c40c077c1640eb85fa9ca615a683fecdf7e00e51e463dcd0a0f49"
30+
"6.3"="32f529f249b15f758b38cc00e4818af3b3478490c2a0ae2825e2b5e980473e08"
31+
"6.4"="d5d517335d1ca33f33412920447fa0c742ef78d645521d650e0e22628fe0d3bd"
32+
"6.5"="27f3ae5a29efbc8267a5a266908dfec205d16d312af8516e0b5bcec871edea0b"
33+
"6.6"="53600506b399bb5ffe1e4c8dec794fd378212f14aaf38ccef9b6f89314d11631"
34+
"6.7"="53600506b399bb5ffe1e4c8dec794fd378212f14aaf38ccef9b6f89314d11631"

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ build-backend = "setuptools.build_meta"
7070
packages = { find = { include = ["language_tool_python*"] } }
7171

7272
[tool.setuptools.package-data]
73-
language_tool_python = ["py.typed", "logging.toml"]
73+
language_tool_python = ["py.typed", "integrity.toml", "logging.toml"]
7474

7575
[tool.ruff.lint]
7676
select = ["F", "W", "I", "B", "SIM", "RET", "Q", "N", "S", "BLE"]

‎tests/test_download.py‎

Lines changed: 148 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,48 @@
11
"""Tests for the download/language functionality of LanguageTool."""
22

3+
import hashlib
34
import io
5+
import re
6+
import zipfile
7+
from collections.abc import Iterator
48
from unittest.mock import MagicMock, patch
59

610
import pytest
711

8-
from language_tool_python.download_lt import LocalLanguageTool
12+
from language_tool_python.download_lt import (
13+
LTP_BYPASS_VERIFIED_DOWNLOADS_ENV_VAR,
14+
LTP_DOWNLOAD_SHA256_ENV_VAR,
15+
LocalLanguageTool,
16+
)
917
from language_tool_python.exceptions import LanguageToolError, PathError
1018

1119

20+
class MockDownloadResponse:
21+
"""
22+
Minimal requests.Response replacement for download tests.
23+
"""
24+
25+
def __init__(self, payload: bytes, status_code: int = 200) -> None:
26+
self.payload = payload
27+
self.status_code = status_code
28+
self.headers = {"Content-Length": str(len(payload))}
29+
30+
def iter_content(self, chunk_size: int) -> Iterator[bytes]:
31+
for index in range(0, len(self.payload), chunk_size):
32+
yield self.payload[index : index + chunk_size]
33+
34+
35+
def make_zip_payload(files: dict[str, bytes]) -> bytes:
36+
"""
37+
Create an in-memory ZIP payload for download tests.
38+
"""
39+
buffer = io.BytesIO()
40+
with zipfile.ZipFile(buffer, "w") as zip_file:
41+
for filename, payload in files.items():
42+
zip_file.writestr(filename, payload)
43+
return buffer.getvalue()
44+
45+
1246
def test_install_inexistent_version() -> None:
1347
"""
1448
Test that attempting to download a non-existent LanguageTool version raises an error.
@@ -101,6 +135,119 @@ def test_http_get_other_error_codes() -> None:
101135
local_language_tool._get_remote_zip(out_file)
102136

103137

138+
def test_http_get_verifies_configured_sha256(
139+
monkeypatch: pytest.MonkeyPatch,
140+
) -> None:
141+
"""
142+
Test that downloads are accepted when the configured SHA-256 matches.
143+
"""
144+
payload = make_zip_payload(
145+
{"LanguageTool-6.9-SNAPSHOT/languagetool-server.jar": b"jar"}
146+
)
147+
local_language_tool = LocalLanguageTool.from_version_name()
148+
suffix = (
149+
re.sub(r"[^A-Za-z0-9]+", "_", local_language_tool.version_name)
150+
.strip("_")
151+
.upper()
152+
)
153+
version_env_var = f"LTP_DOWNLOAD_SHA256_{suffix}"
154+
monkeypatch.setenv(
155+
version_env_var,
156+
hashlib.sha256(payload).hexdigest(),
157+
)
158+
159+
with patch(
160+
"language_tool_python.download_lt.requests.get",
161+
return_value=MockDownloadResponse(payload),
162+
):
163+
out_file = io.BytesIO()
164+
with local_language_tool._get_remote_zip(out_file) as zip_file:
165+
assert zip_file.namelist() == [
166+
"LanguageTool-6.9-SNAPSHOT/languagetool-server.jar"
167+
]
168+
169+
170+
def test_http_get_uses_integrity_manifest_sha256(
171+
monkeypatch: pytest.MonkeyPatch,
172+
) -> None:
173+
"""
174+
Test that bundled integrity.toml checksums are used when no env var is set.
175+
"""
176+
payload = make_zip_payload({"LanguageTool-4.0/languagetool-server.jar": b"jar"})
177+
local_language_tool = LocalLanguageTool.from_version_name("4.0")
178+
monkeypatch.delenv(LTP_BYPASS_VERIFIED_DOWNLOADS_ENV_VAR, raising=False)
179+
monkeypatch.delenv(LTP_DOWNLOAD_SHA256_ENV_VAR, raising=False)
180+
monkeypatch.delenv("LTP_DOWNLOAD_SHA256_4_0", raising=False)
181+
182+
with (
183+
patch(
184+
"language_tool_python.download_lt.requests.get",
185+
return_value=MockDownloadResponse(payload),
186+
),
187+
pytest.raises(PathError, match="checksum mismatch"),
188+
):
189+
local_language_tool._get_remote_zip(io.BytesIO())
190+
191+
192+
def test_http_get_rejects_sha256_mismatch(
193+
monkeypatch: pytest.MonkeyPatch,
194+
) -> None:
195+
"""
196+
Test that downloads are rejected when the configured SHA-256 mismatches.
197+
"""
198+
payload = make_zip_payload(
199+
{"LanguageTool-6.9-SNAPSHOT/languagetool-server.jar": b"jar"}
200+
)
201+
local_language_tool = LocalLanguageTool.from_version_name()
202+
suffix = (
203+
re.sub(r"[^A-Za-z0-9]+", "_", local_language_tool.version_name)
204+
.strip("_")
205+
.upper()
206+
)
207+
version_env_var = f"LTP_DOWNLOAD_SHA256_{suffix}"
208+
monkeypatch.setenv(
209+
version_env_var,
210+
"0" * 64,
211+
)
212+
213+
with (
214+
patch(
215+
"language_tool_python.download_lt.requests.get",
216+
return_value=MockDownloadResponse(payload),
217+
),
218+
pytest.raises(PathError, match="checksum mismatch"),
219+
):
220+
out_file = io.BytesIO()
221+
local_language_tool._get_remote_zip(out_file)
222+
223+
224+
def test_http_get_bypass_skips_sha256_verification(
225+
monkeypatch: pytest.MonkeyPatch,
226+
) -> None:
227+
"""
228+
Test that the bypass disables SHA-256 verification.
229+
"""
230+
payload = make_zip_payload(
231+
{"LanguageTool-6.9-SNAPSHOT/languagetool-server.jar": b"jar"}
232+
)
233+
local_language_tool = LocalLanguageTool.from_version_name()
234+
monkeypatch.setenv(LTP_BYPASS_VERIFIED_DOWNLOADS_ENV_VAR, "true")
235+
monkeypatch.setenv(LTP_DOWNLOAD_SHA256_ENV_VAR, "0" * 64)
236+
237+
with (
238+
patch(
239+
"language_tool_python.download_lt.requests.get",
240+
return_value=MockDownloadResponse(payload),
241+
),
242+
pytest.warns(RuntimeWarning, match="Verified downloads are bypassed"),
243+
):
244+
out_file = io.BytesIO()
245+
with local_language_tool._get_remote_zip(out_file) as zip_file:
246+
assert zip_file.namelist() == [
247+
"LanguageTool-6.9-SNAPSHOT/languagetool-server.jar"
248+
]
249+
250+
104251
def test_install_oldest_supported_version() -> None:
105252
"""
106253
Test that downloading the oldest supported LanguageTool version works correctly.

0 commit comments

Comments
 (0)