From df1df3b74a000626120b6968dcb8086a269f6bcc Mon Sep 17 00:00:00 2001 From: Ibrahim Ezzy Date: Sun, 2 Aug 2026 19:23:52 +0400 Subject: [PATCH] fix(machinery): preserve literal HTML entities XML-based machinery escapes source text before sending it to a provider and unescapes every returned result. That round trip is only lossless while the provider keeps the escaping level it was given, and HTML-mode providers renormalize it. When they do, an inert character reference in the source comes back one level decoded and the unconditional unescape turns it into live markup. With the safe-html flag set, the sanitizer then drops the element and everything after it, so the translation loses content with no check and no error. Protect the character-reference spans as non-translatable, reusing the mechanism which already keeps units flagged xml-text intact. The spans recognized are exactly those Python unescaping consumes, including the recognized prefix of a legacy reference written without its semicolon, so no reference which would survive the round trip is touched. Spans which partially overlap an existing highlight are merged with it, so a placeholder can no longer expose part of a reference. The unconditional unescape stays as it is; raw ampersands returned escaped still restore correctly. Glossary insertion for Azure AI Translator only compared a term against highlights starting after it, so a term matching inside a now-protected reference produced overlapping spans and duplicated text on restore. Skip terms overlapping any highlight instead. --- docs/changes.rst | 1 + weblate/machinery/base.py | 37 +++++++ weblate/machinery/microsoft.py | 22 ++-- weblate/machinery/tests.py | 175 +++++++++++++++++++++++++++++++ weblate/utils/html.py | 26 +++++ weblate/utils/tests/test_html.py | 52 +++++++++ 6 files changed, 300 insertions(+), 13 deletions(-) diff --git a/docs/changes.rst b/docs/changes.rst index 6b9abf6f1394..0a090417be6d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -49,6 +49,7 @@ Weblate 2026.8 * REST API authorization now consistently protects internal accounts, restricted components, add-on configuration, component sharing, repository links, and review states. * :ref:`Project backup imports ` now validate restored data before creating project state, skip repository-linked components the importer cannot access, and no longer load archive-supplied Mercurial configuration or shared-repository indirection. * Rebuilding project translation memory no longer removes entries imported from files. +* Literal HTML character references in source strings are now preserved by :ref:`mt-deepl`, :ref:`mt-google-translate-api-v3` and :ref:`mt-microsoft-translator` machine translation instead of being decoded into live markup. * Suggestion submission and rejection now reject excessively long suggestion text and rejection reasons. * Restricted components are now available on Hosted Weblate when the billing plan permits private projects. * Machine translation and translation memory AJAX lookups no longer disclose whether inaccessible unit IDs exist. diff --git a/weblate/machinery/base.py b/weblate/machinery/base.py index 501039e70596..980635b7f0d7 100644 --- a/weblate/machinery/base.py +++ b/weblate/machinery/base.py @@ -33,6 +33,7 @@ from weblate.utils.errors import log_handled_exception, report_error from weblate.utils.forms import WeblateServiceURLField from weblate.utils.hash import calculate_dict_hash, calculate_hash, hash_to_checksum +from weblate.utils.html import iter_html_entities from weblate.utils.outbound import is_allowlisted_hostname from weblate.utils.requests import ( JSON_RESPONSE_ERRORS, @@ -1548,6 +1549,42 @@ class XMLMachineTranslationMixin(BatchMachineTranslation): highlight_syntax = True force_uncleanup = True + def get_highlights( + self, text: str, unit: Unit + ) -> Iterable[tuple[int, int, str, Highlight | Unit | None]]: + highlights = list(super().get_highlights(text, unit)) + + for entity_start, entity_end, entity_text in iter_html_entities(text): + entity = (entity_start, entity_end, entity_text, None) + overlapping = [ + index + for index, (start, end, _highlight_text, _kind) in enumerate(highlights) + if entity_start < end and entity_end > start + ] + if overlapping: + first = overlapping[0] + last = overlapping[-1] + if any( + start <= entity_start and end >= entity_end + for start, end, _highlight_text, _kind in highlights[ + first : last + 1 + ] + ): + continue + start = min(entity_start, highlights[first][0]) + end = max(entity_end, highlights[last][1]) + highlights[first : last + 1] = [(start, end, text[start:end], None)] + continue + + for index, (start, _end, _highlight_text, _kind) in enumerate(highlights): + if entity_end <= start: + highlights.insert(index, entity) + break + else: + highlights.append(entity) + + yield from highlights + def unescape_text(self, text: str) -> str: """Unescaping of the text with replacements.""" return unescape(text) diff --git a/weblate/machinery/microsoft.py b/weblate/machinery/microsoft.py index 7390698ceb71..c17e7f9c6e27 100644 --- a/weblate/machinery/microsoft.py +++ b/weblate/machinery/microsoft.py @@ -4,7 +4,9 @@ from __future__ import annotations +from bisect import insort from datetime import timedelta +from operator import itemgetter from typing import TYPE_CHECKING, ClassVar from django.utils import timezone @@ -198,18 +200,12 @@ def get_highlights(self, text, unit): for term in get_glossary_terms(unit, include_variants=False): for start, end in term.glossary_positions: - glossary_highlight = (start, end, text[start:end], term) - handled = False - for i, (h_start, _h_end, _h_text, _h_kind) in enumerate(result): - if start < h_start: - if end > h_start: - # Skip as overlaps - break - # Insert before - result.insert(i, glossary_highlight) - handled = True - break - if not handled and (not result or result[-1][1] < start): - result.append(glossary_highlight) + if any( + start < h_end and end > h_start + for h_start, h_end, _h_text, _h_kind in result + ): + # Skip as overlaps + continue + insort(result, (start, end, text[start:end], term), key=itemgetter(0)) yield from result diff --git a/weblate/machinery/tests.py b/weblate/machinery/tests.py index dac8a39662d6..b1646f0f83ed 100644 --- a/weblate/machinery/tests.py +++ b/weblate/machinery/tests.py @@ -12,6 +12,7 @@ from functools import partial from io import StringIO from pathlib import Path +from types import SimpleNamespace from typing import TYPE_CHECKING, ClassVar, NoReturn, cast from unittest.mock import AsyncMock, MagicMock, Mock, call, patch from urllib.parse import parse_qs, urlparse @@ -1256,6 +1257,43 @@ def test_map_codes(self) -> None: self.assertEqual(machine.map_language_code("fr_CA"), "fr-ca") self.assertEqual(machine.map_language_code("iu_Latn"), "iu-Latn") + def test_literal_entity_replacements(self) -> None: + machine = self.get_machine() + unit = make_unit(code="cs", source="Hello <script>.") + replaced = ( + 'Hello &amp;lt;script' + '&amp;gt;.' + ) + replacements = { + '&amp;lt;': "&lt;", + '&amp;gt;': "&gt;", + } + self.assertEqual( + machine.cleanup_text(unit.source, unit), + (replaced, replacements), + ) + self.assertEqual(unit.source, machine.uncleanup_text(replacements, replaced)) + + def test_glossary_term_overlapping_literal_entity(self) -> None: + """A glossary term inside a character reference must not be highlighted.""" + machine = self.get_machine() + source = "The ©right notice and <b> tag." + unit = make_unit(code="cs", source=source) + term = SimpleNamespace(glossary_positions=[(5, 14)], target="Copyright") + + with patch( + "weblate.machinery.microsoft.get_glossary_terms", + return_value=[term], + ): + highlights = list(machine.get_highlights(source, unit)) + replaced, replacements = machine.cleanup_text(source, unit) + + self.assertEqual( + highlights, + [(4, 9, "©", None), (26, 30, "<", None), (31, 35, ">", None)], + ) + self.assertEqual(source, machine.uncleanup_text(replacements, replaced)) + @patch( "weblate.utils.outbound.socket.getaddrinfo", return_value=[(0, 0, 0, "", ("127.0.0.1", 443))], @@ -1527,6 +1565,26 @@ def test_replacements(self) -> None: unit.source, machine_translation.uncleanup_text(replacements, replaced) ) + def test_literal_entity_replacements(self) -> None: + machine_translation = self.get_machine() + unit = make_unit(code="cs", source="Hello <script>.") + replaced = ( + 'Hello &amp;lt;script' + '&amp;gt;.' + ) + replacements = { + '&amp;lt;': "&lt;", + '&amp;gt;': "&gt;", + '
': "\n", + } + self.assertEqual( + machine_translation.cleanup_text(unit.source, unit), + (replaced, replacements), + ) + self.assertEqual( + unit.source, machine_translation.uncleanup_text(replacements, replaced) + ) + # set glossary_count_limit to 1 to also trigger delete_oldest_glossary @patch("weblate.glossary.models.get_glossary_tsv", new=lambda _: "foo\tbar") @patch("weblate.machinery.googlev3.GoogleV3Translation.glossary_count_limit", new=1) @@ -2548,6 +2606,123 @@ def request_callback(request: httpx2.Request): self.assertEqual(translation[0][0]["source"], "Hello&world") self.assertEqual(translation[0][0]["text"], "Hallo&welt") + def mock_entity_normalizing_response(self) -> None: + def request_callback(request: httpx2.Request): + payload = load_request_json(request) + texts = cast("list[str]", payload["text"]) + translated = texts[0].replace("Hello", "Hallo").replace("&", "&") + return httpx2.Response( + 200, + headers={}, + text=json.dumps( + { + "translations": [ + {"detected_source_language": "EN", "text": translated} + ] + } + ), + ) + + self.mock_languages() + http_mock.register_callback( + "POST", + "https://api.deepl.com/v2/translate", + callback=request_callback, + ) + + @http_mock.activate + def test_literal_entities_preserved(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + translation = self.assert_translate( + self.SUPPORTED, + "Hello <script>.", + 1, + machine=machine, + unit_args={"flags": "safe-html"}, + ) + self.assertEqual(translation[0][0]["source"], "Hello <script>.") + self.assertEqual(translation[0][0]["text"], "Hallo <script>.") + + @http_mock.activate + def test_literal_entity_variants_preserved(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + source = "Hello < < < < © © &unknown;." + translation = self.assert_translate(self.SUPPORTED, source, 1, machine=machine) + self.assertEqual(translation[0][0]["source"], source) + self.assertEqual( + translation[0][0]["text"], + "Hallo < < < < © © &unknown;.", + ) + + def test_literal_entity_merges_with_overlapping_highlight(self) -> None: + """A highlight covering part of an entity is widened to cover all of it.""" + machine = self.MACHINE_CLS(self.get_configuration()) + source = "The © is here" + unit = cast( + "Unit", make_unit(code="de", source=source, flags='placeholders:"y; is"') + ) + + self.assertEqual( + list(machine.get_highlights(source, unit)), + [(4, 13, "© is", None)], + ) + replaced, replacements = machine.cleanup_text(source, unit) + self.assertNotIn("&cop", replaced) + self.assertEqual(source, machine.uncleanup_text(replacements, replaced)) + + @http_mock.activate + def test_literal_entities_with_xml_highlights(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + source = 'Hello <script>.' + translation = self.assert_translate( + self.SUPPORTED, + source, + 1, + machine=machine, + unit_args={"flags": "xml-text"}, + ) + self.assertEqual(translation[0][0]["source"], source) + self.assertEqual( + translation[0][0]["text"], + 'Hallo <script>.', + ) + + @http_mock.activate + def test_literal_entity_contains_existing_highlight(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + source = "Hello ©." + translation = self.assert_translate( + self.SUPPORTED, + source, + 1, + machine=machine, + unit_args={"flags": 'placeholders:"copy"'}, + ) + self.assertEqual(translation[0][0]["source"], source) + self.assertEqual(translation[0][0]["text"], "Hallo ©.") + + def test_literal_entity_highlight_excludes_trailing_text(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + source = "Hello a¬b." + unit = cast("Unit", make_unit(code="de", source=source)) + + self.assertEqual( + list(machine.get_highlights(source, unit)), + [(7, 11, "¬", None)], + ) + @http_mock.activate @patch("weblate.glossary.models.get_glossary_tsv", new=lambda _: "foo\tbar") def test_glossary(self) -> None: diff --git a/weblate/utils/html.py b/weblate/utils/html.py index e9d6f8a04fba..deca63e41244 100644 --- a/weblate/utils/html.py +++ b/weblate/utils/html.py @@ -7,6 +7,7 @@ import re import threading from collections import defaultdict +from html.entities import html5 from html.parser import HTMLParser as StdHTMLParser from typing import TYPE_CHECKING, Any, NamedTuple @@ -72,6 +73,31 @@ ) MD_SYNTAX_GROUPS = 8 +HTML_ENTITY_CANDIDATE = re.compile( + r"&(?:#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)" +) + + +def iter_html_entities(text: str) -> Iterable[tuple[int, int, str]]: + """Yield character-reference spans consumed by html.unescape.""" + for match in HTML_ENTITY_CANDIDATE.finditer(text): + entity = match.group() + if entity[1] == "#": + yield match.start(), match.end(), entity + continue + + name = entity[1:] + if name in html5: + yield match.start(), match.end(), entity + continue + + for length in range(len(name) - 1, 1, -1): + if name[:length] in html5: + end = match.start() + length + 1 + yield match.start(), end, text[match.start() : end] + break + + AUTO_SAFE_HTML_START = re.compile(r"<(?=[!/?A-Za-z])") AUTO_SAFE_HTML_SEGMENT = re.compile( r""" diff --git a/weblate/utils/tests/test_html.py b/weblate/utils/tests/test_html.py index e58672266b4d..6ce47ac26e70 100644 --- a/weblate/utils/tests/test_html.py +++ b/weblate/utils/tests/test_html.py @@ -4,6 +4,8 @@ from __future__ import annotations +from html import unescape + from django.test import SimpleTestCase from weblate.checks.flags import Flags @@ -14,6 +16,7 @@ extract_html_attributes, extract_html_tags, is_auto_safe_html_source, + iter_html_entities, list_to_tuples, mail_quote_value, ) @@ -216,3 +219,52 @@ def test_empty_list(self) -> None: def test_single_element_list(self) -> None: self.assertEqual(list(list_to_tuples(["only_one"])), [("only_one",)]) + + +class IterHTMLEntitiesTestCase(SimpleTestCase): + def test_named(self) -> None: + self.assertEqual( + list(iter_html_entities("a < b & c")), + [(2, 6, "<"), (9, 14, "&")], + ) + + def test_numeric(self) -> None: + self.assertEqual( + list(iter_html_entities("< < <")), + [(0, 5, "<"), (6, 12, "<"), (13, 19, "<")], + ) + + def test_missing_semicolon(self) -> None: + # Legacy references are decoded without the trailing semicolon + self.assertEqual(list(iter_html_entities("© 2026")), [(0, 5, "©")]) + + def test_longest_prefix(self) -> None: + # Only the recognized prefix is a reference, the rest is plain text + self.assertEqual(list(iter_html_entities("a¬b")), [(1, 5, "¬")]) + + def test_unknown(self) -> None: + for text in ("&unknown;", "R&D", "Tom & Jerry", "&", "&;", "&#", "&#x"): + with self.subTest(text=text): + self.assertEqual(list(iter_html_entities(text)), []) + + def test_long_name(self) -> None: + # The longest name is 32 characters including the semicolon + longest = "∳" + self.assertEqual(list(iter_html_entities(longest)), [(0, 33, longest)]) + + def test_matches_unescape(self) -> None: + """Yielded spans are exactly what unescaping consumes.""" + for text in ( + "a < b", + "© 2026", + "a¬b", + "&unknown;", + "R&D", + "<<", + "&lt;", + "?a=1¬ify=2", + ): + with self.subTest(text=text): + for start, end, entity in iter_html_entities(text): + self.assertEqual(entity, text[start:end]) + self.assertNotEqual(unescape(entity), entity)