diff --git a/docs/changes.rst b/docs/changes.rst index 6b9abf6f1394..0a090417be6d 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -49,6 +49,7 @@ Weblate 2026.8 * REST API authorization now consistently protects internal accounts, restricted components, add-on configuration, component sharing, repository links, and review states. * :ref:`Project backup imports ` now validate restored data before creating project state, skip repository-linked components the importer cannot access, and no longer load archive-supplied Mercurial configuration or shared-repository indirection. * Rebuilding project translation memory no longer removes entries imported from files. +* Literal HTML character references in source strings are now preserved by :ref:`mt-deepl`, :ref:`mt-google-translate-api-v3` and :ref:`mt-microsoft-translator` machine translation instead of being decoded into live markup. * Suggestion submission and rejection now reject excessively long suggestion text and rejection reasons. * Restricted components are now available on Hosted Weblate when the billing plan permits private projects. * Machine translation and translation memory AJAX lookups no longer disclose whether inaccessible unit IDs exist. diff --git a/weblate/machinery/base.py b/weblate/machinery/base.py index 501039e70596..980635b7f0d7 100644 --- a/weblate/machinery/base.py +++ b/weblate/machinery/base.py @@ -33,6 +33,7 @@ from weblate.utils.errors import log_handled_exception, report_error from weblate.utils.forms import WeblateServiceURLField from weblate.utils.hash import calculate_dict_hash, calculate_hash, hash_to_checksum +from weblate.utils.html import iter_html_entities from weblate.utils.outbound import is_allowlisted_hostname from weblate.utils.requests import ( JSON_RESPONSE_ERRORS, @@ -1548,6 +1549,42 @@ class XMLMachineTranslationMixin(BatchMachineTranslation): highlight_syntax = True force_uncleanup = True + def get_highlights( + self, text: str, unit: Unit + ) -> Iterable[tuple[int, int, str, Highlight | Unit | None]]: + highlights = list(super().get_highlights(text, unit)) + + for entity_start, entity_end, entity_text in iter_html_entities(text): + entity = (entity_start, entity_end, entity_text, None) + overlapping = [ + index + for index, (start, end, _highlight_text, _kind) in enumerate(highlights) + if entity_start < end and entity_end > start + ] + if overlapping: + first = overlapping[0] + last = overlapping[-1] + if any( + start <= entity_start and end >= entity_end + for start, end, _highlight_text, _kind in highlights[ + first : last + 1 + ] + ): + continue + start = min(entity_start, highlights[first][0]) + end = max(entity_end, highlights[last][1]) + highlights[first : last + 1] = [(start, end, text[start:end], None)] + continue + + for index, (start, _end, _highlight_text, _kind) in enumerate(highlights): + if entity_end <= start: + highlights.insert(index, entity) + break + else: + highlights.append(entity) + + yield from highlights + def unescape_text(self, text: str) -> str: """Unescaping of the text with replacements.""" return unescape(text) diff --git a/weblate/machinery/microsoft.py b/weblate/machinery/microsoft.py index 7390698ceb71..c17e7f9c6e27 100644 --- a/weblate/machinery/microsoft.py +++ b/weblate/machinery/microsoft.py @@ -4,7 +4,9 @@ from __future__ import annotations +from bisect import insort from datetime import timedelta +from operator import itemgetter from typing import TYPE_CHECKING, ClassVar from django.utils import timezone @@ -198,18 +200,12 @@ def get_highlights(self, text, unit): for term in get_glossary_terms(unit, include_variants=False): for start, end in term.glossary_positions: - glossary_highlight = (start, end, text[start:end], term) - handled = False - for i, (h_start, _h_end, _h_text, _h_kind) in enumerate(result): - if start < h_start: - if end > h_start: - # Skip as overlaps - break - # Insert before - result.insert(i, glossary_highlight) - handled = True - break - if not handled and (not result or result[-1][1] < start): - result.append(glossary_highlight) + if any( + start < h_end and end > h_start + for h_start, h_end, _h_text, _h_kind in result + ): + # Skip as overlaps + continue + insort(result, (start, end, text[start:end], term), key=itemgetter(0)) yield from result diff --git a/weblate/machinery/tests.py b/weblate/machinery/tests.py index dac8a39662d6..b1646f0f83ed 100644 --- a/weblate/machinery/tests.py +++ b/weblate/machinery/tests.py @@ -12,6 +12,7 @@ from functools import partial from io import StringIO from pathlib import Path +from types import SimpleNamespace from typing import TYPE_CHECKING, ClassVar, NoReturn, cast from unittest.mock import AsyncMock, MagicMock, Mock, call, patch from urllib.parse import parse_qs, urlparse @@ -1256,6 +1257,43 @@ def test_map_codes(self) -> None: self.assertEqual(machine.map_language_code("fr_CA"), "fr-ca") self.assertEqual(machine.map_language_code("iu_Latn"), "iu-Latn") + def test_literal_entity_replacements(self) -> None: + machine = self.get_machine() + unit = make_unit(code="cs", source="Hello <script>.") + replaced = ( + 'Hello &amp;lt;script' + '&amp;gt;.' + ) + replacements = { + '&amp;lt;': "&lt;", + '&amp;gt;': "&gt;", + } + self.assertEqual( + machine.cleanup_text(unit.source, unit), + (replaced, replacements), + ) + self.assertEqual(unit.source, machine.uncleanup_text(replacements, replaced)) + + def test_glossary_term_overlapping_literal_entity(self) -> None: + """A glossary term inside a character reference must not be highlighted.""" + machine = self.get_machine() + source = "The ©right notice and <b> tag." + unit = make_unit(code="cs", source=source) + term = SimpleNamespace(glossary_positions=[(5, 14)], target="Copyright") + + with patch( + "weblate.machinery.microsoft.get_glossary_terms", + return_value=[term], + ): + highlights = list(machine.get_highlights(source, unit)) + replaced, replacements = machine.cleanup_text(source, unit) + + self.assertEqual( + highlights, + [(4, 9, "©", None), (26, 30, "<", None), (31, 35, ">", None)], + ) + self.assertEqual(source, machine.uncleanup_text(replacements, replaced)) + @patch( "weblate.utils.outbound.socket.getaddrinfo", return_value=[(0, 0, 0, "", ("127.0.0.1", 443))], @@ -1527,6 +1565,26 @@ def test_replacements(self) -> None: unit.source, machine_translation.uncleanup_text(replacements, replaced) ) + def test_literal_entity_replacements(self) -> None: + machine_translation = self.get_machine() + unit = make_unit(code="cs", source="Hello <script>.") + replaced = ( + 'Hello &amp;lt;script' + '&amp;gt;.' + ) + replacements = { + '&amp;lt;': "&lt;", + '&amp;gt;': "&gt;", + '
': "\n", + } + self.assertEqual( + machine_translation.cleanup_text(unit.source, unit), + (replaced, replacements), + ) + self.assertEqual( + unit.source, machine_translation.uncleanup_text(replacements, replaced) + ) + # set glossary_count_limit to 1 to also trigger delete_oldest_glossary @patch("weblate.glossary.models.get_glossary_tsv", new=lambda _: "foo\tbar") @patch("weblate.machinery.googlev3.GoogleV3Translation.glossary_count_limit", new=1) @@ -2548,6 +2606,123 @@ def request_callback(request: httpx2.Request): self.assertEqual(translation[0][0]["source"], "Hello&world") self.assertEqual(translation[0][0]["text"], "Hallo&welt") + def mock_entity_normalizing_response(self) -> None: + def request_callback(request: httpx2.Request): + payload = load_request_json(request) + texts = cast("list[str]", payload["text"]) + translated = texts[0].replace("Hello", "Hallo").replace("&", "&") + return httpx2.Response( + 200, + headers={}, + text=json.dumps( + { + "translations": [ + {"detected_source_language": "EN", "text": translated} + ] + } + ), + ) + + self.mock_languages() + http_mock.register_callback( + "POST", + "https://api.deepl.com/v2/translate", + callback=request_callback, + ) + + @http_mock.activate + def test_literal_entities_preserved(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + translation = self.assert_translate( + self.SUPPORTED, + "Hello <script>.", + 1, + machine=machine, + unit_args={"flags": "safe-html"}, + ) + self.assertEqual(translation[0][0]["source"], "Hello <script>.") + self.assertEqual(translation[0][0]["text"], "Hallo <script>.") + + @http_mock.activate + def test_literal_entity_variants_preserved(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + source = "Hello < < < < © © &unknown;." + translation = self.assert_translate(self.SUPPORTED, source, 1, machine=machine) + self.assertEqual(translation[0][0]["source"], source) + self.assertEqual( + translation[0][0]["text"], + "Hallo < < < < © © &unknown;.", + ) + + def test_literal_entity_merges_with_overlapping_highlight(self) -> None: + """A highlight covering part of an entity is widened to cover all of it.""" + machine = self.MACHINE_CLS(self.get_configuration()) + source = "The © is here" + unit = cast( + "Unit", make_unit(code="de", source=source, flags='placeholders:"y; is"') + ) + + self.assertEqual( + list(machine.get_highlights(source, unit)), + [(4, 13, "© is", None)], + ) + replaced, replacements = machine.cleanup_text(source, unit) + self.assertNotIn("&cop", replaced) + self.assertEqual(source, machine.uncleanup_text(replacements, replaced)) + + @http_mock.activate + def test_literal_entities_with_xml_highlights(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + source = 'Hello <script>.' + translation = self.assert_translate( + self.SUPPORTED, + source, + 1, + machine=machine, + unit_args={"flags": "xml-text"}, + ) + self.assertEqual(translation[0][0]["source"], source) + self.assertEqual( + translation[0][0]["text"], + 'Hallo <script>.', + ) + + @http_mock.activate + def test_literal_entity_contains_existing_highlight(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + machine.delete_cache() + self.mock_entity_normalizing_response() + + source = "Hello ©." + translation = self.assert_translate( + self.SUPPORTED, + source, + 1, + machine=machine, + unit_args={"flags": 'placeholders:"copy"'}, + ) + self.assertEqual(translation[0][0]["source"], source) + self.assertEqual(translation[0][0]["text"], "Hallo ©.") + + def test_literal_entity_highlight_excludes_trailing_text(self) -> None: + machine = self.MACHINE_CLS(self.get_configuration()) + source = "Hello a¬b." + unit = cast("Unit", make_unit(code="de", source=source)) + + self.assertEqual( + list(machine.get_highlights(source, unit)), + [(7, 11, "¬", None)], + ) + @http_mock.activate @patch("weblate.glossary.models.get_glossary_tsv", new=lambda _: "foo\tbar") def test_glossary(self) -> None: diff --git a/weblate/utils/html.py b/weblate/utils/html.py index e9d6f8a04fba..deca63e41244 100644 --- a/weblate/utils/html.py +++ b/weblate/utils/html.py @@ -7,6 +7,7 @@ import re import threading from collections import defaultdict +from html.entities import html5 from html.parser import HTMLParser as StdHTMLParser from typing import TYPE_CHECKING, Any, NamedTuple @@ -72,6 +73,31 @@ ) MD_SYNTAX_GROUPS = 8 +HTML_ENTITY_CANDIDATE = re.compile( + r"&(?:#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)" +) + + +def iter_html_entities(text: str) -> Iterable[tuple[int, int, str]]: + """Yield character-reference spans consumed by html.unescape.""" + for match in HTML_ENTITY_CANDIDATE.finditer(text): + entity = match.group() + if entity[1] == "#": + yield match.start(), match.end(), entity + continue + + name = entity[1:] + if name in html5: + yield match.start(), match.end(), entity + continue + + for length in range(len(name) - 1, 1, -1): + if name[:length] in html5: + end = match.start() + length + 1 + yield match.start(), end, text[match.start() : end] + break + + AUTO_SAFE_HTML_START = re.compile(r"<(?=[!/?A-Za-z])") AUTO_SAFE_HTML_SEGMENT = re.compile( r""" diff --git a/weblate/utils/tests/test_html.py b/weblate/utils/tests/test_html.py index e58672266b4d..6ce47ac26e70 100644 --- a/weblate/utils/tests/test_html.py +++ b/weblate/utils/tests/test_html.py @@ -4,6 +4,8 @@ from __future__ import annotations +from html import unescape + from django.test import SimpleTestCase from weblate.checks.flags import Flags @@ -14,6 +16,7 @@ extract_html_attributes, extract_html_tags, is_auto_safe_html_source, + iter_html_entities, list_to_tuples, mail_quote_value, ) @@ -216,3 +219,52 @@ def test_empty_list(self) -> None: def test_single_element_list(self) -> None: self.assertEqual(list(list_to_tuples(["only_one"])), [("only_one",)]) + + +class IterHTMLEntitiesTestCase(SimpleTestCase): + def test_named(self) -> None: + self.assertEqual( + list(iter_html_entities("a < b & c")), + [(2, 6, "<"), (9, 14, "&")], + ) + + def test_numeric(self) -> None: + self.assertEqual( + list(iter_html_entities("< < <")), + [(0, 5, "<"), (6, 12, "<"), (13, 19, "<")], + ) + + def test_missing_semicolon(self) -> None: + # Legacy references are decoded without the trailing semicolon + self.assertEqual(list(iter_html_entities("© 2026")), [(0, 5, "©")]) + + def test_longest_prefix(self) -> None: + # Only the recognized prefix is a reference, the rest is plain text + self.assertEqual(list(iter_html_entities("a¬b")), [(1, 5, "¬")]) + + def test_unknown(self) -> None: + for text in ("&unknown;", "R&D", "Tom & Jerry", "&", "&;", "&#", "&#x"): + with self.subTest(text=text): + self.assertEqual(list(iter_html_entities(text)), []) + + def test_long_name(self) -> None: + # The longest name is 32 characters including the semicolon + longest = "∳" + self.assertEqual(list(iter_html_entities(longest)), [(0, 33, longest)]) + + def test_matches_unescape(self) -> None: + """Yielded spans are exactly what unescaping consumes.""" + for text in ( + "a < b", + "© 2026", + "a¬b", + "&unknown;", + "R&D", + "<<", + "&lt;", + "?a=1¬ify=2", + ): + with self.subTest(text=text): + for start, end, entity in iter_html_entities(text): + self.assertEqual(entity, text[start:end]) + self.assertNotEqual(unescape(entity), entity)