fix(machinery): preserve literal HTML entities

docs/changes.rst

@@ -49,6 +49,7 @@ Weblate 2026.8
 * REST API authorization now consistently protects internal accounts, restricted components, add-on configuration, component sharing, repository links, and review states.
 * :ref:`Project backup imports <projectbackup>` now validate restored data before creating project state, skip repository-linked components the importer cannot access, and no longer load archive-supplied Mercurial configuration or shared-repository indirection.
 * Rebuilding project translation memory no longer removes entries imported from files.
+* Literal HTML character references in source strings are now preserved by :ref:`mt-deepl`, :ref:`mt-google-translate-api-v3` and :ref:`mt-microsoft-translator` machine translation instead of being decoded into live markup.
 * Suggestion submission and rejection now reject excessively long suggestion text and rejection reasons.
 * Restricted components are now available on Hosted Weblate when the billing plan permits private projects.
 * Machine translation and translation memory AJAX lookups no longer disclose whether inaccessible unit IDs exist.

weblate/machinery/base.py

@@ -33,6 +33,7 @@
 from weblate.utils.errors import log_handled_exception, report_error
 from weblate.utils.forms import WeblateServiceURLField
 from weblate.utils.hash import calculate_dict_hash, calculate_hash, hash_to_checksum
+from weblate.utils.html import iter_html_entities
 from weblate.utils.outbound import is_allowlisted_hostname
 from weblate.utils.requests import (
     JSON_RESPONSE_ERRORS,
@@ -1548,6 +1549,42 @@ class XMLMachineTranslationMixin(BatchMachineTranslation):
     highlight_syntax = True
     force_uncleanup = True
 
+    def get_highlights(
+        self, text: str, unit: Unit
+    ) -> Iterable[tuple[int, int, str, Highlight | Unit | None]]:
+        highlights = list(super().get_highlights(text, unit))
+
+        for entity_start, entity_end, entity_text in iter_html_entities(text):
+            entity = (entity_start, entity_end, entity_text, None)
+            overlapping = [
+                index
+                for index, (start, end, _highlight_text, _kind) in enumerate(highlights)
+                if entity_start < end and entity_end > start
+            ]
+            if overlapping:
+                first = overlapping[0]
+                last = overlapping[-1]
+                if any(
+                    start <= entity_start and end >= entity_end
+                    for start, end, _highlight_text, _kind in highlights[
+                        first : last + 1
+                    ]
+                ):
+                    continue
+                start = min(entity_start, highlights[first][0])
+                end = max(entity_end, highlights[last][1])
+                highlights[first : last + 1] = [(start, end, text[start:end], None)]
+                continue
+
+            for index, (start, _end, _highlight_text, _kind) in enumerate(highlights):
+                if entity_end <= start:
+                    highlights.insert(index, entity)
+                    break
+            else:
+                highlights.append(entity)
+
+        yield from highlights
+
     def unescape_text(self, text: str) -> str:
         """Unescaping of the text with replacements."""
         return unescape(text)

weblate/machinery/microsoft.py

@@ -4,7 +4,9 @@
 
 from __future__ import annotations
 
+from bisect import insort
 from datetime import timedelta
+from operator import itemgetter
 from typing import TYPE_CHECKING, ClassVar
 
 from django.utils import timezone
@@ -191,25 +193,19 @@
         if "read-only" in flags:
             # Use terminology format
             return self.format_replacement(h_start, h_end, h_text, None)
         return f'<mstrans:dictionary translation="{self.escape_text(h_kind.target)}">{self.escape_text(h_text)}</mstrans:dictionary>'
 
     def get_highlights(self, text, unit):
         result = list(super().get_highlights(text, unit))
 
         for term in get_glossary_terms(unit, include_variants=False):
             for start, end in term.glossary_positions:
-                glossary_highlight = (start, end, text[start:end], term)
-                handled = False
-                for i, (h_start, _h_end, _h_text, _h_kind) in enumerate(result):
-                    if start < h_start:
-                        if end > h_start:
-                            # Skip as overlaps
-                            break
-                        # Insert before
-                        result.insert(i, glossary_highlight)
-                        handled = True
-                        break
-                if not handled and (not result or result[-1][1] < start):
-                    result.append(glossary_highlight)
+                if any(
+                    start < h_end and end > h_start
+                    for h_start, h_end, _h_text, _h_kind in result
+                ):
+                    # Skip as overlaps
+                    continue
+                insort(result, (start, end, text[start:end], term), key=itemgetter(0))
 
         yield from result

weblate/machinery/tests.py

@@ -12,6 +12,7 @@
 from functools import partial
 from io import StringIO
 from pathlib import Path
+from types import SimpleNamespace
 from typing import TYPE_CHECKING, ClassVar, NoReturn, cast
 from unittest.mock import AsyncMock, MagicMock, Mock, call, patch
 from urllib.parse import parse_qs, urlparse
@@ -1256,6 +1257,43 @@
         self.assertEqual(machine.map_language_code("fr_CA"), "fr-ca")
         self.assertEqual(machine.map_language_code("iu_Latn"), "iu-Latn")
 
+    def test_literal_entity_replacements(self) -> None:
+        machine = self.get_machine()
+        unit = make_unit(code="cs", source="Hello <script>.")
+        replaced = (
+            'Hello <span class="notranslate" id="6">&amp;amp;lt;</span>script'
+            '<span class="notranslate" id="16">&amp;amp;gt;</span>.'
+        )
+        replacements = {
+            '<span class="notranslate" id="6">&amp;amp;lt;</span>': "&amp;lt;",
+            '<span class="notranslate" id="16">&amp;amp;gt;</span>': "&amp;gt;",
+        }
+        self.assertEqual(
+            machine.cleanup_text(unit.source, unit),
+            (replaced, replacements),
+        )
+        self.assertEqual(unit.source, machine.uncleanup_text(replacements, replaced))
+
+    def test_glossary_term_overlapping_literal_entity(self) -> None:
+        """A glossary term inside a character reference must not be highlighted."""
+        machine = self.get_machine()
+        source = "The &copyright notice and &lt;b&gt; tag."
+        unit = make_unit(code="cs", source=source)
+        term = SimpleNamespace(glossary_positions=[(5, 14)], target="Copyright")
+
+        with patch(
+            "weblate.machinery.microsoft.get_glossary_terms",
+            return_value=[term],
+        ):
+            highlights = list(machine.get_highlights(source, unit))
+            replaced, replacements = machine.cleanup_text(source, unit)
+
+        self.assertEqual(
+            highlights,
+            [(4, 9, "&copy", None), (26, 30, "&lt;", None), (31, 35, "&gt;", None)],
+        )
+        self.assertEqual(source, machine.uncleanup_text(replacements, replaced))
+
     @patch(
         "weblate.utils.outbound.socket.getaddrinfo",
         return_value=[(0, 0, 0, "", ("127.0.0.1", 443))],
@@ -1527,6 +1565,26 @@
             unit.source, machine_translation.uncleanup_text(replacements, replaced)
         )
 
+    def test_literal_entity_replacements(self) -> None:
+        machine_translation = self.get_machine()
+        unit = make_unit(code="cs", source="Hello &lt;script&gt;.")
+        replaced = (
+            'Hello <span translate="no" id="6">&amp;amp;lt;</span>script'
+            '<span translate="no" id="16">&amp;amp;gt;</span>.'
+        )
+        replacements = {
+            '<span translate="no" id="6">&amp;amp;lt;</span>': "&amp;lt;",
+            '<span translate="no" id="16">&amp;amp;gt;</span>': "&amp;gt;",
+            '<br translate="no">': "\n",
+        }
+        self.assertEqual(
+            machine_translation.cleanup_text(unit.source, unit),
+            (replaced, replacements),
+        )
+        self.assertEqual(
+            unit.source, machine_translation.uncleanup_text(replacements, replaced)
+        )
+
     # set glossary_count_limit to 1 to also trigger delete_oldest_glossary
     @patch("weblate.glossary.models.get_glossary_tsv", new=lambda _: "foo\tbar")
     @patch("weblate.machinery.googlev3.GoogleV3Translation.glossary_count_limit", new=1)
@@ -2548,6 +2606,123 @@
         self.assertEqual(translation[0][0]["source"], "Hello&world")
         self.assertEqual(translation[0][0]["text"], "Hallo&welt")
 
+    def mock_entity_normalizing_response(self) -> None:
+        def request_callback(request: httpx2.Request):
+            payload = load_request_json(request)
+            texts = cast("list[str]", payload["text"])
+            translated = texts[0].replace("Hello", "Hallo").replace("&amp;", "&")
+            return httpx2.Response(
+                200,
+                headers={},
+                text=json.dumps(
+                    {
+                        "translations": [
+                            {"detected_source_language": "EN", "text": translated}
+                        ]
+                    }
+                ),
+            )
+
+        self.mock_languages()
+        http_mock.register_callback(
+            "POST",
+            "https://api.deepl.com/v2/translate",
+            callback=request_callback,
+        )
+
+    @http_mock.activate
+    def test_literal_entities_preserved(self) -> None:
+        machine = self.MACHINE_CLS(self.get_configuration())
+        machine.delete_cache()
+        self.mock_entity_normalizing_response()
+
+        translation = self.assert_translate(
+            self.SUPPORTED,
+            "Hello &lt;script&gt;.",
+            1,
+            machine=machine,
+            unit_args={"flags": "safe-html"},
+        )
+        self.assertEqual(translation[0][0]["source"], "Hello &lt;script&gt;.")
+        self.assertEqual(translation[0][0]["text"], "Hallo &lt;script&gt;.")
+
+    @http_mock.activate
+    def test_literal_entity_variants_preserved(self) -> None:
+        machine = self.MACHINE_CLS(self.get_configuration())
+        machine.delete_cache()
+        self.mock_entity_normalizing_response()
+
+        source = "Hello &lt; &#60; &#x3C; &#X3C; &copy; &copy &unknown;."
+        translation = self.assert_translate(self.SUPPORTED, source, 1, machine=machine)
+        self.assertEqual(translation[0][0]["source"], source)
+        self.assertEqual(
+            translation[0][0]["text"],
+            "Hallo &lt; &#60; &#x3C; &#X3C; &copy; &copy &unknown;.",
+        )
+
+    def test_literal_entity_merges_with_overlapping_highlight(self) -> None:
+        """A highlight covering part of an entity is widened to cover all of it."""
+        machine = self.MACHINE_CLS(self.get_configuration())
+        source = "The &copy; is here"
+        unit = cast(
+            "Unit", make_unit(code="de", source=source, flags='placeholders:"y; is"')
+        )
+
+        self.assertEqual(
+            list(machine.get_highlights(source, unit)),
+            [(4, 13, "&copy; is", None)],
+        )
+        replaced, replacements = machine.cleanup_text(source, unit)
+        self.assertNotIn("&amp;cop", replaced)
+        self.assertEqual(source, machine.uncleanup_text(replacements, replaced))
+
+    @http_mock.activate
+    def test_literal_entities_with_xml_highlights(self) -> None:
+        machine = self.MACHINE_CLS(self.get_configuration())
+        machine.delete_cache()
+        self.mock_entity_normalizing_response()
+
+        source = '<b title="&copy;">Hello &lt;script&gt;.</b>'
+        translation = self.assert_translate(
+            self.SUPPORTED,
+            source,
+            1,
+            machine=machine,
+            unit_args={"flags": "xml-text"},
+        )
+        self.assertEqual(translation[0][0]["source"], source)
+        self.assertEqual(
+            translation[0][0]["text"],
+            '<b title="&copy;">Hallo &lt;script&gt;.</b>',
+        )
+
+    @http_mock.activate
+    def test_literal_entity_contains_existing_highlight(self) -> None:
+        machine = self.MACHINE_CLS(self.get_configuration())
+        machine.delete_cache()
+        self.mock_entity_normalizing_response()
+
+        source = "Hello &copy;."
+        translation = self.assert_translate(
+            self.SUPPORTED,
+            source,
+            1,
+            machine=machine,
+            unit_args={"flags": 'placeholders:"copy"'},
+        )
+        self.assertEqual(translation[0][0]["source"], source)
+        self.assertEqual(translation[0][0]["text"], "Hallo &copy;.")
+
+    def test_literal_entity_highlight_excludes_trailing_text(self) -> None:
+        machine = self.MACHINE_CLS(self.get_configuration())
+        source = "Hello a&notb."
+        unit = cast("Unit", make_unit(code="de", source=source))
+
+        self.assertEqual(
+            list(machine.get_highlights(source, unit)),
+            [(7, 11, "&not", None)],
+        )
+
     @http_mock.activate
     @patch("weblate.glossary.models.get_glossary_tsv", new=lambda _: "foo\tbar")
     def test_glossary(self) -> None:
@@ -2945,8 +3120,8 @@
             with self.subTest(test_case["name"]):
                 machine = self.MACHINE_CLS(
                     {
                         "key": test_case["key"],
                         "url": test_case["url"],
                     }
                 )
                 self.assertEqual(machine.api_base_url, test_case["base_url"])
@@ -3615,7 +3790,7 @@
         )
 
     def test_language_instructions_empty_initial_renders_blank(self) -> None:
         for initial in (
             {},
             {"language_instructions": None},
             {"language_instructions": {}},
@@ -6236,15 +6411,15 @@
             try:
                 highlights = highlight_string(unit.source, unit)
             except Exception as error:  # pragma: no cover - diagnostic path
                 highlights = f"{error.__class__.__name__}: {error}"
             try:
                 machine_highlights = list(machine.get_highlights(unit.source, unit))
             except Exception as error:  # pragma: no cover - diagnostic path
                 machine_highlights = f"{error.__class__.__name__}: {error}"
             try:
                 placeholder_values = unit.all_flags.get_value_raw("placeholders")
             except Exception as error:  # pragma: no cover - diagnostic path
                 placeholder_values = f"{error.__class__.__name__}: {error}"
 
             return json.dumps(
                 {
@@ -8589,7 +8764,7 @@
 
         for service in third_party_services:
             self.assertTrue(service.sends_data_to_third_party, service.name)
         for service in local_services:
             self.assertFalse(service.sends_data_to_third_party, service.name)
 
     @override_settings(OFFER_HOSTING=False)

weblate/utils/html.py

@@ -7,6 +7,7 @@
 import re
 import threading
 from collections import defaultdict
+from html.entities import html5
 from html.parser import HTMLParser as StdHTMLParser
 from typing import TYPE_CHECKING, Any, NamedTuple
 
@@ -72,6 +73,31 @@
 )
 MD_SYNTAX_GROUPS = 8
 
+HTML_ENTITY_CANDIDATE = re.compile(
+    r"&(?:#[0-9]+;?|#[xX][0-9a-fA-F]+;?|[^\t\n\f <&#;]{1,32};?)"
+)
+
+
+def iter_html_entities(text: str) -> Iterable[tuple[int, int, str]]:
+    """Yield character-reference spans consumed by html.unescape."""
+    for match in HTML_ENTITY_CANDIDATE.finditer(text):
+        entity = match.group()
+        if entity[1] == "#":
+            yield match.start(), match.end(), entity
+            continue
+
+        name = entity[1:]
+        if name in html5:
+            yield match.start(), match.end(), entity
+            continue
+
+        for length in range(len(name) - 1, 1, -1):
+            if name[:length] in html5:
+                end = match.start() + length + 1
+                yield match.start(), end, text[match.start() : end]
+                break
+
+
 AUTO_SAFE_HTML_START = re.compile(r"<(?=[!/?A-Za-z])")
 AUTO_SAFE_HTML_SEGMENT = re.compile(
     r"""