pythongh-81055: Support CDATA sections in xml.etree.ElementTree

CDATA is a new factory, like Comment and ProcessingInstruction, which creates
a special element serialized as a CDATA section.  Its content is character
data: it is not escaped, "]]>" in it is split between two sections, and it is
returned by itertext() and by the "text" serialization method.

TreeBuilder gets the cdata_factory and insert_cdata arguments.  When
insert_cdata is set, a CDATA section in the input is kept as such instead of
being parsed as text, so that the document can be written back unchanged.
Expat reports the content of a CDATA section as ordinary character data, but
it reports the boundaries, so the builder gets the start_cdata() and
end_cdata() methods, and XMLParser calls them like comment() and pi().

_set_factories() takes the third factory, which the C implementation needs for
itertext(), and the pyexpat capsule gets SetCdataSectionHandler.

Author: serhiy-storchaka

Doc/library/xml.etree.elementtree.rst

@@ -548,6 +548,28 @@ Functions
    .. versionadded:: 3.8
 
 
+.. function:: CDATA(text=None)
+
+   CDATA section factory.
+   This factory function creates a special element
+   which the standard serializer serializes as a CDATA section.
+   *text* is a string containing the content of the CDATA section.
+
+   The content of a CDATA section is character data:
+   it is not escaped when serialized, and it is returned
+   by :meth:`Element.itertext` and by the ``"text"`` serialization method.
+   ``"]]>"`` cannot occur in a CDATA section,
+   so the content which contains it is split into several sections.
+
+   Note that a CDATA section in the input is parsed as text by default:
+   its content is added to the tree as ordinary character data.
+   A tree only contains CDATA sections if they have been inserted into it
+   using one of the :class:`Element` methods,
+   or if the parser target collects them; see :class:`TreeBuilder`.
+
+   .. versionadded:: next
+
+
 .. function:: Comment(text=None)
 
    Comment element factory.  This factory function creates a special element
@@ -1051,9 +1073,14 @@ Element Objects
 
       Creates a text iterator.  The iterator loops over this element and all
       subelements, in document order, and returns all inner text.
+      The content of CDATA sections is a part of the inner text,
+      but the content of comments and processing instructions is not.
 
       .. versionadded:: 3.2
 
+      .. versionchanged:: next
+         The content of CDATA sections is returned.
+
 
    .. method:: makeelement(tag, attrib)
 
@@ -1270,7 +1297,9 @@ TreeBuilder Objects
 
 
 .. class:: TreeBuilder(element_factory=None, *, comment_factory=None, \
-                       pi_factory=None, insert_comments=False, insert_pis=False)
+                       pi_factory=None, cdata_factory=None, \
+                       insert_comments=False, insert_pis=False, \
+                       insert_cdata=False)
 
    Generic element structure builder.  This builder converts a sequence of
    start, data, end, comment and pi method calls to a well-formed element
@@ -1288,6 +1317,16 @@ TreeBuilder Objects
    comments/pis will be inserted into the tree if they appear within the root
    element (but not outside of it).
 
+   The *cdata_factory* function, when given,
+   should behave like the :func:`CDATA` function.
+   If *insert_cdata* is true, a CDATA section in the input is created
+   with this factory and inserted into the tree;
+   otherwise its content is added to the tree as ordinary character data
+   and the factory is not called.
+
+   .. versionchanged:: next
+      Added the *cdata_factory* and *insert_cdata* arguments.
+
    .. method:: close()
 
       Flushes the builder buffers, and returns the toplevel document
@@ -1328,6 +1367,26 @@ TreeBuilder Objects
       .. versionadded:: 3.8
 
 
+   .. method:: start_cdata()
+
+      Begins a CDATA section.
+      The text added by :meth:`data` until the matching :meth:`end_cdata`
+      call is the content of the section.
+
+      .. versionadded:: next
+
+
+   .. method:: end_cdata()
+
+      Ends a CDATA section.
+      If ``insert_cdata`` is true, creates a CDATA section
+      with the collected content and adds it to the tree,
+      and returns it.  Otherwise returns ``None``
+      and the collected content is left as ordinary character data.
+
+      .. versionadded:: next
+
+
    In addition, a custom :class:`TreeBuilder` object can provide the
    following methods:
 

Doc/whatsnew/3.16.rst

@@ -648,6 +648,22 @@ xml
   rather than defaulted from the DTD.
   (Contributed by Jason Orendorff and Serhiy Storchaka in :gh:`44871`.)
 
+* Add :func:`~xml.etree.ElementTree.CDATA` to :mod:`xml.etree.ElementTree`,
+  a factory of special elements which are serialized as CDATA sections,
+  like :func:`~xml.etree.ElementTree.Comment` and
+  :func:`~xml.etree.ElementTree.ProcessingInstruction`.
+  The content of a CDATA section is character data:
+  it is not escaped when serialized, and it is returned by
+  :meth:`~xml.etree.ElementTree.Element.itertext`
+  and by the ``"text"`` serialization method.
+  :class:`~xml.etree.ElementTree.TreeBuilder` supports the *cdata_factory*
+  and *insert_cdata* arguments, which make the parser keep CDATA sections
+  instead of parsing them as text, and the new :meth:`!start_cdata` and
+  :meth:`!end_cdata` methods, which :class:`~xml.etree.ElementTree.XMLParser`
+  calls for the boundaries of a CDATA section, like :meth:`!comment` and
+  :meth:`!pi`.
+  (Contributed by Serhiy Storchaka in :gh:`81055`.)
+
 zipfile
 -------
 

Include/pyexpat.h

@@ -65,6 +65,9 @@ struct PyExpat_CAPI
     /* might be NULL for expat < 2.8.0 */
     XML_Bool (*SetHashSalt16Bytes)(
         XML_Parser parser, const uint8_t entropy[16]);
+    void (*SetCdataSectionHandler)(
+        XML_Parser parser, XML_StartCdataSectionHandler start,
+        XML_EndCdataSectionHandler end);
     /* always add new stuff to the end! */
 };
 

Lib/test/test_xml_etree.py

@@ -1310,6 +1310,35 @@ def test_comment_serialization(self):
         # no comments in text serialization
         self.assertEqual(ET.tostring(comm, method='text'), b'')
 
+    def test_cdata_serialization(self):
+        cdata = ET.CDATA('<spam> & ham')
+        # the content of a CDATA section is not escaped
+        self.assertEqual(ET.tostring(cdata), b'<![CDATA[<spam> & ham]]>')
+        self.assertEqual(ET.tostring(cdata, method='html'),
+                         b'<![CDATA[<spam> & ham]]>')
+        # but it is character data
+        self.assertEqual(ET.tostring(cdata, method='text'), b'<spam> & ham')
+        # an empty CDATA section
+        self.assertEqual(ET.tostring(ET.CDATA()), b'<![CDATA[]]>')
+        self.assertEqual(ET.tostring(ET.CDATA('')), b'<![CDATA[]]>')
+        # "]]>" cannot occur in a CDATA section, it is split in two
+        self.assertEqual(ET.tostring(ET.CDATA('a]]>b')),
+                         b'<![CDATA[a]]]]><![CDATA[>b]]>')
+        self.assertRaises(TypeError, ET.tostring, ET.CDATA(42))
+
+    def test_cdata_in_element(self):
+        elem = ET.XML('<root>before</root>')
+        cdata = ET.CDATA('<spam> & ham')
+        cdata.tail = 'after'
+        elem.append(cdata)
+        self.assertEqual(ET.tostring(elem),
+                         b'<root>before<![CDATA[<spam> & ham]]>after</root>')
+        self.assertEqual(ET.tostring(elem, method='text'),
+                         b'before<spam> & hamafter')
+        # the written form is parsed back to the same text
+        self.assertEqual(''.join(ET.fromstring(ET.tostring(elem)).itertext()),
+                         'before<spam> & hamafter')
+
     def test_processinginstruction_serialization(self):
         # Test ProcessingInstruction directly
 
@@ -3771,6 +3800,21 @@ def test_processinginstruction(self):
         self.assertEqual(''.join(pi.itertext()), '')
         self.assertEqual(list(pi.iter()), [pi])
 
+    def test_cdata(self):
+        e = ET.Element('root')
+        e.text = 'before'
+        cdata = ET.CDATA('content')
+        self.assertEqual(cdata.text, 'content')
+        cdata.tail = 'after'
+        e.append(cdata)
+        # unlike a comment or a processing instruction,
+        # a CDATA section contains character data
+        self.assertEqual(''.join(e.itertext()), 'beforecontentafter')
+        self.assertEqual(list(e.iter()), [e, cdata])
+        self.assertEqual(list(e.iter('root')), [e])
+        self.assertEqual(''.join(cdata.itertext()), 'content')
+        self.assertEqual(list(cdata.iter()), [cdata])
+
     def test_corners(self):
         # single root, no subelements
         a = ET.Element('a')
@@ -3904,6 +3948,101 @@ def test_treebuilder_pi(self):
         self.assertEqual(b.pi('target'), (len('target'), None))
         self.assertEqual(b.pi('pitarget', ' text '), (len('pitarget'), ' text '))
 
+    def test_treebuilder_cdata(self):
+        b = ET.TreeBuilder()
+        # nothing is created unless insert_cdata is true
+        self.assertIsNone(b.start_cdata())
+        self.assertIsNone(b.end_cdata())
+
+        b = ET.TreeBuilder(insert_cdata=True)
+        b.start('a', {})
+        b.data('before')
+        b.start_cdata()
+        b.data('a < b')
+        cdata = b.end_cdata()
+        self.assertEqual(cdata.tag, ET.CDATA)
+        self.assertEqual(cdata.text, 'a < b')
+        b.data('after')
+        b.end('a')
+        a = b.close()
+        self.assertEqual(ET.tostring(a),
+                         b'<a>before<![CDATA[a < b]]>after</a>')
+
+    def test_treebuilder_cdata_factory(self):
+        # the factory is only called if insert_cdata is true
+        b = ET.TreeBuilder(cdata_factory=len)
+        b.start_cdata()
+        self.assertIsNone(b.end_cdata())
+
+        b = ET.TreeBuilder(insert_cdata=True,
+                           cdata_factory=lambda text: ET.Comment('was: ' + text))
+        b.start('a', {})
+        b.start_cdata()
+        b.data('abc')
+        self.assertEqual(b.end_cdata().tag, ET.Comment)
+        b.end('a')
+        self.assertEqual(ET.tostring(b.close()), b'<a><!--was: abc--></a>')
+
+    def test_parse_cdata(self):
+        xml = '<a>before<![CDATA[a < b]]>after<b><![CDATA[deep]]></b></a>'
+        # by default the content of a CDATA section is ordinary text
+        a = ET.fromstring(xml)
+        self.assertEqual(ET.tostring(a),
+                         b'<a>beforea &lt; bafter<b>deep</b></a>')
+
+        parser = ET.XMLParser(target=ET.TreeBuilder(insert_cdata=True))
+        parser.feed(xml)
+        a = parser.close()
+        self.assertEqual(summarize_list(a), [ET.CDATA, 'b'])
+        self.assertEqual(a.text, 'before')
+        self.assertEqual(a[0].text, 'a < b')
+        self.assertEqual(a[0].tail, 'after')
+        # the tree is serialized back to the source
+        self.assertEqual(ET.tostring(a, encoding='unicode'), xml)
+
+    def test_parse_empty_cdata(self):
+        parser = ET.XMLParser(target=ET.TreeBuilder(insert_cdata=True))
+        parser.feed('<a><![CDATA[]]></a>')
+        a = parser.close()
+        self.assertEqual(summarize_list(a), [ET.CDATA])
+        self.assertEqual(a[0].text, '')
+
+    def test_parse_cdata_subclass(self):
+        class TreeBuilderSubclass(ET.TreeBuilder):
+            pass
+
+        xml = '<a>text<![CDATA[a < b]]>tail</a>'
+        parser = ET.XMLParser(target=TreeBuilderSubclass(insert_cdata=True))
+        parser.feed(xml)
+        a = parser.close()
+        self.assertEqual(a.text, 'text')
+        self.assertEqual(a[0].text, 'a < b')
+        self.assertEqual(a[0].tail, 'tail')
+
+    def test_parse_cdata_custom_target(self):
+        events = []
+        class Target:
+            def start(self, tag, attrib):
+                events.append(('start', tag))
+            def end(self, tag):
+                events.append(('end', tag))
+            def data(self, data):
+                events.append(('data', data))
+            def start_cdata(self):
+                events.append(('start_cdata',))
+            def end_cdata(self):
+                events.append(('end_cdata',))
+            def close(self):
+                return events
+
+        parser = ET.XMLParser(target=Target())
+        parser.feed('<a>text<![CDATA[a < b]]>tail</a>')
+        self.assertEqual(parser.close(), [
+            ('start', 'a'), ('data', 'text'),
+            ('start_cdata',), ('data', 'a < b'), ('end_cdata',),
+            ('data', 'tail'), ('end', 'a'),
+        ])
+
     def test_late_tail(self):
         # Issue #37399: The tail of an ignored comment could overwrite the text before it.
         class TreeBuilderSubclass(ET.TreeBuilder):
@@ -4981,9 +5120,9 @@ def cleanup():
     unittest.addModuleCleanup(setattr, ElementPath, "_cache", path_cache)
     ElementPath._cache = path_cache.copy()
 
-    # Align the Comment/PI factories.
+    # Align the Comment/PI/CDATA factories.
     if hasattr(ET, '_set_factories'):
-        old_factories = ET._set_factories(ET.Comment, ET.PI)
+        old_factories = ET._set_factories(ET.Comment, ET.PI, ET.CDATA)
         unittest.addModuleCleanup(ET._set_factories, *old_factories)