Skip to content

Commit 2cd0a8c

Browse files
authored
Merge pull request #1093 from Lyin-OTTF/fix/xml-serialize-escape-whitespace
Fix TAB/LF/CR escape when serializing to XML
2 parents bc90ee4 + fea3a2c commit 2cd0a8c

3 files changed

Lines changed: 107 additions & 21 deletions

File tree

‎src/main/java/org/json/XML.java‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -156,34 +156,45 @@ public static String escape(String string) {
156156

157157
/**
158158
* @param cp code point to test
159-
* @return true if the code point is not valid for an XML
159+
* @return true if the code point must be escaped when serializing to XML
160160
*/
161161
static boolean mustEscape(int cp) {
162+
// Every ISO control character must be written as a numeric character
163+
// reference when serializing. This includes #x9 (tab), #xA (LF) and
164+
// #xD (CR): literal occurrences of these characters are subject to
165+
// whitespace normalization by conforming XML parsers, so they can
166+
// only survive a round trip as 	, 
 and 
 (see #1092).
167+
//
168+
// This method is only about the serialization path. Numeric character
169+
// references found while parsing are validated with
170+
// isValidXmlCharacter(int) (see #1059).
171+
return Character.isISOControl(cp) || !isValidXmlCharacter(cp);
172+
}
173+
174+
/**
175+
* @param cp code point to test
176+
* @return true if the code point can be written as a numeric character
177+
* reference in a document generated by this class
178+
*/
179+
static boolean isValidXmlCharacter(int cp) {
162180
/* Valid range from https://www.w3.org/TR/REC-xml/#charsets
163181
*
164182
* #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
165183
*
166184
* any Unicode character, excluding the surrogate blocks, FFFE, and FFFF.
167185
*/
168-
// isISOControl is true when (cp >= 0 && cp <= 0x1F) || (cp >= 0x7F && cp <= 0x9F)
169-
// all ISO control characters are out of range except tabs and new lines
170-
return (Character.isISOControl(cp)
171-
&& cp != 0x9
172-
&& cp != 0xA
173-
&& cp != 0xD
174-
) || !(
175-
// Valid character range per W3C XML 1.0 spec:
176-
// #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
177-
// Previously omitted #x9/#xA/#xD, causing unescape("&#10;") etc.
178-
// to reject valid LF/TAB/CR as illegal (see #1059)
179-
cp == 0x9
180-
|| cp == 0xA
181-
|| cp == 0xD
182-
|| (cp >= 0x20 && cp <= 0xD7FF)
183-
|| (cp >= 0xE000 && cp <= 0xFFFD)
184-
|| (cp >= 0x10000 && cp <= 0x10FFFF)
185-
)
186-
;
186+
// The C1 control characters (#x7F-#x9F) fall inside [#x20-#xD7FF] and
187+
// are therefore legal XML 1.0 characters, but this class has always
188+
// rejected &#NN; references to them while parsing. Keeping them
189+
// rejected here keeps the parse result of this fix identical to the
190+
// previous release; accepting them would be a separate behaviour
191+
// change (see #1093).
192+
if (Character.isISOControl(cp)) {
193+
return cp == 0x9 || cp == 0xA || cp == 0xD;
194+
}
195+
return (cp >= 0x20 && cp <= 0xD7FF)
196+
|| (cp >= 0xE000 && cp <= 0xFFFD)
197+
|| (cp >= 0x10000 && cp <= 0x10FFFF);
187198
}
188199

189200
/**

‎src/main/java/org/json/XMLTokener.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,11 @@ static String unescapeEntity(String e) throws JSONException {
167167
int cp = (e.charAt(1) == 'x' || e.charAt(1) == 'X')
168168
? parseHexEntity(e)
169169
: parseDecimalEntity(e);
170-
if (XML.mustEscape(cp)) {
170+
// Validate with isValidXmlCharacter, not with XML.mustEscape:
171+
// mustEscape also escapes TAB/LF/CR for the serialization path
172+
// (#1092), while the numeric references &#9;, &#10; and &#13; are
173+
// valid XML and must keep being accepted here (see #1059).
174+
if (!XML.isValidXmlCharacter(cp)) {
171175
throw new JSONException("Invalid numeric character reference: &#" + e.substring(1) + ";");
172176
}
173177
return new String(new int[] {cp}, 0, 1);

‎src/test/java/org/json/junit/XMLTest.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import static org.junit.Assert.assertEquals;
88
import static org.junit.Assert.assertFalse;
99
import static org.junit.Assert.assertNotEquals;
10+
import static org.junit.Assert.assertThrows;
1011
import static org.junit.Assert.assertTrue;
1112
import static org.junit.Assert.fail;
1213

@@ -1610,6 +1611,76 @@ public void testValidWhitespaceNumericEntityToJSONObject() {
16101611
XML.toJSONObject("<a>&#13;</a>");
16111612
}
16121613

1614+
/**
1615+
* Tests that numeric character references to C1 control characters are
1616+
* still rejected. #x7F-#x9F is inside the XML 1.0 {@code Char} range, but
1617+
* this library has always refused those references while parsing
1618+
* ({@code &#133;} is NEL, #x85), and fixing the serialization of TAB/LF/CR
1619+
* must not change what the parser accepts.
1620+
*/
1621+
@Test
1622+
public void testRejectsC1ControlCharacterReference() {
1623+
assertThrows(JSONException.class,
1624+
() -> XML.toJSONObject("<name>Joe &#133; Tester</name>"));
1625+
// the neighbouring boundary codepoints behave the same way
1626+
assertThrows(JSONException.class,
1627+
() -> XML.toJSONObject("<name>Joe &#127; Tester</name>"));
1628+
assertThrows(JSONException.class,
1629+
() -> XML.toJSONObject("<name>Joe &#159; Tester</name>"));
1630+
// ... while the characters the parser does accept still work
1631+
XML.toJSONObject("<name>Joe &#32; Tester</name>");
1632+
}
1633+
1634+
/**
1635+
* Tests that serializing a string value containing literal TAB, LF and CR
1636+
* characters writes numeric character references (&amp;#9;, &amp;#10;,
1637+
* &amp;#13;) instead of the literal characters. Literal occurrences of
1638+
* these characters are subject to whitespace normalization by conforming
1639+
* XML parsers, so the original value would be lost when the generated XML
1640+
* is read back. Regression test for #1092, where #1070 stopped
1641+
* XML.toString() and JSONML.toString() from escaping these characters.
1642+
*/
1643+
@Test
1644+
public void testEscapesLiteralWhitespaceControlCharsOnSerialization() {
1645+
// LF in a text node must survive as a numeric character reference
1646+
JSONObject jo = new JSONObject().put("value", "Lettuce\nTomato");
1647+
String xmlOutput = XML.toString(jo);
1648+
assertTrue("XML should escape literal LF to preserve it, got: " + xmlOutput,
1649+
xmlOutput.contains("&#xa;"));
1650+
1651+
// TAB and CR likewise
1652+
JSONObject tabJo = new JSONObject().put("value", "a\tb\rc");
1653+
String tabXml = XML.toString(tabJo);
1654+
assertTrue("XML should escape literal TAB, got: " + tabXml, tabXml.contains("&#x9;"));
1655+
assertTrue("XML should escape literal CR, got: " + tabXml, tabXml.contains("&#xd;"));
1656+
}
1657+
1658+
/**
1659+
* Tests that the generated XML round-trips: the numeric character
1660+
* references produced for TAB/LF/CR are read back by
1661+
* {@code XML.toJSONObject} as the original characters.
1662+
*/
1663+
@Test
1664+
public void testWhitespaceControlCharsRoundTripThroughXml() {
1665+
String original = "Lettuce\nTomato\tMustard";
1666+
JSONObject jo = new JSONObject().put("value", original);
1667+
String xmlOutput = XML.toString(jo);
1668+
JSONObject reparsed = XML.toJSONObject(xmlOutput);
1669+
assertEquals("LF/TAB must survive the XML round trip",
1670+
original, reparsed.getString("value"));
1671+
}
1672+
1673+
/**
1674+
* Same escaping requirement for {@code JSONML.toString}.
1675+
*/
1676+
@Test
1677+
public void testJSONMLToStringEscapesLiteralNewlines() {
1678+
JSONObject jo = new JSONObject("{\"tagName\":\"MyTag\", \"value\":\"Lettuce\\nTomato\"}");
1679+
String xmlOutput = JSONML.toString(jo);
1680+
assertTrue("JSONML should escape literal LF to preserve it, got: " + xmlOutput,
1681+
xmlOutput.contains("&#xa;"));
1682+
}
1683+
16131684
}
16141685

16151686

0 commit comments

Comments
 (0)