Skip to content

Commit 9c65437

Browse files
authored
fix: the TOML reader accepts every escape TOML 1.0 defines (#867)
Closes #865.
1 parent 9459220 commit 9c65437

2 files changed

Lines changed: 112 additions & 26 deletions

File tree

‎modules/libs/src/toml.cppm‎

Lines changed: 71 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,75 @@ inline bool is_bare_key_char(char c) {
186186
|| (c >= '0' && c <= '9') || c == '_' || c == '-';
187187
}
188188

189+
// A Unicode scalar value as UTF-8.
190+
inline void append_utf8(std::string& out, std::uint32_t cp) {
191+
auto byte = [&](std::uint32_t b) { out.push_back(static_cast<char>(b)); };
192+
if (cp < 0x80) {
193+
byte(cp);
194+
} else if (cp < 0x800) {
195+
byte(0xC0 | (cp >> 6));
196+
byte(0x80 | (cp & 0x3F));
197+
} else if (cp < 0x10000) {
198+
byte(0xE0 | (cp >> 12));
199+
byte(0x80 | ((cp >> 6) & 0x3F));
200+
byte(0x80 | (cp & 0x3F));
201+
} else {
202+
byte(0xF0 | (cp >> 18));
203+
byte(0x80 | ((cp >> 12) & 0x3F));
204+
byte(0x80 | ((cp >> 6) & 0x3F));
205+
byte(0x80 | (cp & 0x3F));
206+
}
207+
}
208+
209+
// The escape after a backslash in a basic string, appended to `out`; the
210+
// lexer stands on the character after the backslash and is left after the
211+
// escape. TOML 1.0's set, plus `\0`, which this reader has always accepted.
212+
// `\uXXXX` and `\UXXXXXXXX` name a Unicode scalar value and are written out
213+
// as UTF-8; a surrogate or a value above U+10FFFF is not one, and is refused.
214+
inline std::expected<void, ParseError> read_escape(Lexer& L, std::string& out) {
215+
const Position at = L.position();
216+
const char esc = L.peek();
217+
switch (esc) {
218+
case 'b': out.push_back('\b'); break;
219+
case 't': out.push_back('\t'); break;
220+
case 'n': out.push_back('\n'); break;
221+
case 'f': out.push_back('\f'); break;
222+
case 'r': out.push_back('\r'); break;
223+
case '"': out.push_back('"'); break;
224+
case '\\': out.push_back('\\'); break;
225+
case '0': out.push_back('\0'); break;
226+
case 'u':
227+
case 'U': {
228+
const std::size_t digits = esc == 'u' ? 4 : 8;
229+
std::uint32_t cp = 0;
230+
for (std::size_t i = 1; i <= digits; ++i) {
231+
const char h = L.peek(i);
232+
std::uint32_t d = 0;
233+
if (h >= '0' && h <= '9') d = static_cast<std::uint32_t>(h - '0');
234+
else if (h >= 'a' && h <= 'f') d = static_cast<std::uint32_t>(h - 'a' + 10);
235+
else if (h >= 'A' && h <= 'F') d = static_cast<std::uint32_t>(h - 'A' + 10);
236+
else return std::unexpected(ParseError{
237+
std::format("invalid escape '\\{}': expected {} hex digits", esc, digits),
238+
at});
239+
cp = (cp << 4) | d;
240+
}
241+
if ((cp >= 0xD800 && cp <= 0xDFFF) || cp > 0x10FFFF)
242+
return std::unexpected(ParseError{
243+
std::format("invalid escape '\\{}{}': not a Unicode scalar value",
244+
esc, L.src.substr(L.pos + 1, digits)),
245+
at});
246+
append_utf8(out, cp);
247+
for (std::size_t i = 0; i < digits; ++i) L.advance();
248+
break;
249+
}
250+
default:
251+
return std::unexpected(ParseError{
252+
std::format("invalid escape '\\{}'", esc), at});
253+
}
254+
L.advance();
255+
return {};
256+
}
257+
189258
// """multi-line basic""" / '''multi-line literal''' strings (TOML 1.0).
190259
// Entered from read_string once it has seen the triple quote. A newline
191260
// immediately after the opening delimiter is trimmed (spec); basic form
@@ -221,19 +290,7 @@ inline std::expected<std::string, ParseError> read_multiline_string(Lexer& L, ch
221290
continue;
222291
}
223292
L.advance();
224-
char esc = L.peek();
225-
switch (esc) {
226-
case 'n': out.push_back('\n'); break;
227-
case 't': out.push_back('\t'); break;
228-
case 'r': out.push_back('\r'); break;
229-
case '"': out.push_back('"'); break;
230-
case '\\': out.push_back('\\'); break;
231-
case '0': out.push_back('\0'); break;
232-
default:
233-
return std::unexpected(ParseError{
234-
std::format("invalid escape '\\{}'", esc), L.position()});
235-
}
236-
L.advance();
293+
if (auto e = read_escape(L, out); !e) return std::unexpected(e.error());
237294
continue;
238295
}
239296
out.push_back(c);
@@ -256,19 +313,7 @@ inline std::expected<std::string, ParseError> read_string(Lexer& L) {
256313
char c = L.peek();
257314
if (c == '\\' && quote == '"') {
258315
L.advance();
259-
char esc = L.peek();
260-
switch (esc) {
261-
case 'n': out.push_back('\n'); break;
262-
case 't': out.push_back('\t'); break;
263-
case 'r': out.push_back('\r'); break;
264-
case '"': out.push_back('"'); break;
265-
case '\\': out.push_back('\\'); break;
266-
case '0': out.push_back('\0'); break;
267-
default:
268-
return std::unexpected(ParseError{
269-
std::format("invalid escape '\\{}'", esc), L.position()});
270-
}
271-
L.advance();
316+
if (auto e = read_escape(L, out); !e) return std::unexpected(e.error());
272317
} else if (c == '\n') {
273318
return std::unexpected(ParseError{"unterminated string", L.position()});
274319
} else {

‎tests/unit/test_toml.cpp‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,47 @@ TEST(Toml, EscapedString) {
6060
EXPECT_EQ(d->get_string("s").value_or(""), std::string("a\tb\nc"));
6161
}
6262

63+
// TOML 1.0's escapes: \b, \f and the Unicode forms were refused as "invalid
64+
// escape", so a valid manifest written with them could not be read.
65+
TEST(Toml, EveryTomlEscapeIsRead) {
66+
auto d = t::parse(R"(s = "a\bb\fc")");
67+
ASSERT_TRUE(d.has_value()) << d.error().message;
68+
EXPECT_EQ(d->get_string("s").value_or(""), std::string("a\bb\fc"));
69+
70+
// One, two, three and four UTF-8 bytes, upper- and lower-case hex.
71+
d = t::parse(R"(s = "\u0041 caf\u00e9 \u4E2D \U0001F600")");
72+
ASSERT_TRUE(d.has_value()) << d.error().message;
73+
EXPECT_EQ(d->get_string("s").value_or(""),
74+
std::string("A caf\xC3\xA9 \xE4\xB8\xAD \xF0\x9F\x98\x80"));
75+
76+
d = t::parse("s = \"\"\"\nJos\\u00e9\\U0000004A\"\"\"");
77+
ASSERT_TRUE(d.has_value()) << d.error().message;
78+
EXPECT_EQ(d->get_string("s").value_or(""), std::string("Jos\xC3\xA9J"));
79+
80+
// A quoted key is a basic string too.
81+
d = t::parse(R"("caf\u00e9" = 1)");
82+
ASSERT_TRUE(d.has_value()) << d.error().message;
83+
EXPECT_EQ(d->root().count("caf\xC3\xA9"), 1u);
84+
85+
// A literal string keeps its backslashes.
86+
d = t::parse(R"(s = 'C:\u0041')");
87+
ASSERT_TRUE(d.has_value()) << d.error().message;
88+
EXPECT_EQ(d->get_string("s").value_or(""), R"(C:\u0041)");
89+
}
90+
91+
TEST(Toml, AUnicodeEscapeNamesAScalarValue) {
92+
EXPECT_FALSE(t::parse(R"(s = "\u12")").has_value()); // too few digits
93+
EXPECT_FALSE(t::parse(R"(s = "\u12G4")").has_value()); // not hex
94+
EXPECT_FALSE(t::parse(R"(s = "\U0001F60")").has_value()); // \U takes eight
95+
EXPECT_FALSE(t::parse(R"(s = "\uD800")").has_value()); // a surrogate
96+
EXPECT_FALSE(t::parse(R"(s = "\U00110000")").has_value()); // past U+10FFFF
97+
EXPECT_FALSE(t::parse(R"(s = "\x41")").has_value()); // not TOML 1.0
98+
99+
auto bad = t::parse("s = \"ok\\uDFFF\"");
100+
ASSERT_FALSE(bad.has_value());
101+
EXPECT_NE(bad.error().message.find("\\uDFFF"), std::string::npos) << bad.error().message;
102+
}
103+
63104
TEST(Toml, RejectUnterminatedString) {
64105
auto d = t::parse(R"(x = "no end)");
65106
EXPECT_FALSE(d.has_value());

0 commit comments

Comments
 (0)