Skip to content

Commit a3545be

Browse files
committed
fix: py_repr escapes all non-printables per CPython str.isprintable()
Codex review (PR #173): a pasted zero-width space (or any Cf/Zl/Zp/Zs/Co/Cn char) reached the error text raw and invisible, while Python's repr() renders '\u200b' — breaking byte-parity exactly where the diagnostic matters most. py_repr now classifies chars via Unicode general categories (unicode-properties, workspace-pinned) with CPython's isprintable() rule and escapes by code-point width (\xNN / \uNNNN / \UNNNNNNNN). Ground truth pinned two ways: unit tests generated from CPython repr() outputs across the category zoo, and a new shared fixture case (unexpected_char_zero_width_space) replayed from both sides. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TSqD7TSzxAnGWszaFwTZVt
1 parent 6553c17 commit a3545be

6 files changed

Lines changed: 77 additions & 8 deletions

File tree

‎rust/Cargo.lock‎

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎rust/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@ publish = false
1919
[workspace.dependencies]
2020
serde = { version = "1", features = ["derive"] }
2121
serde_json = "1"
22+
# Unicode general categories — the data behind CPython's str.isprintable(),
23+
# which py_repr (error-text parity) must reproduce. unicode-rs org, zero deps.
24+
unicode-properties = { version = "0.1", default-features = false, features = ["general-category"] }
2225

2326
# Strictness per P-022 §"Compiler strictness" — inherited by every crate via
2427
# `[lints] workspace = true`. pedantic/nursery stay WARN (surgical, justified

‎rust/crates/own-syntax/Cargo.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,9 @@ license.workspace = true
77
publish.workspace = true
88
version = "0.1.0"
99

10+
[dependencies]
11+
unicode-properties = { workspace = true }
12+
1013
[dev-dependencies]
1114
serde_json = { workspace = true }
1215

‎rust/crates/own-syntax/src/pyrepr.rs‎

Lines changed: 56 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,37 @@
88
//! * single quotes by default; **double quotes** when the string contains a
99
//! `'` and no `"`;
1010
//! * escape the backslash, the chosen quote, `\n`, `\r`, `\t`;
11-
//! * other C0 controls and DEL as `\xNN`;
12-
//! * printable non-ASCII stays as-is (Python 3 `repr`).
11+
//! * every other character `str.isprintable()` rejects — C0/C1 controls, DEL,
12+
//! format chars (a pasted U+200B/BOM), separators, private-use, unassigned —
13+
//! as `\xNN` / `\uNNNN` / `\UNNNNNNNN` by code-point width;
14+
//! * printable characters (including non-ASCII) stay as-is (Python 3 `repr`).
1315
//!
14-
//! This is not a general `repr` (no `\u`/`\U` for unprintable non-ASCII —
15-
//! `OwnLang` sources in the corpus don't contain them; the parity fixtures
16-
//! would catch it loudly if one ever did).
16+
//! `is_printable` mirrors `CPython`'s `str.isprintable()`: false exactly for
17+
//! general categories Cc, Cf, Cs, Co, Cn, Zl, Zp and Zs-other-than-space.
1718
1819
use std::fmt::Write as _;
1920

21+
use unicode_properties::{GeneralCategory, UnicodeGeneralCategory};
22+
23+
/// `CPython` `str.isprintable()` for one char. (Cs is unreachable — a Rust
24+
/// `char` is never a surrogate — but harmless to name.)
25+
fn is_printable(c: char) -> bool {
26+
if c == ' ' {
27+
return true;
28+
}
29+
!matches!(
30+
c.general_category(),
31+
GeneralCategory::Control // Cc
32+
| GeneralCategory::Format // Cf
33+
| GeneralCategory::Surrogate // Cs
34+
| GeneralCategory::PrivateUse // Co
35+
| GeneralCategory::Unassigned // Cn
36+
| GeneralCategory::LineSeparator // Zl
37+
| GeneralCategory::ParagraphSeparator // Zp
38+
| GeneralCategory::SpaceSeparator // Zs (space itself handled above)
39+
)
40+
}
41+
2042
// pub(crate) in a private module is technically redundant (nursery lint), but
2143
// plain `pub` here trips the denied `unreachable_pub` — keep the honest scope.
2244
#[allow(clippy::redundant_pub_crate)]
@@ -37,9 +59,17 @@ pub(crate) fn py_repr(s: &str) -> String {
3759
out.push('\\');
3860
out.push(c);
3961
}
40-
c if (c as u32) < 0x20 || c as u32 == 0x7f => {
41-
// write! into a String cannot fail; ignore the Ok(()).
42-
let _ = write!(out, "\\x{:02x}", c as u32);
62+
c if !is_printable(c) => {
63+
// CPython picks the escape by code-point width. write! into a
64+
// String cannot fail; ignore the Ok(()).
65+
let cp = c as u32;
66+
if cp < 0x100 {
67+
let _ = write!(out, "\\x{cp:02x}");
68+
} else if cp < 0x10000 {
69+
let _ = write!(out, "\\u{cp:04x}");
70+
} else {
71+
let _ = write!(out, "\\U{cp:08x}");
72+
}
4373
}
4474
c => out.push(c),
4575
}
@@ -63,4 +93,22 @@ mod tests {
6393
assert_eq!(py_repr("\u{1}"), "'\\x01'");
6494
assert_eq!(py_repr("@"), "'@'");
6595
}
96+
97+
/// Expectations produced by running `CPython` `repr()` on each char
98+
/// (`str.isprintable()` categories: Cc, Cf, Co, Cn, Zl, Zp, Zs≠space).
99+
#[test]
100+
fn matches_cpython_nonprintable_escapes() {
101+
assert_eq!(py_repr("\u{200b}"), r"'\u200b'"); // Cf zero-width space
102+
assert_eq!(py_repr("\u{feff}"), r"'\ufeff'"); // Cf BOM
103+
assert_eq!(py_repr("\u{a0}"), r"'\xa0'"); // Zs NBSP — \xNN below U+0100
104+
assert_eq!(py_repr("\u{85}"), r"'\x85'"); // Cc C1 control
105+
assert_eq!(py_repr("\u{2028}"), r"'\u2028'"); // Zl line separator
106+
assert_eq!(py_repr("\u{2029}"), r"'\u2029'"); // Zp paragraph separator
107+
assert_eq!(py_repr("\u{3000}"), r"'\u3000'"); // Zs ideographic space
108+
assert_eq!(py_repr("\u{378}"), r"'\u0378'"); // Cn unassigned
109+
assert_eq!(py_repr("\u{e000}"), r"'\ue000'"); // Co private use
110+
assert_eq!(py_repr("\u{10ffff}"), r"'\U0010ffff'"); // Cn astral
111+
assert_eq!(py_repr("\u{1f600}"), "'\u{1f600}'"); // So printable emoji
112+
assert_eq!(py_repr("é ж"), "'é ж'"); // printable non-ASCII + space
113+
}
66114
}

‎tests/fixtures/syntax_parity.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,11 @@
5151
"source": "module m \u0001",
5252
"error": "1:10: unexpected character '\\x01'"
5353
},
54+
{
55+
"name": "unexpected_char_zero_width_space",
56+
"source": "module m ​",
57+
"error": "1:10: unexpected character '\\u200b'"
58+
},
5459
{
5560
"name": "unexpected_char_after_unicode",
5661
"source": "module m\nfn f() { let ф = @; }",

‎tests/test_syntax_fixtures.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@
130130
("unexpected_char", "@"),
131131
("unexpected_char_backslash", "module m \\"),
132132
("unexpected_char_control", "module m \x01"),
133+
# Cf format char (a pasted zero-width space): repr() renders it via the
134+
# \uNNNN path of the isprintable()-parity quoting.
135+
("unexpected_char_zero_width_space", "module m \u200b"),
133136
("unexpected_char_after_unicode", "module m\nfn f() { let ф = @; }"),
134137
("unterminated_string", 'module m resource R { emit_type "oops'),
135138
("rejected_keyword_top_level", "module m for"),

0 commit comments

Comments
 (0)