Skip to content

Commit 48ea2c5

Browse files
committed
console_mode
1 parent 98f490b commit 48ea2c5

1 file changed

Lines changed: 63 additions & 0 deletions

File tree

‎language_tool/console_mode.py‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
"""Write to stdout without causing UnicodeEncodeError
2+
"""
3+
4+
import sys
5+
6+
7+
if (getattr(sys.stdout, "errors", "") == "strict" and
8+
not getattr(sys.stdout, "encoding", "").lower().startswith("utf")):
9+
try:
10+
import translit
11+
sys.stdout = translit.StreamFilter(sys.stdout)
12+
except ImportError:
13+
import codecs
14+
import unicodedata
15+
import warnings
16+
17+
TRANSLIT_MAP = {
18+
0x2018: "'",
19+
0x2019: "'",
20+
0x201c: '"',
21+
0x201d: '"',
22+
}
23+
24+
def simplify(s):
25+
s = s.translate(TRANSLIT_MAP)
26+
return "".join([c for c in unicodedata.normalize("NFKD", s)
27+
if not unicodedata.combining(c)])
28+
29+
def simple_translit_error_handler(error):
30+
if not isinstance(error, UnicodeEncodeError):
31+
raise error
32+
chunk = error.object[error.start:error.end]
33+
repl = simplify(chunk)
34+
repl = (repl.encode(error.encoding, "backslashreplace")
35+
.decode(error.encoding))
36+
return repl, error.end
37+
38+
class SimpleTranslitStreamFilter:
39+
"""Filter a stream through simple transliteration.
40+
"""
41+
errors = "simple_translit"
42+
43+
def __init__(self, target):
44+
self.target = target
45+
46+
def __getattr__(self, name):
47+
return getattr(self.target, name)
48+
49+
def write(self, s):
50+
self.target.write(self.downgrade(s))
51+
52+
def writelines(self, lines):
53+
self.target.writelines(
54+
[self.downgrade(line) for line in lines])
55+
56+
def downgrade(self, s):
57+
return (s.encode(self.target.encoding, self.errors)
58+
.decode(self.target.encoding))
59+
60+
codecs.register_error(SimpleTranslitStreamFilter.errors,
61+
simple_translit_error_handler)
62+
sys.stdout = SimpleTranslitStreamFilter(sys.stdout)
63+
warnings.warn("translit is unavailable", ImportWarning)

0 commit comments

Comments
 (0)