Skip to content

Commit 5b25582

Browse files
authored
gh-160522: Restore CJK encoder state after a failed incremental encode (GH-160523)
A failed IncrementalEncoder.encode() or StreamWriter.write() of a stateful CJK codec kept the shift state reached before the error although its output was discarded.
1 parent f32067f commit 5b25582

3 files changed

Lines changed: 51 additions & 0 deletions

File tree

‎Lib/test/test_multibytecodec.py‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -461,6 +461,48 @@ def test_incrementalencoder_final(self):
461461
self.assertEqual(output, self.expected_reset)
462462
self.assertEqual(encoder.encode('', final=True), b'')
463463

464+
def test_incrementalencoder_error(self):
465+
# The output of a failed encode() is lost, so the shift state
466+
# must not be changed.
467+
encoder = codecs.getincrementalencoder(self.encoding)()
468+
state = encoder.getstate()
469+
for final in False, True:
470+
self.assertRaises(UnicodeEncodeError, encoder.encode,
471+
self.text + '\U0001f600', final)
472+
self.assertEqual(encoder.getstate(), state)
473+
self.assertEqual(encoder.encode(self.text), self.expected)
474+
state = encoder.getstate()
475+
for final in False, True:
476+
self.assertRaises(UnicodeEncodeError, encoder.encode,
477+
'a\U0001f600', final)
478+
self.assertEqual(encoder.getstate(), state)
479+
self.assertEqual(encoder.encode('a'), self.reset + b'a')
480+
481+
def test_streamwriter_error(self):
482+
stream = io.BytesIO()
483+
writer = codecs.getwriter(self.encoding)(stream)
484+
self.assertRaises(UnicodeEncodeError, writer.write,
485+
self.text + '\U0001f600')
486+
self.assertEqual(stream.getvalue(), b'')
487+
writer.write(self.text)
488+
self.assertEqual(stream.getvalue(), self.expected)
489+
490+
def test_textiowrapper_error(self):
491+
raw = io.BytesIO()
492+
with io.TextIOWrapper(raw, encoding=self.encoding) as f:
493+
self.assertRaises(UnicodeEncodeError, f.write,
494+
self.text + '\U0001f600')
495+
f.write(self.text)
496+
f.flush()
497+
self.assertEqual(raw.getvalue(), self.expected)
498+
499+
class TestISO2022KRStateful(TestStateful):
500+
text = '\uac00\uac00'
501+
encoding = 'iso-2022-kr'
502+
expected = b'\x1b$)C\x0e0!0!'
503+
reset = b'\x0f'
504+
expected_reset = expected + reset
505+
464506
class TestHZStateful(TestStateful):
465507
text = '\u804a\u804a'
466508
encoding = 'hz'
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Fix the incremental encoders and stream writers of the stateful CJK codecs
2+
(``iso2022_*`` and ``hz``) keeping a changed shift state after an
3+
:meth:`!encode` or :meth:`!write` call failed. The next call could omit the
4+
escape sequence, writing multibyte characters in ASCII mode.

‎Modules/cjkcodecs/multibytecodec.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -794,6 +794,7 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
794794
PyObject *inbuf = NULL;
795795
Py_ssize_t inpos, datalen;
796796
PyObject *origpending = NULL;
797+
MultibyteCodec_State origstate;
797798

798799
if (PyUnicode_Check(unistr))
799800
ucvt = NULL;
@@ -809,6 +810,7 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
809810
}
810811
}
811812

813+
origstate = ctx->state;
812814
if (ctx->pending) {
813815
PyObject *inbuf_tmp;
814816

@@ -866,6 +868,9 @@ encoder_encode_stateful(MultibyteStatefulEncoderContext *ctx,
866868
return r;
867869

868870
errorexit:
871+
/* recover the original codec state: the discarded output may have
872+
* included an escape sequence for a state change */
873+
ctx->state = origstate;
869874
Py_XDECREF(r);
870875
Py_XDECREF(ucvt);
871876
Py_XDECREF(origpending);

0 commit comments

Comments
 (0)