Skip to content

common: add VM-independent codec and text engines #8622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 7 commits into from
Aug 31, 2026
Merged

common: add VM-independent codec and text engines #8622

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
c52db4a
common: add PyPy CJK codec engines
youknowone Aug 30, 2026
3f9d63e
common: mark CJK mapping tables generated
youknowone Aug 30, 2026
617bdcf
common: add reusable byte and text engines
youknowone Aug 30, 2026
7b18e78
common: record opt-in engine dependencies
youknowone Aug 30, 2026
25cec71
stdlib: add `_multibytecodec` and the six `_codecs_*` modules
youknowone Aug 30, 2026
3e6dc97
stdlib: use the byte and text engines from `rustpython-common`
youknowone Aug 30, 2026
c3bd3e6
test: enable the Windows cp932 codec case
youknowone Aug 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .cspell.dict/cpython.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ cfws
CFWS
chans
cidx
cjkcodecs
CLASSDEREF
classdict
cmpop
Expand Down Expand Up @@ -65,6 +66,7 @@ elts
eooh
eofs
EOOH
esize
evalloop
excepthandler
exceptiontable
Expand Down Expand Up @@ -138,6 +140,9 @@ lslpp
lsprof
MAXBLOCKS
maxdepth
mbiencoder
mbstreamreader
mbstreamwriter
metavars
miscompiles
mult
Expand Down
16 changes: 16 additions & 0 deletions .cspell.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,22 @@
"Lib/**",
"crates/host_env/**"
],
"overrides": [
{
"filename": "crates/common/src/encodings/cjk/**",
"words": [
"CHOSEONG",
"EUCKR",
"famille",
"GBCOMMON",
"jong",
"JONGSEONG",
"JUNGSEONG",
"PHINT",
"Xabc"
]
}
],
// words - list of words to be always considered correct
// (compound words like pyarg, baseclass, microbenchmark are handled by allowCompoundWords)
"words": [
Expand Down
1 change: 1 addition & 0 deletions .gitattributes
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ Lib/*.py text working-tree-encoding=UTF-8 eol=LF
crates/vm/src/stdlib/ast/gen.rs linguist-generated -merge
crates/doc/src/data.inc.rs generated
crates/compiler-core/src/bytecode/opcode_metadata.rs generated
crates/common/src/encodings/cjk/mappings_*.rs generated

.github/workflows/*.lock.yml linguist-generated=true merge=ours

Expand Down
5 changes: 3 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 0 additions & 4 deletions Lib/test/test_codecencodings_cn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb2312")
class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'gb2312'
tstring = multibytecodec_support.load_teststring('gb2312')
Expand All @@ -20,7 +19,6 @@ class Test_GB2312(multibytecodec_support.TestBase, unittest.TestCase):
(b"\xc1\x64", "strict", None),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gbk")
class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'gbk'
tstring = multibytecodec_support.load_teststring('gbk')
Expand All @@ -35,7 +33,6 @@ class Test_GBK(multibytecodec_support.TestBase, unittest.TestCase):
("\u30fb", "strict", None),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: gb18030")
class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'gb18030'
tstring = multibytecodec_support.load_teststring('gb18030')
Expand All @@ -60,7 +57,6 @@ class Test_GB18030(multibytecodec_support.TestBase, unittest.TestCase):
)
has_iso10646 = True

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: hz")
class Test_HZ(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'hz'
tstring = multibytecodec_support.load_teststring('hz')
Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_codecencodings_hk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5hkscs")
class Test_Big5HKSCS(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'big5hkscs'
tstring = multibytecodec_support.load_teststring('big5hkscs')
Expand Down
5 changes: 0 additions & 5 deletions Lib/test/test_codecencodings_iso2022.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,23 +10,20 @@
(b'ab\x1B$def', 'replace', 'ab\uFFFD'),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp")
class Test_ISO2022_JP(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
codectests = COMMON_CODEC_TESTS + (
(b'ab\x1BNdef', 'replace', 'ab\x1BNdef'),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2")
class Test_ISO2022_JP2(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp_2'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
codectests = COMMON_CODEC_TESTS + (
(b'ab\x1BNdef', 'replace', 'abdef'),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_3")
class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp_3'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
Expand All @@ -50,7 +47,6 @@ class Test_ISO2022_JP3(multibytecodec_support.TestBase, unittest.TestCase):
b'\x1B$(O\x29\x28\x1B(Bℜ\x1B$(O\x29\x32\x1B(B = ⟨ሴ⟩'
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_jp_2004")
class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_jp_2004'
tstring = multibytecodec_support.load_teststring('iso2022_jp')
Expand All @@ -74,7 +70,6 @@ class Test_ISO2022_JP2004(multibytecodec_support.TestBase, unittest.TestCase):
b'\x1B$(Q\x29\x28\x1B(Bℜ\x1B$(Q\x29\x32\x1B(B = ⟨ሴ⟩'
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: iso2022_kr")
class Test_ISO2022_KR(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'iso2022_kr'
tstring = multibytecodec_support.load_teststring('iso2022_kr')
Expand Down
7 changes: 0 additions & 7 deletions Lib/test/test_codecencodings_jp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp932")
class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'cp932'
tstring = multibytecodec_support.load_teststring('shift_jis')
Expand Down Expand Up @@ -40,7 +39,6 @@ class Test_CP932(multibytecodec_support.TestBase, unittest.TestCase):
(b"\x8eXY", "replace", "\ufffdXY"),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jis_2004")
class Test_EUC_JIS_2004(multibytecodec_support.TestBase,
unittest.TestCase):
encoding = 'euc_jis_2004'
Expand All @@ -51,7 +49,6 @@ class Test_EUC_JIS_2004(multibytecodec_support.TestBase,
b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩"
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jisx0213")
class Test_EUC_JISX0213(multibytecodec_support.TestBase,
unittest.TestCase):
encoding = 'euc_jisx0213'
Expand All @@ -62,7 +59,6 @@ class Test_EUC_JISX0213(multibytecodec_support.TestBase,
b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩"
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_jp")
class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase,
unittest.TestCase):
encoding = 'euc_jp'
Expand All @@ -78,7 +74,6 @@ class Test_EUC_JP_COMPAT(multibytecodec_support.TestBase,
(b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis")
class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jis'
tstring = multibytecodec_support.load_teststring('shift_jis')
Expand All @@ -93,7 +88,6 @@ class Test_SJIS_COMPAT(multibytecodec_support.TestBase, unittest.TestCase):
(b"abc\xFF\x58", "replace", "abc\ufffdX"),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jis_2004")
class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jis_2004'
tstring = multibytecodec_support.load_teststring('shift_jis')
Expand All @@ -112,7 +106,6 @@ class Test_SJIS_2004(multibytecodec_support.TestBase, unittest.TestCase):
b"\x85Gℜ\x85Q = ⟨ሴ⟩"
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: shift_jisx0213")
class Test_SJISX0213(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jisx0213'
tstring = multibytecodec_support.load_teststring('shift_jisx0213')
Expand Down
3 changes: 0 additions & 3 deletions Lib/test/test_codecencodings_kr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: cp949")
class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'cp949'
tstring = multibytecodec_support.load_teststring('cp949')
Expand All @@ -19,7 +18,6 @@ class Test_CP949(multibytecodec_support.TestBase, unittest.TestCase):
(b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: euc_kr")
class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'euc_kr'
tstring = multibytecodec_support.load_teststring('euc_kr')
Expand Down Expand Up @@ -51,7 +49,6 @@ class Test_EUCKR(multibytecodec_support.TestBase, unittest.TestCase):
(b"\xc1\xc4", "strict", "\uc894"),
)

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: johab")
class Test_JOHAB(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'johab'
tstring = multibytecodec_support.load_teststring('johab')
Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_codecencodings_tw.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
from test import multibytecodec_support
import unittest

@unittest.skip("TODO: RUSTPYTHON; unknown encoding: big5")
class Test_Big5(multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'big5'
tstring = multibytecodec_support.load_teststring('big5')
Expand Down
12 changes: 0 additions & 12 deletions Lib/test/test_codecmaps_cn.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,28 +11,16 @@ class TestGB2312Map(multibytecodec_support.TestBase_Mapping,
encoding = 'gb2312'
mapfileurl = 'http://www.pythontest.net/unicode/EUC-CN.TXT'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb2312
def test_mapping_file(self):
return super().test_mapping_file()

class TestGBKMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'gbk'
mapfileurl = 'http://www.pythontest.net/unicode/CP936.TXT'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gbk
def test_mapping_file(self):
return super().test_mapping_file()

class TestGB18030Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'gb18030'
mapfileurl = 'http://www.pythontest.net/unicode/gb-18030-2000.xml'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: gb18030
def test_mapping_file(self):
return super().test_mapping_file()


if __name__ == "__main__":
unittest.main()
4 changes: 0 additions & 4 deletions Lib/test/test_codecmaps_hk.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,5 @@ class TestBig5HKSCSMap(multibytecodec_support.TestBase_Mapping,
encoding = 'big5hkscs'
mapfileurl = 'http://www.pythontest.net/unicode/BIG5HKSCS-2004.TXT'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: big5hkscs
def test_mapping_file(self):
return super().test_mapping_file()

if __name__ == "__main__":
unittest.main()
24 changes: 0 additions & 24 deletions Lib/test/test_codecmaps_jp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,25 +20,13 @@ class TestCP932Map(multibytecodec_support.TestBase_Mapping,
for i in range(0xa1, 0xe0):
supmaps.append((bytes([i]), chr(i+0xfec0)))

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932
def test_mapping_file(self):
return super().test_mapping_file()

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: cp932
def test_mapping_supplemental(self):
return super().test_mapping_supplemental()


class TestEUCJPCOMPATMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'euc_jp'
mapfilename = 'EUC-JP.TXT'
mapfileurl = 'http://www.pythontest.net/unicode/EUC-JP.TXT'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jp
def test_mapping_file(self):
return super().test_mapping_file()


class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
Expand All @@ -54,31 +42,19 @@ class TestSJISCOMPATMap(multibytecodec_support.TestBase_Mapping,
(b'\x81_', '\\'),
]

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jis
def test_mapping_file(self):
return super().test_mapping_file()

class TestEUCJISX0213Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'euc_jisx0213'
mapfilename = 'EUC-JISX0213.TXT'
mapfileurl = 'http://www.pythontest.net/unicode/EUC-JISX0213.TXT'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_jisx0213
def test_mapping_file(self):
return super().test_mapping_file()


class TestSJISX0213Map(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'shift_jisx0213'
mapfilename = 'SHIFT_JISX0213.TXT'
mapfileurl = 'http://www.pythontest.net/unicode/SHIFT_JISX0213.TXT'

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: shift_jisx0213
def test_mapping_file(self):
return super().test_mapping_file()


if __name__ == "__main__":
unittest.main()
12 changes: 0 additions & 12 deletions Lib/test/test_codecmaps_kr.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,6 @@ class TestCP949Map(multibytecodec_support.TestBase_Mapping,
encoding = 'cp949'
mapfileurl = 'http://www.pythontest.net/unicode/CP949.TXT'

@unittest.expectedFailureIf(__import__("sys").platform in ("android", "darwin", "linux"), "TODO: RUSTPYTHON; LookupError: unknown encoding: cp949")
def test_mapping_file(self):
return super().test_mapping_file()


class TestEUCKRMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
Expand All @@ -25,10 +21,6 @@ class TestEUCKRMap(multibytecodec_support.TestBase_Mapping,
pass_enctest = [(b'\xa4\xd4', '\u3164')]
pass_dectest = [(b'\xa4\xd4', '\u3164')]

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: euc_kr
def test_mapping_file(self):
return super().test_mapping_file()


class TestJOHABMap(multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
Expand All @@ -41,9 +33,5 @@ class TestJOHABMap(multibytecodec_support.TestBase_Mapping,
pass_enctest = [(b'\\', '\u20a9')]
pass_dectest = [(b'\\', '\u20a9')]

@unittest.expectedFailure # TODO: RUSTPYTHON; LookupError: unknown encoding: johab
def test_mapping_file(self):
return super().test_mapping_file()

if __name__ == "__main__":
unittest.main()
Loading
Loading