Skip to content

Fix formatting, numeric, unicode, codec, complex and marshal mismatches found by differential testing #8600

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 11 commits into from
Aug 29, 2026
Merged

Fix formatting, numeric, unicode, codec, complex and marshal mismatches found by differential testing #8600

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
8106f19
Fix %-formatting of bytes precision, width and %c errors
youknowone Aug 25, 2026
057637b
Fix format spec handling of grouping, bool, 'c' and complex
youknowone Aug 25, 2026
1d7a331
Round floats at a negative ndigits from the exact decimal digits
youknowone Aug 25, 2026
ff47c3f
Report the requested base in int literal errors and reject a bare sign
youknowone Aug 25, 2026
09c0d33
Split, strip and expand tabs over the Py_UNICODE_ISSPACE set
youknowone Aug 25, 2026
c2e15b0
Read Unicode numeric values as exact rationals
youknowone Aug 25, 2026
1ed8725
Return a mappingproxy from Pattern.groupindex
youknowone Aug 25, 2026
cb9fe18
Fix the utf-16/32/7 codecs' error handling
youknowone Aug 25, 2026
4b54828
Rework complex multiplication, division and power
youknowone Aug 25, 2026
f1b0f68
Intern the string constants that read as names
youknowone Aug 25, 2026
a5fbe53
Honour the marshal version and separate its EOF errors
youknowone Aug 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
132 changes: 74 additions & 58 deletions Lib/_pycodecs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -479,6 +479,24 @@ def STORECHAR32(ch, byteorder):
return [b3, b2, b1, b0]


def _encoded_replacement(replacement, store, unit_bytes, encoding, s, pos):
"""The bytes an error handler's replacement adds to the output.

A str is encoded the way the rest of the input is. A bytes object is taken
as it stands, and only when it fills whole code units.
"""
if isinstance(replacement, bytes):
if len(replacement) % unit_bytes:
raise UnicodeEncodeError(
encoding, s, pos, pos + 1, "surrogates not allowed"
)
return list(replacement)
encoded = []
for c in replacement:
encoded += store(ord(c))
return encoded


def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"):
"""Encode a Unicode string to UTF-32."""
p = []
Expand All @@ -494,21 +512,23 @@ def PyUnicode_EncodeUTF32(s, size, errors, byteorder="little"):
elif byteorder == "big":
bom = "big"

# Only the mode that writes a BOM leaves the order out of its name.
encoding = "utf-32" if byteorder == "native" else "utf-32-" + bom[0] + "e"

def store(cp):
return STORECHAR32(cp, bom)

pos = 0
while pos < len(s):
ch = ord(s[pos])
if 0xD800 <= ch <= 0xDFFF:
if errors == "surrogatepass":
p += STORECHAR32(ch, bom)
pos += 1
else:
res, pos = unicode_call_errorhandler(
errors, "utf-32", "surrogates not allowed", s, pos, pos + 1, False
)
for c in res:
p += STORECHAR32(ord(c), bom)
if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass":
start = pos
res, pos = unicode_call_errorhandler(
errors, encoding, "surrogates not allowed", s, pos, pos + 1, False
)
p += _encoded_replacement(res, store, 4, encoding, s, start)
else:
p += STORECHAR32(ch, bom)
p += store(ch)
pos += 1

return p
Expand Down Expand Up @@ -541,6 +561,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=
result = []
pos = 0
aligned_size = (size // 4) * 4
encoding = "utf-32-le" if byteorder == "little" else "utf-32-be"

while pos + 3 < aligned_size:
if byteorder == "little":
Expand All @@ -560,26 +581,24 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=

# Validate code point
if ch > 0x10FFFF:
if errors == "strict":
raise UnicodeDecodeError(
"utf-32",
bytes(data),
pos,
pos + 4,
"codepoint not in range(0x110000)",
)
elif errors == "replace":
result.append("\ufffd")
# 'ignore' - skip this character
pos += 4
res, pos = unicode_call_errorhandler(
errors,
encoding,
"code point not in range(0x110000)",
data,
pos,
pos + 4,
True,
)
result.append(res)
elif 0xD800 <= ch <= 0xDFFF:
if errors == "surrogatepass":
result.append(chr(ch))
pos += 4
else:
msg = "code point in surrogate code point range(0xd800, 0xe000)"
res, pos = unicode_call_errorhandler(
errors, "utf-32", msg, data, pos, pos + 4, True
errors, encoding, msg, data, pos, pos + 4, True
)
result.append(res)
else:
Expand All @@ -590,7 +609,7 @@ def PyUnicode_DecodeUTF32Stateful(data, size, errors, byteorder="little", final=
if pos < size:
if final:
res, pos = unicode_call_errorhandler(
errors, "utf-32", "truncated data", data, pos, size, True
errors, encoding, "truncated data", data, pos, size, True
)
if res:
result.append(res)
Expand Down Expand Up @@ -915,7 +934,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
i += 1
errmsg = "partial character in shift sequence"
out, i = unicode_call_errorhandler(
errors, "utf-7", errmsg, s, startinpos, i
errors, "utf7", errmsg, s, startinpos, i
)
p.append(out)
continue
Expand All @@ -924,7 +943,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
i += 1
errmsg = "non-zero padding bits in shift sequence"
out, i = unicode_call_errorhandler(
errors, "utf-7", errmsg, s, startinpos, i
errors, "utf7", errmsg, s, startinpos, i
)
p.append(out)
continue
Expand All @@ -943,7 +962,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
i += 1
errmsg = "ill-formed sequence"
out, i = unicode_call_errorhandler(
errors, "utf-7", errmsg, s, startinpos, i
errors, "utf7", errmsg, s, startinpos, i
)
p.append(out)
else:
Expand All @@ -960,7 +979,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
i += 1
errmsg = "unexpected special character"
out, i = unicode_call_errorhandler(
errors, "utf-7", errmsg, s, startinpos, i
errors, "utf7", errmsg, s, startinpos, i
)
p.append(out)

Expand All @@ -971,7 +990,7 @@ def PyUnicode_DecodeUTF7(s, size, errors, final=False):
if surrogate or base64bits >= 6 or (base64bits > 0 and base64buffer != 0):
errmsg = "unterminated shift sequence"
out, i = unicode_call_errorhandler(
errors, "utf-7", errmsg, s, startinpos, size
errors, "utf7", errmsg, s, startinpos, size
)
p.append(out)

Expand Down Expand Up @@ -1219,13 +1238,16 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
ihi = 0
ilo = 1

# The order the data turned out to be in is the one an error names.
encoding = "utf-16-be" if ihi == 0 else "utf-16-le"

while q < len(s):
# /* remaining bytes at the end? (size should be even) */
if len(s) - q < 2:
if not final:
break
res, q = unicode_call_errorhandler(
errors, "utf-16", "truncated data", s, q, len(s), True
errors, encoding, "truncated data", s, q, len(s), True
)
p.append(res)
break
Expand Down Expand Up @@ -1253,7 +1275,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
q += 2
continue
res, q = unicode_call_errorhandler(
errors, "utf-16", "illegal UTF-16 surrogate", s, q, q + 2, True
errors, encoding, "illegal UTF-16 surrogate", s, q, q + 2, True
)
p.append(res)
else:
Expand All @@ -1265,7 +1287,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
q += 2
continue
res, q = unicode_call_errorhandler(
errors, "utf-16", "unexpected end of data", s, q, len(s), True
errors, encoding, "unexpected end of data", s, q, len(s), True
)
p.append(res)
else:
Expand All @@ -1275,7 +1297,7 @@ def PyUnicode_DecodeUTF16Stateful(s, size, errors, byteorder="native", final=Tru
q += 2
continue
res, q = unicode_call_errorhandler(
errors, "utf-16", "illegal encoding", s, q, q + 2, True
errors, encoding, "illegal encoding", s, q, q + 2, True
)
p.append(res)

Expand Down Expand Up @@ -1309,34 +1331,28 @@ def PyUnicode_EncodeUTF16(s, size, errors, byteorder="little"):
elif byteorder == "big":
bom = "big"

# Only the mode that writes a BOM leaves the order out of its name.
encoding = "utf-16" if byteorder == "native" else "utf-16-" + bom[0] + "e"

def store(cp):
if cp >= 0x10000:
cp -= 0x10000
return STORECHAR(0xD800 | (cp >> 10), bom) + STORECHAR(
0xDC00 | (cp & 0x3FF), bom
)
return STORECHAR(cp, bom)

pos = 0
while pos < len(s):
ch = ord(s[pos])
if 0xD800 <= ch <= 0xDFFF:
if errors == "surrogatepass":
p += STORECHAR(ch, bom)
pos += 1
else:
res, pos = unicode_call_errorhandler(
errors, "utf-16", "surrogates not allowed", s, pos, pos + 1, False
)
for c in res:
cp = ord(c)
cp2 = 0
if cp >= 0x10000:
cp2 = 0xDC00 | ((cp - 0x10000) & 0x3FF)
cp = 0xD800 | ((cp - 0x10000) >> 10)
p += STORECHAR(cp, bom)
if cp2:
p += STORECHAR(cp2, bom)
if 0xD800 <= ch <= 0xDFFF and errors != "surrogatepass":
start = pos
res, pos = unicode_call_errorhandler(
errors, encoding, "surrogates not allowed", s, pos, pos + 1, False
)
p += _encoded_replacement(res, store, 2, encoding, s, start)
else:
ch2 = 0
if ch >= 0x10000:
ch2 = 0xDC00 | ((ch - 0x10000) & 0x3FF)
ch = 0xD800 | ((ch - 0x10000) >> 10)
p += STORECHAR(ch, bom)
if ch2:
p += STORECHAR(ch2, bom)
p += store(ch)
pos += 1

return p
Expand Down
4 changes: 0 additions & 4 deletions Lib/test/test_codeccallbacks.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -854,8 +854,6 @@ def unencrepl(exc):
self.assertEqual(exc.end, 2)
self.assertEqual(exc.object, input)

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_encode_bytes_replacement(self):
def handle(exc):
if isinstance(exc, UnicodeEncodeError):
Expand All @@ -878,8 +876,6 @@ def handle(exc):
res = input.encode(enc, "test.replacing")
self.assertEqual(res, "[".encode(enc) + repl + "]".encode(enc))

# TODO: RUSTPYTHON
@unittest.expectedFailure
def test_encode_odd_bytes_replacement(self):
def handle(exc):
if isinstance(exc, UnicodeEncodeError):
Expand Down
2 changes: 0 additions & 2 deletions Lib/test/test_complex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,6 @@ def check_div(self, x, y):
if y:
self.assertClose(z / y, x)

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: floats nan and inf are not identical
def test_truediv(self):
simple_real = [float(i) for i in range(-5, 6)]
simple_complex = [complex(x, y) for x in simple_real for y in simple_real]
Expand Down Expand Up @@ -289,7 +288,6 @@ def test_sub(self):
self.assertRaises(TypeError, operator.sub, 1j, None)
self.assertRaises(TypeError, operator.sub, None, 1j)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_mul(self):
self.assertEqual(1j * int(20), complex(0, 20))
self.assertEqual(1j * int(-1), complex(0, -1))
Expand Down
1 change: 0 additions & 1 deletion Lib/test/test_re.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,7 +772,6 @@ def test_repeat_minmax(self):
self.checkPatternError(r'x{2,1}',
'min repeat greater than max repeat', 2)

@unittest.expectedFailure # TODO: RUSTPYTHON
def test_getattr(self):
self.assertEqual(re.compile("(?i)(a)(b)").pattern, "(?i)(a)(b)")
self.assertEqual(re.compile("(?i)(a)(b)").flags, re.I | re.U)
Expand Down
15 changes: 8 additions & 7 deletions crates/common/src/cformat.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -426,15 +426,16 @@ impl CFormatSpec {

#[must_use]
pub fn format_bytes(&self, bytes: &[u8]) -> Vec<u8> {
let bytes = if let Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) =
self.precision
{
&bytes[..cmp::min(bytes.len(), precision)]
} else {
bytes
let bytes = match self.precision {
Some(CFormatPrecision::Quantity(CFormatQuantity::Amount(precision))) => {
&bytes[..cmp::min(bytes.len(), precision)]
}
// A precision written as a bare dot truncates to nothing.
Some(CFormatPrecision::Dot) => &bytes[..0],
_ => bytes,
};
if let Some(CFormatQuantity::Amount(width)) = self.min_field_width {
let fill = cmp::max(0, width - bytes.len());
let fill = width.saturating_sub(bytes.len());
let mut v = Vec::with_capacity(bytes.len() + fill);
if self.flags.contains(CConversionFlags::LEFT_ADJUST) {
v.extend_from_slice(bytes);
Expand Down
Loading
Loading