Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 64 additions & 0 deletions Lib/test/test_free_threading/test_bz2.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import unittest

from test.support import import_helper, threading_helper
from test.support.threading_helper import run_concurrently

bz2 = import_helper.import_module("bz2")
from bz2 import BZ2Compressor, BZ2Decompressor

from test.test_bz2 import ext_decompress, BaseTest


NTHREADS = 10
TEXT = BaseTest.TEXT


@threading_helper.requires_working_threading()
class TestBZ2(unittest.TestCase):
def test_compressor(self):
bz2c = BZ2Compressor()

def worker():
# it should return empty bytes as it buffers data internally
data = bz2c.compress(TEXT)
self.assertEqual(data, b"")

run_concurrently(worker_func=worker, nthreads=NTHREADS)
data = bz2c.flush()
# The decompressed data should be TEXT repeated NTHREADS times
decompressed = ext_decompress(data)
self.assertEqual(decompressed, TEXT * NTHREADS)

def test_decompressor(self):
chunk_size = 128
chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)]
input_data = b"".join(chunks)
compressed = bz2.compress(input_data)

bz2d = BZ2Decompressor()
output = []

def worker():
data = bz2d.decompress(compressed, chunk_size)
self.assertEqual(len(data), chunk_size)
output.append(data)
# Read attributes concurrently with other threads decompressing
self.assertIsInstance(bz2d.eof, bool)
self.assertIsInstance(bz2d.needs_input, bool)
self.assertIsInstance(bz2d.unused_data, bytes)

run_concurrently(worker_func=worker, nthreads=NTHREADS)
self.assertEqual(len(output), NTHREADS)
# Verify the expected chunks (order doesn't matter due to append race)
self.assertEqual(set(output), set(chunks))
self.assertTrue(bz2d.eof)
self.assertFalse(bz2d.needs_input)
# Each thread added full compressed data to the buffer, but only 1 copy
# is consumed to produce the output. The rest remains as unused_data.
self.assertEqual(
len(bz2d.unused_data), len(compressed) * (NTHREADS - 1)
)


if __name__ == "__main__":
unittest.main()
69 changes: 69 additions & 0 deletions Lib/test/test_free_threading/test_lzma.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import unittest

from test.support import import_helper, threading_helper
from test.support.threading_helper import run_concurrently

lzma = import_helper.import_module("lzma")
from lzma import LZMACompressor, LZMADecompressor

from test.test_lzma import INPUT


NTHREADS = 10


@threading_helper.requires_working_threading()
class TestLZMA(unittest.TestCase):
def test_compressor(self):
lzc = LZMACompressor()

# First compress() outputs LZMA header
header = lzc.compress(INPUT)
self.assertGreater(len(header), 0)

def worker():
# it should return empty bytes as it buffers data internally
data = lzc.compress(INPUT)
self.assertEqual(data, b"")

run_concurrently(worker_func=worker, nthreads=NTHREADS - 1)
full_compressed = header + lzc.flush()
decompressed = lzma.decompress(full_compressed)
# The decompressed data should be INPUT repeated NTHREADS times
self.assertEqual(decompressed, INPUT * NTHREADS)

def test_decompressor(self):
chunk_size = 128
chunks = [bytes([ord("a") + i]) * chunk_size for i in range(NTHREADS)]
input_data = b"".join(chunks)
compressed = lzma.compress(input_data)

lzd = LZMADecompressor()
output = []

def worker():
data = lzd.decompress(compressed, chunk_size)
self.assertEqual(len(data), chunk_size)
output.append(data)
# Read attributes concurrently with other threads decompressing
self.assertEqual(lzd.check, lzma.CHECK_CRC64)
self.assertIsInstance(lzd.eof, bool)
self.assertIsInstance(lzd.needs_input, bool)
self.assertIsInstance(lzd.unused_data, bytes)

run_concurrently(worker_func=worker, nthreads=NTHREADS)
self.assertEqual(len(output), NTHREADS)
# Verify the expected chunks (order doesn't matter due to append race)
self.assertSetEqual(set(output), set(chunks))
self.assertEqual(lzd.check, lzma.CHECK_CRC64)
self.assertTrue(lzd.eof)
self.assertFalse(lzd.needs_input)
# Each thread added full compressed data to the buffer, but only 1 copy
# is consumed to produce the output. The rest remains as unused_data.
self.assertEqual(
len(lzd.unused_data), len(compressed) * (NTHREADS - 1)
)


if __name__ == "__main__":
unittest.main()
78 changes: 78 additions & 0 deletions Lib/test/test_free_threading/test_pickle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
import pickle
import threading
import unittest

from test.support import threading_helper


@threading_helper.requires_working_threading()
class TestPickleFreeThreading(unittest.TestCase):

def test_pickle_dumps_with_concurrent_dict_mutation(self):
# gh-146452: Pickling a dict while another thread mutates it
# used to segfault. batch_dict_exact() iterated dict items via
# PyDict_Next() which returns borrowed references, and a
# concurrent pop/replace could free the value before Py_INCREF
# got to it.
shared = {str(i): list(range(20)) for i in range(50)}

def dumper():
for _ in range(1000):
try:
pickle.dumps(shared)
except RuntimeError:
# "dictionary changed size during iteration" is expected
pass

def mutator():
for j in range(1000):
key = str(j % 50)
shared[key] = list(range(j % 20))
if j % 10 == 0:
shared.pop(key, None)
shared[key] = [j]

threads = []
for _ in range(10):
threads.append(threading.Thread(target=dumper))
threads.append(threading.Thread(target=mutator))

with threading_helper.start_threads(threads):
pass

def test_pickle_dumps_with_concurrent_list_mutations(self):
# gh-149816: Pickling a list while another thread mutates it
# used to be a UAF in free-threaded mode. batch_list_exact()
# used PyList_GET_ITEM (borrowed) followed by Py_INCREF, and a
# concurrent replace/pop could free the item between those two
# operations.
shared = [list(range(20)) for _ in range(50)]

def dumper():
for _ in range(1000):
try:
pickle.dumps(shared)
except (RuntimeError, IndexError):
pass

def mutator():
for i in range(1000):
idx = i % 50
shared[idx] = list(range(i % 20))
if i % 10 == 0:
try:
shared.pop()
except IndexError:
pass
shared.append([i])

threads = []
for _ in range(10):
threads.append(threading.Thread(target=dumper))
threads.append(threading.Thread(target=mutator))

with threading_helper.start_threads(threads):
pass

if __name__ == "__main__":
unittest.main()
2 changes: 0 additions & 2 deletions Lib/test/test_lzma.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,6 @@ def test_decompressor_chunks_empty(self):
self.assertTrue(lzd.eof)
self.assertEqual(lzd.unused_data, b"")

@unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'LZMADecompressor' object has no attribute 'check'
def test_decompressor_chunks_maxsize(self):
lzd = LZMADecompressor()
max_length = 100
Expand Down Expand Up @@ -1349,7 +1348,6 @@ def test_tell_bad_args(self):
f.close()
self.assertRaises(ValueError, f.tell)

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: True is not false
def test_issue21872(self):
# sometimes decompress data incompletely

Expand Down
69 changes: 10 additions & 59 deletions Lib/test/test_pyrepl/test_unix_console.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,14 @@ def unix_console(events, **kwargs):
)


def __rustpython_patch_terminfo_tparm(s, *args):
return s + b":" + b",".join(str(i).encode() for i in args)


def __rustpython_patch_termios_tcgetattr(_):
return [
@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows")
@patch(
"_pyrepl.terminfo.tparm",
lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args),
)
@patch(
"termios.tcgetattr",
lambda _: [
27394,
3,
19200,
Expand Down Expand Up @@ -95,60 +97,9 @@ def __rustpython_patch_termios_tcgetattr(_):
b"\x14",
b"\x00",
],
]


def __rustpython_patch_termios_tcsetattr(a, b, c):
return None

@unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows")
@patch(
"_pyrepl.terminfo.tparm",
# TODO: RUSTPYTHON; SyntaxError: the symbol 's' must be present in the symbol table
# lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args),
__rustpython_patch_terminfo_tparm,
)
# TODO: RUSTPYTHON; SyntaxError: no symbol table available in lambda (type: Lambda)
# @patch(
# "termios.tcgetattr",
# lambda _: [
# 27394,
# 3,
# 19200,
# 536872399,
# 38400,
# 38400,
# [
# b"\x04",
# b"\xff",
# b"\xff",
# b"\x7f",
# b"\x17",
# b"\x15",
# b"\x12",
# b"\x00",
# b"\x03",
# b"\x1c",
# b"\x1a",
# b"\x19",
# b"\x11",
# b"\x13",
# b"\x16",
# b"\x0f",
# b"\x01",
# b"\x00",
# b"\x14",
# b"\x00",
# ],
# ],
# )
@patch("termios.tcgetattr", __rustpython_patch_termios_tcgetattr)
@patch(
"termios.tcsetattr",
# TODO: RUSTPYTHON; SyntaxError: no symbol table available in lambda (type: Lambda)
# lambda a, b, c: None
__rustpython_patch_termios_tcsetattr,
],
)
@patch("termios.tcsetattr", lambda a, b, c: None)
@patch("os.write")
@force_not_colorized_test_class
class TestConsole(TestCase):
Expand Down
Loading
Loading