Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
85 changes: 68 additions & 17 deletions crates/stdlib/src/pickle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ mod _pickle {
PyInt, PyList, PySet, PyStr, PyTuple, PyTupleRef, PyType, PyTypeRef,
},
function::{FuncArgs, OptionalArg, PySetterValue},
object::{Traverse, TraverseFn},
protocol::{PyBuffer, PyIter, PyIterReturn},
types::{AsBuffer, Constructor, Initializer, Representable},
};
Expand Down Expand Up @@ -250,6 +251,15 @@ mod _pickle {
frame_end: Option<usize>,
}

// SAFETY: these are all the Python references owned by the input state.
unsafe impl Traverse for ReadState {
fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) {
self.read.traverse(traverse_fn);
self.readline.traverse(traverse_fn);
self.peek.traverse(traverse_fn);
}
}

fn to_byte_vec(obj: &PyObject, what: &str, vm: &VirtualMachine) -> PyResult<Vec<u8>> {
let bytes = obj
.downcast_ref::<PyBytes>()
Expand Down Expand Up @@ -418,7 +428,7 @@ mod _pickle {
// memo proxies

#[pyattr]
#[pyclass(module = "_pickle", name = "UnpicklerMemoProxy")]
#[pyclass(module = "_pickle", name = "UnpicklerMemoProxy", traverse)]
#[derive(Debug, PyPayload)]
struct UnpicklerMemoProxy {
unpickler: PyRef<PyUnpickler>,
Expand Down Expand Up @@ -484,6 +494,13 @@ mod _pickle {
buffers: Option<PyObjectRef>,
}

// SAFETY: buffers is the only Python reference owned by the configuration.
unsafe impl Traverse for UnpicklerConfig {
fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) {
self.buffers.traverse(traverse_fn);
}
}

impl Default for UnpicklerConfig {
fn default() -> Self {
Self {
Expand All @@ -498,14 +515,42 @@ mod _pickle {
}

#[pyattr]
#[pyclass(module = "_pickle", name = "Unpickler")]
#[pyclass(module = "_pickle", name = "Unpickler", traverse = "manual")]
#[derive(Debug, PyPayload)]
pub(super) struct PyUnpickler {
read_state: PyMutex<ReadState>,
memo: PyRwLock<UnpicklerMemo>,
config: PyRwLock<UnpicklerConfig>,
}

// SAFETY: visit each owned reference once, including sparse memo entries.
unsafe impl Traverse for PyUnpickler {
fn traverse(&self, traverse_fn: &mut TraverseFn<'_>) {
self.read_state.traverse(traverse_fn);
self.config.traverse(traverse_fn);
if let Some(memo) = self.memo.try_read_recursive() {
#[expect(
clippy::iter_over_hash_type,
reason = "GC tracing visits every memo reference regardless of index order"
)]
for value in memo.values() {
value.traverse(traverse_fn);
}
}
}

fn clear(&mut self, out: &mut Vec<PyObjectRef>) {
let state = self.read_state.get_mut();
out.extend(state.read.take());
out.extend(state.readline.take());
out.extend(state.peek.take());
out.extend(self.memo.get_mut().drain().map(|(_, value)| value));
let config = self.config.get_mut();
out.extend(config.buffers.take());
config.initialized = false;
}
}

#[derive(FromArgs)]
pub(super) struct UnpicklerNewArgs {
#[pyarg(any)]
Expand Down Expand Up @@ -576,23 +621,29 @@ mod _pickle {
return Err(vm.new_type_error("file must have 'read' and 'readline' attributes"));
}

*state = ReadState {
read,
readline,
peek,
..ReadState::default()
};
let old_state = core::mem::replace(
&mut *state,
ReadState {
read,
readline,
peek,
..ReadState::default()
},
);
let old_memo = core::mem::take(&mut *zelf.memo.write());
*zelf.config.write() = UnpicklerConfig {
initialized: true,
proto: 0,
fix_imports: args.fix_imports,
encoding,
errors,
buffers,
};
let old_config = core::mem::replace(
&mut *zelf.config.write(),
UnpicklerConfig {
initialized: true,
proto: 0,
fix_imports: args.fix_imports,
encoding,
errors,
buffers,
},
);
drop(state);
drop(old_memo);
drop((old_state, old_memo, old_config));
Ok(())
}
}
Expand Down
41 changes: 41 additions & 0 deletions extra_tests/snippets/stdlib_pickle.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
import _pickle
import gc
import io
import sys
import weakref
from types import MappingProxyType, MethodType

from testutils import assert_raises
Expand Down Expand Up @@ -116,3 +118,42 @@ def __name__(self):
with assert_raises(RuntimeError) as caught:
exec("method.__reduce__()", namespace)
assert caught.exception is name_error


# Unpickler payloads can retain a cycle through their memo proxy.
class UnpicklerCycleStream:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Remove newly added test logic

This newly added regression block changes test logic in an existing test file, but the repository's test-modification policy says test logic, data, and assertions must never be modified and limits acceptable test-file changes to adding or removing expectedFailure decorators; revert this block rather than landing an unsupported test-file edit.

AGENTS.md reference: AGENTS.md:L278-L282

Useful? React with 👍 / 👎.

def __init__(self):
self.source = io.BytesIO(_pickle.dumps([1, 2, 3], protocol=4))
self.read = self.source.read
self.readline = self.source.readline
self.peek = self.peek_bytes

def peek_bytes(self, size=0):
position = self.source.tell()
data = self.source.read(size)
self.source.seek(position)
return data


class UnpicklerCycleBuffers:
def __iter__(self):
return self

def __next__(self):
raise StopIteration


def unpickler_payload_cycle(kind):
stream = UnpicklerCycleStream()
buffers = UnpicklerCycleBuffers()
unpickler = _pickle.Unpickler(stream, buffers=buffers)
holder = stream if kind == "stream" else buffers
holder.proxy = unpickler.memo
return weakref.ref(holder)


for kind in ("stream", "buffers"):

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Run the required validation suites

For this interpreter change and the added extra_tests/snippets code, the commit's validation explicitly says the workspace/C-API suites and full pytest gate were not run; repository policy requires the workspace test command, C-API cargo test, and pytest -v in extra_tests (plus relevant interpreter-behavior tests) before completion, so run those gates before landing.

AGENTS.md reference: AGENTS.md:L35-L35

Useful? React with 👍 / 👎.

reference = unpickler_payload_cycle(kind)
for _ in range(3):
gc.collect()
assert reference() is None, kind
Loading