Skip to content

Preserve ctypes element views and their references #9229

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
youknowone merged 6 commits into from
Oct 20, 2026
Merged

Preserve ctypes element views and their references #9229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4a82ab6
Preserve ctypes element views and their references
1ndahous3 Oct 19, 2026
cb3ec53
Fix ctypes reentrant ownership and scalar view handling
1ndahous3 Oct 19, 2026
d65ee31
Fix ctypes pointer view writes and indexed object lookup
1ndahous3 Oct 19, 2026
f23e581
Fix ctypes field byte order and encoded instance copies
1ndahous3 Oct 19, 2026
6dd75f6
Fix ctypes effective field types and overlapping pointer copies
1ndahous3 Oct 20, 2026
ea2c45e
Reject arrays for incomplete ctypes pointer fields
1ndahous3 Oct 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 129 additions & 120 deletions crates/vm/src/stdlib/_ctypes/array.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -545,23 +545,51 @@ impl PyCArray {
let index = index as usize;
let element_size = stg.as_ref().map_or(0, |i| i.element_size);
let offset = index * element_size;
if let Some(element_type) = stg.as_ref().and_then(|info| info.element_type.as_ref())
&& !super::simple::returns_scalar(element_type)
{
let length = element_type.stg_info(vm)?.length;
let buffer = zelf.0.buffer.read();
let Some(data) = buffer.get(offset..offset + element_size) else {
drop(buffer);
return Err(vm.new_index_error("invalid index"));
};
// SAFETY: this element lies within the array's live buffer. The view
// retains the array; ctypes users must still avoid reallocating or
// concurrently accessing the same native memory through aliases.
let view = unsafe {
PyCData::from_base_obj(
data.as_ptr().cast_mut(),
element_size,
zelf.to_owned().into(),
index,
)
};
view.base_offset.store(offset);
view.length.store(length);
drop(buffer);
return view.into_view(element_type.clone(), vm);
Comment thread
1ndahous3 marked this conversation as resolved.
Comment thread
1ndahous3 marked this conversation as resolved.
}
let type_code = Self::get_element_type_code(zelf, vm);

// Get target buffer and offset (base's buffer if available, otherwise own)
let base_obj = zelf.0.base.read().clone();
let (buffer_lock, final_offset) = if let Some(cdata) = base_obj
.as_ref()
.and_then(|b| b.downcast_ref::<super::PyCData>())
let buffer = zelf.0.buffer.read();
if type_code.as_deref() == Some("O") {
let address = rustpython_host_env::ctypes::read_pointer_from_buffer(
&buffer[offset..offset + element_size],
);
drop(buffer);
return zelf.0.object_at(address, Some(index), vm);
}
if let Some(element_type) = stg.as_ref().and_then(|info| info.element_type.as_ref())
&& super::simple::is_swapped(element_type)
{
(&cdata.buffer, zelf.0.base_offset.load() + offset)
} else {
(&zelf.0.buffer, offset)
};

let buffer = buffer_lock.read();
let mut bytes = buffer[offset..offset + element_size].to_vec();
bytes.reverse();
return Ok(super::base::bytes_to_pyobject(element_type, &bytes, vm));
}
Ok(Self::read_element_from_buffer(
&buffer,
final_offset,
offset,
element_size,
type_code.as_deref(),
vm,
Expand Down Expand Up @@ -589,10 +617,7 @@ impl PyCArray {
}

/// Helper to write an element value to a buffer at given offset
/// This is extracted to share code between direct write and base-buffer write
#[allow(clippy::too_many_arguments)]
fn write_element_to_buffer(
buffer: &mut [u8],
offset: usize,
element_size: usize,
type_code: Option<&str>,
Expand All @@ -601,38 +626,41 @@ impl PyCArray {
index: usize,
vm: &VirtualMachine,
) -> PyResult<()> {
// Conversions and reference replacement may call Python. Only the raw
// storage write runs under the buffer lock.
let write = |value| {
let mut buffer = zelf.0.buffer.write();
write_array_element(cow_bytes_as_mut(&mut buffer), offset, value);
};
match type_code {
Some("c") => {
if let Some(b) = value.downcast_ref::<PyBytes>() {
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Byte(b.as_bytes().first().copied().unwrap_or(0)),
);
write(ArrayElementWriteValue::Byte(
b.as_bytes().first().copied().unwrap_or(0),
));
} else if let Ok(int_val) = value.try_int(vm) {
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Byte(int_val.as_bigint().to_u8().unwrap_or(0)),
);
write(ArrayElementWriteValue::Byte(
int_val.as_bigint().to_u8().unwrap_or(0),
));
} else {
return Err(vm.new_type_error("an integer or bytes of length 1 is required"));
}
}
Some("u") => {
if let Some(s) = value.downcast_ref::<PyStr>() {
let code = s.as_wtf8().code_points().next().map_or(0, |c| c.to_u32());
write_array_element(buffer, offset, ArrayElementWriteValue::Wchar(code));
write(ArrayElementWriteValue::Wchar(code));
} else {
return Err(vm.new_type_error("unicode string expected"));
}
}
Some("z") => {
let mut hidden = None;
let (ptr_val, converted) = if value.is(&vm.ctx.none) {
(0usize, None)
} else if let Some(bytes) = value.downcast_ref::<PyBytes>() {
let (kept_alive, ptr) = super::base::ensure_z_null_terminated(bytes, vm);
zelf.0.keep_alive(index, kept_alive);
hidden = Some(kept_alive);
(ptr, Some(value.to_owned()))
} else if let Some(simple) = value.downcast_ref::<super::PyCSimple>()
&& value.class().type_code(vm).as_deref() == Some("z")
Expand All @@ -650,16 +678,12 @@ impl PyCArray {
value.class().name()
)));
};
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Pointer {
value: ptr_val,
size: element_size,
},
);
write(ArrayElementWriteValue::Pointer {
value: ptr_val,
size: element_size,
});
if let Some(c) = converted {
return zelf.0.keep_ref(index, c, vm);
return zelf.0.keep_ref_with_hidden(index, c, hidden, vm);
}
}
Some("Z") => {
Expand All @@ -681,18 +705,40 @@ impl PyCArray {
} else {
return Err(vm.new_type_error("unicode string or integer address expected"));
};
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Pointer {
value: ptr_val,
size: element_size,
},
);
write(ArrayElementWriteValue::Pointer {
value: ptr_val,
size: element_size,
});
if let Some(c) = converted {
return zelf.0.keep_ref(index, c, vm);
}
}
Some("O") => {
// A py_object instance contributes its stored reference, while
// other Python objects are the value itself.
if value.downcast_ref::<PyCData>().is_none() {
write(ArrayElementWriteValue::Pointer {
value: value.get_id(),
size: element_size,
});
return zelf.0.keep_ref(index, value.to_owned(), vm);
}
let stg = zelf.class().stg_info(vm)?;
let element_type = stg.element_type.as_ref().unwrap();
if !value.fast_isinstance(element_type) {
return Err(vm.new_type_error(format!(
"expected {} instance, not {}",
element_type.name(),
value.class().name()
)));
}
let cdata = value.downcast_ref::<PyCData>().unwrap();
let bytes = cdata.buffer.read().to_vec();
write(ArrayElementWriteValue::Bytes {
bytes: &bytes,
size: element_size,
});
}
Some("f") => {
// c_float: convert int/float to f32 bytes
let f32_val = if let Ok(float_val) = value.try_float(vm) {
Expand All @@ -702,14 +748,10 @@ impl PyCArray {
} else {
return Err(vm.new_type_error("a float is required"));
};
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Float {
value: f32_val.into(),
size: 4,
},
);
write(ArrayElementWriteValue::Float {
value: f32_val.into(),
size: 4,
});
}
Some("d" | "g") => {
// c_double / c_longdouble: convert int/float to f64 bytes
Expand All @@ -720,39 +762,27 @@ impl PyCArray {
} else {
return Err(vm.new_type_error("a float is required"));
};
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Float {
value: f64_val,
size: 8,
},
);
write(ArrayElementWriteValue::Float {
value: f64_val,
size: 8,
});
// For "g" type, remaining bytes stay zero
}
_ => {
// Handle ctypes instances (copy their buffer)
if let Some(cdata) = value.downcast_ref::<PyCData>() {
let src_buffer = cdata.buffer.read();
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Bytes {
bytes: &src_buffer,
size: element_size,
},
);
let src_buffer = cdata.buffer.read().to_vec();
write(ArrayElementWriteValue::Bytes {
bytes: &src_buffer,
size: element_size,
});
// Other types: use int_to_bytes
} else if let Ok(int_val) = value.try_int(vm) {
let bytes = Self::int_to_bytes(int_val.as_bigint(), element_size);
write_array_element(
buffer,
offset,
ArrayElementWriteValue::Bytes {
bytes: &bytes,
size: element_size,
},
);
write(ArrayElementWriteValue::Bytes {
bytes: &bytes,
size: element_size,
});
} else {
return Err(vm.new_type_error(format!(
"expected {} instance, not {}",
Expand All @@ -765,8 +795,7 @@ impl PyCArray {

// KeepRef
if super::base::PyCData::should_keep_ref(value) {
let to_keep = super::base::PyCData::get_kept_objects(value, vm);
zelf.0.keep_ref(index, to_keep, vm)?;
zelf.0.keep_cdata(index, value, vm)?;
}

Ok(())
Expand All @@ -789,49 +818,29 @@ impl PyCArray {
let offset = index * element_size;
let type_code = Self::get_element_type_code(zelf, vm);

// Get target buffer and offset (base's buffer if available, otherwise own)
let base_obj = zelf.0.base.read().clone();
let (buffer_lock, final_offset) = if let Some(cdata) = base_obj
.as_ref()
.and_then(|b| b.downcast_ref::<super::PyCData>())
if let Some(element_type) = stg.as_ref().and_then(|info| info.element_type.as_ref())
&& super::simple::is_swapped(element_type)
&& value.downcast_ref::<PyCData>().is_none()
{
(&cdata.buffer, zelf.0.base_offset.load() + offset)
} else {
(&zelf.0.buffer, offset)
};

let mut buffer = buffer_lock.write();

// For shared memory (Cow::Borrowed), we need to write directly to the memory
// For owned memory (Cow::Owned), we can write to the owned buffer
match &mut *buffer {
Cow::Borrowed(slice) => {
// SAFETY: For from_buffer, the slice points to writable shared memory.
// Python's from_buffer requires writable buffer, so this is safe.
let owned_slice =
unsafe { rustpython_host_env::ctypes::borrowed_slice_as_mut(slice) };
Self::write_element_to_buffer(
owned_slice,
final_offset,
element_size,
type_code.as_deref(),
value,
zelf,
index,
vm,
)
}
Cow::Owned(vec) => Self::write_element_to_buffer(
vec,
final_offset,
element_size,
type_code.as_deref(),
value,
zelf,
index,
vm,
),
let type_code = type_code
.as_deref()
.ok_or_else(|| vm.new_type_error("no _type_ attribute"))?;
let bytes = super::simple::swapped_value_bytes(type_code, value, vm)?;
zelf.0.write_bytes_at_offset(offset, &bytes);
return Ok(());
}

// A view's buffer already points to its element. Its base can be a
// pointer object, whose own buffer stores an address rather than data.
Self::write_element_to_buffer(
offset,
element_size,
type_code.as_deref(),
value,
zelf,
index,
vm,
)
}

// Array_subscript
Expand Down
Loading
Loading