diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 684c7a5b614..a9fbc8f4318 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -154,6 +154,7 @@ prec preinitialized pybuilddir pycore +pyinner pydecimal Pyfunc pylifecycle diff --git a/crates/vm/src/builtins/complex.rs b/crates/vm/src/builtins/complex.rs index c36024dcce3..182962a4b2e 100644 --- a/crates/vm/src/builtins/complex.rs +++ b/crates/vm/src/builtins/complex.rs @@ -58,7 +58,7 @@ impl PyPayload for PyComplex { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { COMPLEX_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/dict.rs b/crates/vm/src/builtins/dict.rs index f2a7e6a5a29..0e64e9e66ac 100644 --- a/crates/vm/src/builtins/dict.rs +++ b/crates/vm/src/builtins/dict.rs @@ -93,7 +93,7 @@ impl PyPayload for PyDict { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { DICT_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/float.rs b/crates/vm/src/builtins/float.rs index 4eacfaf45b3..eeddd6b2eb9 100644 --- a/crates/vm/src/builtins/float.rs +++ b/crates/vm/src/builtins/float.rs @@ -65,7 +65,7 @@ impl PyPayload for PyFloat { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { FLOAT_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/int.rs b/crates/vm/src/builtins/int.rs index d5bb3cbecd7..a253506eba1 100644 --- a/crates/vm/src/builtins/int.rs +++ b/crates/vm/src/builtins/int.rs @@ -86,7 +86,7 @@ impl PyPayload for PyInt { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { INT_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/list.rs b/crates/vm/src/builtins/list.rs index 34c40bba209..cdb8a73ead2 100644 --- a/crates/vm/src/builtins/list.rs +++ b/crates/vm/src/builtins/list.rs @@ -105,7 +105,7 @@ impl PyPayload for PyList { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { LIST_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/range.rs b/crates/vm/src/builtins/range.rs index 795ec230ba9..153a82bb43b 100644 --- a/crates/vm/src/builtins/range.rs +++ b/crates/vm/src/builtins/range.rs @@ -101,7 +101,7 @@ impl PyPayload for PyRange { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { RANGE_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/slice.rs b/crates/vm/src/builtins/slice.rs index aeb3337c7d8..b46f7a3a56a 100644 --- a/crates/vm/src/builtins/slice.rs +++ b/crates/vm/src/builtins/slice.rs @@ -76,7 +76,7 @@ impl PyPayload for PySlice { } #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { SLICE_FREELIST .try_with(|fl| { let mut list = fl.take(); diff --git a/crates/vm/src/builtins/tuple.rs b/crates/vm/src/builtins/tuple.rs index 03f88f1b5fe..e1dc1ef306b 100644 --- a/crates/vm/src/builtins/tuple.rs +++ b/crates/vm/src/builtins/tuple.rs @@ -27,6 +27,8 @@ use crate::{ vm::VirtualMachine, }; use alloc::fmt; +use core::cell::Cell; +use core::ptr::NonNull; #[pyclass(module = false, name = "tuple", traverse = "manual")] pub struct PyTuple { @@ -53,14 +55,95 @@ unsafe impl Traverse for PyTuple { } } -// No freelist for PyTuple: structseq types (stat_result, struct_time, etc.) -// are static subtypes sharing the same Rust payload, making type-safe reuse -// impractical without a type-pointer comparison at push time. +// spell-checker:ignore MAXSAVESIZE +/// Per-size freelist storage for tuples, matching tuples[PyTuple_MAXSAVESIZE]. +/// Each bucket caches tuples of a specific element count (index = len - 1). +struct TupleFreeList { + buckets: [Vec>; Self::MAX_SAVE_SIZE], +} + +impl TupleFreeList { + /// Largest tuple size to cache on the freelist (sizes 1..=20). + const MAX_SAVE_SIZE: usize = 20; + const fn new() -> Self { + Self { + buckets: [const { Vec::new() }; Self::MAX_SAVE_SIZE], + } + } +} + +impl Default for TupleFreeList { + fn default() -> Self { + Self::new() + } +} + +impl Drop for TupleFreeList { + fn drop(&mut self) { + // Same safety pattern as FreeList::drop — free raw allocation + // without running payload destructors to avoid TLS-after-destruction panics. + let layout = crate::object::pyinner_layout::(); + for bucket in &mut self.buckets { + for ptr in bucket.drain(..) { + unsafe { + alloc::alloc::dealloc(ptr.as_ptr() as *mut u8, layout); + } + } + } + } +} + +thread_local! { + static TUPLE_FREELIST: Cell = const { Cell::new(TupleFreeList::new()) }; +} + impl PyPayload for PyTuple { + const MAX_FREELIST: usize = 2000; + const HAS_FREELIST: bool = true; + #[inline] fn class(ctx: &Context) -> &'static Py { ctx.types.tuple_type } + + #[inline] + unsafe fn freelist_push(obj: *mut PyObject) -> bool { + let len = unsafe { &*(obj as *const crate::Py) }.elements.len(); + if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { + return false; + } + TUPLE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let bucket = &mut list.buckets[len - 1]; + let stored = if bucket.len() < Self::MAX_FREELIST { + bucket.push(unsafe { NonNull::new_unchecked(obj) }); + true + } else { + false + }; + fl.set(list); + stored + }) + .unwrap_or(false) + } + + #[inline] + unsafe fn freelist_pop(payload: &Self) -> Option> { + let len = payload.elements.len(); + if len == 0 || len > TupleFreeList::MAX_SAVE_SIZE { + return None; + } + TUPLE_FREELIST + .try_with(|fl| { + let mut list = fl.take(); + let result = list.buckets[len - 1].pop(); + fl.set(list); + result + }) + .ok() + .flatten() + } } pub trait IntoPyTuple { diff --git a/crates/vm/src/object/core.rs b/crates/vm/src/object/core.rs index e31470fd12e..feaf0508686 100644 --- a/crates/vm/src/object/core.rs +++ b/crates/vm/src/object/core.rs @@ -188,27 +188,32 @@ pub(super) unsafe fn default_dealloc(obj: *mut PyObject) { ); } - // Extract child references before deallocation to break circular refs (tp_clear) + // Try to store in freelist for reuse BEFORE tp_clear, so that + // size-based freelists (e.g. PyTuple) can read the payload directly. + // Only exact base types (not heaptype or structseq subtypes) go into the freelist. + let typ = obj_ref.class(); + let pushed = if T::HAS_FREELIST + && typ.heaptype_ext.is_none() + && core::ptr::eq(typ, T::class(crate::vm::Context::genesis())) + { + unsafe { T::freelist_push(obj) } + } else { + false + }; + + // Extract child references to break circular refs (tp_clear). + // This runs regardless of freelist push — the object's children must be released. let mut edges = Vec::new(); if let Some(clear_fn) = vtable.clear { unsafe { clear_fn(obj, &mut edges) }; } - // Try to store in freelist for reuse; otherwise deallocate. - // Only exact types (not heaptype subclasses) go into the freelist, - // because the pop site assumes the cached typ matches the base type. - let pushed = if T::HAS_FREELIST && obj_ref.class().heaptype_ext.is_none() { - unsafe { T::freelist_push(obj) } - } else { - false - }; if !pushed { // Deallocate the object memory (handles ObjExt prefix if present) unsafe { PyInner::dealloc(obj as *mut PyInner) }; } // Drop child references - may trigger recursive destruction. - // The object is already deallocated, so circular refs are broken. drop(edges); // Trashcan: decrement depth and process deferred objects at outermost level @@ -1094,6 +1099,11 @@ impl PyInner { } } +/// Returns the allocation layout for `PyInner`, for use in freelist Drop impls. +pub(crate) const fn pyinner_layout() -> core::alloc::Layout { + core::alloc::Layout::new::>() +} + /// Thread-local freelist storage for reusing object allocations. /// /// Wraps a `Vec<*mut PyObject>`. On thread teardown, `Drop` frees raw @@ -2175,9 +2185,9 @@ impl PyRef { let has_dict = dict.is_some(); let is_heaptype = typ.heaptype_ext.is_some(); - // Try to reuse from freelist (exact type only, no dict, no heaptype) + // Try to reuse from freelist (no dict, no heaptype) let cached = if !has_dict && !is_heaptype { - unsafe { T::freelist_pop() } + unsafe { T::freelist_pop(&payload) } } else { None }; @@ -2189,11 +2199,16 @@ impl PyRef { (*inner).gc_bits.store(0, Ordering::Relaxed); core::ptr::drop_in_place(&mut (*inner).payload); core::ptr::write(&mut (*inner).payload, payload); - // typ, vtable, slots are preserved; dict is None, weak_list was - // cleared by drop_slow_inner before freelist push + // Freelist only stores exact base types (push-side filter), + // but subtypes sharing the same Rust payload (e.g. structseq) + // may pop entries. Update typ if it differs. + let cached_typ: *const Py = &*(*inner).typ; + if core::ptr::eq(cached_typ, &*typ) { + drop(typ); + } else { + let _old = (*inner).typ.swap(typ); + } } - // Drop the caller's typ since the cached object already holds one - drop(typ); unsafe { NonNull::new_unchecked(inner.cast::>()) } } else { let inner = PyInner::new(payload, typ, dict); diff --git a/crates/vm/src/object/payload.rs b/crates/vm/src/object/payload.rs index 1af954505f7..a615123c680 100644 --- a/crates/vm/src/object/payload.rs +++ b/crates/vm/src/object/payload.rs @@ -57,10 +57,11 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// Try to push a dead object onto this type's freelist for reuse. /// Returns true if the object was stored (caller must NOT free the memory). + /// Called before tp_clear, so the payload is still intact. /// /// # Safety - /// `obj` must be a valid pointer to a `PyInner` with refcount 0, - /// after `drop_slow_inner` and `tp_clear` have already run. + /// `obj` must be a valid pointer to a `PyInner` with refcount 0. + /// The payload is still initialized and can be read for bucket selection. #[inline] unsafe fn freelist_push(_obj: *mut PyObject) -> bool { false @@ -75,7 +76,7 @@ pub trait PyPayload: MaybeTraverse + PyThreadingConstraint + Sized + 'static { /// whose payload is still initialized from a previous allocation. The caller /// will drop and overwrite `payload` before reuse. #[inline] - unsafe fn freelist_pop() -> Option> { + unsafe fn freelist_pop(_payload: &Self) -> Option> { None }