Skip to content

Commit d501fa0

Browse files
daanxDinoV
daanx
authored and
DinoV
committed
Integrate mimalloc v3
1 parent ce841a9 commit d501fa0

12 files changed

Lines changed: 285 additions & 327 deletions

File tree

‎Include/internal/pycore_mimalloc.h‎

Lines changed: 20 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# error "this header requires Py_BUILD_CORE define"
66
#endif
77

8-
#if defined(MIMALLOC_H) || defined(MIMALLOC_TYPES_H)
8+
#if defined(MIMALLOC_H) || defined(MI_TYPES_H)
99
# error "pycore_mimalloc.h must be included before mimalloc.h"
1010
#endif
1111

@@ -20,9 +20,21 @@ typedef enum {
2020
#include "pycore_pymem.h"
2121

2222
#ifdef WITH_MIMALLOC
23+
# define MI_NO_PTHREAD_ATFORK 1
2324
# ifdef Py_GIL_DISABLED
2425
# define MI_PRIM_THREAD_ID _Py_ThreadId
25-
# define MI_VISIT_ABANDONED 1
26+
# define MI_CPYTHON 1
27+
28+
struct mi_page_qsbr_s {
29+
uint64_t goal;
30+
};
31+
32+
struct mi_heap_qsbr_s {
33+
struct _qsbr_shared *shared;
34+
};
35+
# else
36+
// Keep full pages in their owning thread's allocation statistics.
37+
# define MI_DEFAULT_PAGE_FULL_RETAIN -1
2638
# endif
2739
# define MI_DEBUG_UNINIT PYMEM_CLEANBYTE
2840
# define MI_DEBUG_FREED PYMEM_DEADBYTE
@@ -63,20 +75,17 @@ extern "C++" {
6375

6476
#ifdef Py_GIL_DISABLED
6577
struct _mimalloc_interp_state {
66-
// When exiting, threads abandon any segments with live blocks. The
67-
// abandoned segments are tracked per mimalloc sub-process so that other
68-
// threads of the same interpreter can claim and reuse them, and so that
69-
// the GC can visit them.
70-
mi_subproc_t *subproc;
78+
mi_heap_t *heaps[_Py_MIMALLOC_HEAP_COUNT];
7179
};
7280

7381
struct _mimalloc_thread_state {
74-
mi_heap_t *current_object_heap;
75-
mi_heap_t heaps[_Py_MIMALLOC_HEAP_COUNT];
76-
mi_tld_t tld;
82+
mi_theap_t *current_object_heap;
83+
mi_theap_t *heaps[_Py_MIMALLOC_HEAP_COUNT];
7784
int initialized;
78-
struct llist_node page_list;
7985
};
86+
87+
extern int _PyMem_InitMimalloc(PyInterpreterState *interp);
88+
extern void _PyMem_FiniMimalloc(PyInterpreterState *interp);
8089
#endif
8190

8291
#endif // Py_INTERNAL_MIMALLOC_H

‎Include/internal/pycore_object_alloc.h‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -14,18 +14,18 @@ extern "C" {
1414
#endif
1515

1616
#ifdef Py_GIL_DISABLED
17-
static inline mi_heap_t *
17+
static inline mi_theap_t *
1818
_PyObject_GetAllocationHeap(_PyThreadStateImpl *tstate, PyTypeObject *tp)
1919
{
2020
struct _mimalloc_thread_state *m = &tstate->mimalloc;
2121
if (_PyType_HasFeature(tp, Py_TPFLAGS_PREHEADER)) {
22-
return &m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
22+
return m->heaps[_Py_MIMALLOC_HEAP_GC_PRE];
2323
}
2424
else if (_PyType_IS_GC(tp)) {
25-
return &m->heaps[_Py_MIMALLOC_HEAP_GC];
25+
return m->heaps[_Py_MIMALLOC_HEAP_GC];
2626
}
2727
else {
28-
return &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
28+
return m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
2929
}
3030
}
3131
#endif
@@ -45,7 +45,7 @@ _PyObject_MallocWithType(PyTypeObject *tp, size_t size)
4545
#endif
4646
void *mem = PyObject_Malloc(size);
4747
#ifdef Py_GIL_DISABLED
48-
m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
48+
m->current_object_heap = m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
4949
#endif
5050
return mem;
5151
}
@@ -60,7 +60,7 @@ _PyObject_ReallocWithType(PyTypeObject *tp, void *ptr, size_t size)
6060
#endif
6161
void *mem = PyObject_Realloc(ptr, size);
6262
#ifdef Py_GIL_DISABLED
63-
m->current_object_heap = &m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
63+
m->current_object_heap = m->heaps[_Py_MIMALLOC_HEAP_OBJECT];
6464
#endif
6565
return mem;
6666
}

‎InternalDocs/qsbr.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,47 @@ by slightly delaying when memory can be reclaimed; the size-based thresholds
136136
above bound that extra memory.
137137

138138

139+
### Mimalloc Integration
140+
141+
Each interpreter owns four mimalloc heaps: one for `PyMem` allocations and
142+
three for objects with different header layouts. Only the object heaps enable
143+
QSBR. Each thread state retains a `mi_theap_t` handle for each heap. Clearing a
144+
thread state abandons its pages without invalidating live objects; heap
145+
visitation includes both owned and abandoned pages, so they remain visible to
146+
the garbage collector.
147+
148+
CPython supplies four typed hooks: `mi_heap_qsbr_init` associates a heap with its
149+
interpreter's QSBR state, `mi_page_qsbr_clear` and `mi_page_qsbr_is_clear` manage
150+
a page's goal, and `mi_page_qsbr_should_delay` establishes a goal if needed and
151+
checks cached read-sequence progress. These hooks may run without an attached
152+
Python thread state and while allocator locks are held. They must not poll the
153+
thread registry. `_PyMem_ProcessDelayed` polls outside those locks, then drains
154+
each object heap's thread-local delayed-page queue.
155+
156+
Mimalloc owns the delayed-page queues and page transitions. A delayed empty
157+
page can still supply allocations of its existing size class. If it becomes
158+
empty again, its previous goal is cleared and a new grace period is required.
159+
Queues are FIFO; an incomplete goal at the head may delay later pages. During
160+
stop-the-world collection, CPython also sweeps abandoned pages using the same
161+
cached QSBR progress.
162+
163+
The object's reference-count fields and any pre-header remain untouched by
164+
debug fills, using the heap's `debug_fill_preserve` setting. At interpreter
165+
shutdown, empty heaps can be destroyed, but live allocations may still be
166+
referenced elsewhere, including by a shared interned-string dictionary.
167+
Retained object heaps are disconnected from the interpreter's QSBR state and
168+
conservatively keep their pages rather than consulting a deleted domain.
169+
170+
CPython disables mimalloc's automatic `pthread_atfork` handler and invokes its
171+
child cleanup from `PyOS_AfterFork_Child`, after a coordinated fork. Exec-only
172+
subprocess forks do not stop other allocator users and must not perform that
173+
cleanup or use the allocator before exec.
174+
175+
The regular GIL build does not enable QSBR and retains full pages in their
176+
owning thread, as in mimalloc v2. Its allocation statistics visit only the
177+
current thread's pages, so early abandonment would make those counts incomplete.
178+
179+
139180
## Limitations
140181

141182
Determining the `rd_seq` requires scanning over all thread states. This operation

‎Lib/test/test_free_threading/test_gc.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,29 @@ class MyObj:
1717

1818
@threading_helper.requires_working_threading()
1919
class TestGC(TestCase):
20+
def test_collect_cycles_after_thread_exit(self):
21+
class Slotted:
22+
__slots__ = ("cycle", "__weakref__")
23+
24+
refs = []
25+
26+
def worker():
27+
for cls in (MyObj, Slotted):
28+
for _ in range(200):
29+
obj = cls()
30+
obj.cycle = obj
31+
refs.append(weakref.ref(obj))
32+
obj = Slotted()
33+
obj.cycle = (obj,) * 100_000
34+
refs.append(weakref.ref(obj))
35+
36+
with support.disable_gc():
37+
with threading_helper.start_threads([Thread(target=worker)]):
38+
pass
39+
self.assertTrue(all(ref() is not None for ref in refs))
40+
gc.collect()
41+
self.assertTrue(all(ref() is None for ref in refs))
42+
2043
def test_get_objects(self):
2144
event = threading.Event()
2245

‎Lib/test/test_free_threading/test_interpreters.py‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,22 @@
1010

1111

1212
class InterpreterTeardownTests(unittest.TestCase):
13+
def test_shared_interned_strings_survive_teardown(self):
14+
import_helper.import_module('_testcapi')
15+
script = textwrap.dedent("""
16+
import _testcapi
17+
import gc
18+
import sys
19+
20+
for i in range(10):
21+
code = f'import sys; sys.intern("retained_{i}_" + "x" * 5000)'
22+
assert _testcapi.run_in_subinterp(code) == 0
23+
gc.collect()
24+
expected = f"retained_{i}_" + "x" * 5000
25+
assert sys.intern(expected) == expected
26+
""")
27+
script_helper.assert_python_ok('-c', script)
28+
1329
def test_destroy_subinterpreter_does_not_abort(self):
1430
# gh-153176: destroy_interpreter(basic=True) used to call
1531
# PyThreadState_Clear() on a non-current thread state, which on a

‎Makefile.pre.in‎

Lines changed: 13 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -391,13 +391,9 @@ IO_OBJS= \
391391

392392
MIMALLOC_HEADERS= \
393393
$(srcdir)/Include/internal/pycore_mimalloc.h \
394-
$(srcdir)/Include/internal/mimalloc/mimalloc.h \
395-
$(srcdir)/Include/internal/mimalloc/mimalloc-stats.h \
396-
$(srcdir)/Include/internal/mimalloc/mimalloc/atomic.h \
397-
$(srcdir)/Include/internal/mimalloc/mimalloc/internal.h \
398-
$(srcdir)/Include/internal/mimalloc/mimalloc/prim.h \
399-
$(srcdir)/Include/internal/mimalloc/mimalloc/track.h \
400-
$(srcdir)/Include/internal/mimalloc/mimalloc/types.h
394+
$(wildcard $(srcdir)/Include/internal/mimalloc/*.h \
395+
$(srcdir)/Include/internal/mimalloc/mimalloc/*.h \
396+
$(srcdir)/Include/internal/mimalloc/mimalloc/*/*.h)
401397

402398

403399
##########################################################################
@@ -2149,30 +2145,11 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS)
21492145
Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h
21502146
Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
21512147

2152-
Objects/obmalloc.o: $(srcdir)/Objects/mimalloc/alloc.c \
2153-
$(srcdir)/Objects/mimalloc/alloc-aligned.c \
2154-
$(srcdir)/Objects/mimalloc/alloc-posix.c \
2155-
$(srcdir)/Objects/mimalloc/arena.c \
2156-
$(srcdir)/Objects/mimalloc/arena-abandon.c \
2157-
$(srcdir)/Objects/mimalloc/bitmap.c \
2158-
$(srcdir)/Objects/mimalloc/free.c \
2159-
$(srcdir)/Objects/mimalloc/heap.c \
2160-
$(srcdir)/Objects/mimalloc/init.c \
2161-
$(srcdir)/Objects/mimalloc/libc.c \
2162-
$(srcdir)/Objects/mimalloc/options.c \
2163-
$(srcdir)/Objects/mimalloc/os.c \
2164-
$(srcdir)/Objects/mimalloc/page.c \
2165-
$(srcdir)/Objects/mimalloc/random.c \
2166-
$(srcdir)/Objects/mimalloc/segment.c \
2167-
$(srcdir)/Objects/mimalloc/segment-map.c \
2168-
$(srcdir)/Objects/mimalloc/stats.c \
2169-
$(srcdir)/Objects/mimalloc/prim/prim.c \
2170-
$(srcdir)/Objects/mimalloc/prim/emscripten/prim.c \
2171-
$(srcdir)/Objects/mimalloc/prim/osx/prim.c \
2172-
$(srcdir)/Objects/mimalloc/prim/unix/prim.c \
2173-
$(srcdir)/Objects/mimalloc/prim/wasi/prim.c
2174-
2175-
Objects/mimalloc/page.o: $(srcdir)/Objects/mimalloc/page-queue.c
2148+
Objects/obmalloc.o: $(wildcard $(srcdir)/Objects/mimalloc/*.c \
2149+
$(srcdir)/Objects/mimalloc/*/*.c \
2150+
$(srcdir)/Objects/mimalloc/*/*/*.c \
2151+
$(srcdir)/Objects/mimalloc/*/*.h \
2152+
$(srcdir)/Objects/mimalloc/*/*/*.h)
21762153

21772154

21782155
# Regenerate various files from Python/bytecodes.c
@@ -3002,15 +2979,12 @@ inclinstall:
30022979
$(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal; \
30032980
done
30042981
@if test "$(INSTALL_MIMALLOC)" = "yes"; then \
3005-
for i in $(srcdir)/Include/internal/mimalloc/*.h; \
2982+
for i in $(filter-out $(srcdir)/Include/internal/pycore_mimalloc.h,$(MIMALLOC_HEADERS)); \
30062983
do \
3007-
echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal/mimalloc; \
3008-
$(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal/mimalloc; \
3009-
done; \
3010-
for i in $(srcdir)/Include/internal/mimalloc/mimalloc/*.h; \
3011-
do \
3012-
echo $(INSTALL_DATA) $$i $(INCLUDEPY)/internal/mimalloc/mimalloc; \
3013-
$(INSTALL_DATA) $$i $(DESTDIR)$(INCLUDEPY)/internal/mimalloc/mimalloc; \
2984+
dest=$(DESTDIR)$(INCLUDEPY)/$${i#$(srcdir)/Include/}; \
2985+
$(INSTALL) -d -m $(DIRMODE) "$$(dirname "$$dest")"; \
2986+
echo $(INSTALL_DATA) $$i $$dest; \
2987+
$(INSTALL_DATA) $$i $$dest; \
30142988
done; \
30152989
fi
30162990
echo $(INSTALL_DATA) pyconfig.h $(DESTDIR)$(CONFINCLUDEPY)/pyconfig.h

‎Modules/posixmodule.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -753,6 +753,10 @@ PyOS_AfterFork_Child(void)
753753
PyStatus status;
754754
_PyRuntimeState *runtime = &_PyRuntime;
755755

756+
#ifdef WITH_MIMALLOC
757+
_mi_fork_child();
758+
#endif
759+
756760
// re-creates runtime->interpreters.mutex (HEAD_UNLOCK)
757761
status = _PyRuntimeState_ReInitThreads(runtime);
758762
if (_PyStatus_EXCEPTION(status)) {

0 commit comments

Comments
 (0)