update CLAUDE.md - implementation details

Author: sumerc

CLAUDE.md

@@ -36,6 +36,50 @@ tests/               — All tests (unittest)
   utils.py               — Shared helpers and base test class
 ```
 
+## How it works
+
+### Hook mechanism
+`yappi.start()` calls `sys.setprofile(_profile_thread_callback)` on the current thread. When a new thread is encountered, yappi transplants its own `c_profilefunc` into the new thread's `PyThreadState` — this is how it profiles all threads without each one needing an explicit `setprofile` call.
+
+Every `call`/`return`/`c_call`/`c_return` event from the interpreter is delivered to `_yappi._profile_event()` (C function).
+
+### Core data structures (C layer)
+```
+contexts (global htab)
+  └── context_id → _ctx
+        ├── cs         — call stack (_cstack), tracks the current call chain
+        ├── rec_levels — htab tracking recursion depth per function
+        ├── t0         — profiling start tick
+        ├── sched_cnt  — how many times this thread was scheduled
+        └── tags (htab)
+              └── tag_id → pits (htab)
+                    └── code_obj / m_ml → _pit (profile item)
+                          ├── callcount, nonrecursive_callcount
+                          ├── ttotal   — total time including children
+                          ├── tsubtotal — self time (excluding children)
+                          ├── children — linked list of _pit_children_info (callee timing per caller-callee pair)
+                          └── coroutines — linked list of _coro (active coroutine frames + start tick)
+```
+
+### Contexts
+A *context* maps to a thread by default. Context identity is stored as `_yappi_tid` in `ThreadState.dict` — a monotonic counter rather than the OS tid (which can be recycled). This design allows alternative context backends: for **greenlets**, a `context_id_callback` returns a per-greenlet ID so multiple greenlets sharing one OS thread appear as separate contexts.
+
+### Tags
+An optional `tag_callback` returns an integer per call event. Stats are bucketed per `(context, tag)`, allowing you to segregate profiling data (e.g. by request, task type, etc.) without separate profiling sessions.
+
+### Coroutines
+Each `_pit` (function) holds a linked list of `_coro` entries — one per concurrently suspended coroutine frame. When a coroutine is suspended (`FRAME_SUSPENDED`), its elapsed time is accumulated into the `_coro` entry without closing the `_pit`. On resumption, timing continues from where it left off. This correctly handles multiple concurrent coroutines calling the same function.
+
+### Stat collection (Python layer)
+`get_func_stats()` / `get_thread_stats()` enumerate the C-side hash tables and materialize them as Python objects:
+
+| C struct | Python wrapper | Collection |
+|----------|---------------|------------|
+| `_pit`   | `YFuncStat`   | `YFuncStats` |
+| `_ctx`   | `YThreadStat` | `YThreadStats` |
+
+`YFuncStat.children` is a `YChildFuncStats` collection (from `_pit_children_info`) representing direct callees with per-pair timing. Export formats (callgrind, pstat) are produced by converting these collections in Python.
+
 ## Key constraints
 - **Don't assume GIL protection in callbacks**: profiler callbacks can fire on any thread; C code must be thread-safe
 - **clear_stats() sequence**: pause → wait for in-flight callbacks → clear. Never free memory while callbacks may still be running

tests/test_functionality.py

@@ -1441,6 +1441,40 @@ def worker():
         t1.join()
         yappi.stop()
 
+    def test_clear_stats_race(self):
+        # Regression test for issue #188: SIGSEGV when clear_stats() races
+        # with a tag_callback that releases the GIL (e.g. via time.sleep).
+        stop = threading.Event()
+
+        def _tag_cbk():
+            time.sleep(0.001)
+            return 1
+
+        def _worker():
+            def a():
+                pass
+            while not stop.is_set():
+                a()
+
+        def _clearer():
+            while not stop.is_set():
+                yappi.clear_stats()
+                time.sleep(0.001)
+
+        yappi.set_tag_callback(_tag_cbk)
+        yappi.start()
+
+        worker = threading.Thread(target=_worker)
+        clearer = threading.Thread(target=_clearer)
+        worker.start()
+        clearer.start()
+
+        time.sleep(2)
+        stop.set()
+        worker.join(timeout=2)
+        clearer.join(timeout=2)
+        yappi.stop()
+
 
 class NonRecursiveFunctions(utils.YappiUnitTestCase):
 

yappi/_yappi.c

@@ -166,6 +166,7 @@ static long long ycurthreadindex = 0;
 static int yapphavestats;   // start() called at least once or stats cleared?
 static int yapprunning;
 static int paused;
+static volatile int _callback_depth = 0;
 static time_t yappstarttime;
 static long long yappstarttick;
 static long long yappstoptick;
@@ -335,7 +336,9 @@ _call_funcobjargs(PyObject *func, PyObject *args)
 
     _local_current_ctx = current_ctx;
     _local_prev_ctx = prev_ctx;
+    _callback_depth++;
     result = PyObject_CallFunctionObjArgs(func, args);
+    _callback_depth--;
     current_ctx = _local_current_ctx;
     prev_ctx = _local_prev_ctx;
 
@@ -2164,6 +2167,16 @@ _resume(PyObject *self, PyObject *args)
     Py_RETURN_NONE;
 }
 
+static PyObject*
+_wait_for_callbacks(PyObject *self, PyObject *args)
+{
+    Py_BEGIN_ALLOW_THREADS
+    while (_callback_depth > 0)
+        ;
+    Py_END_ALLOW_THREADS
+    Py_RETURN_NONE;
+}
+
 static PyMethodDef yappi_methods[] = {
     {"start", start, METH_VARARGS, NULL},
     {"stop", stop, METH_NOARGS, NULL},
@@ -2187,6 +2200,7 @@ static PyMethodDef yappi_methods[] = {
     {"_profile_event", profile_event, METH_VARARGS, NULL}, // for internal usage.
     {"_pause", _pause, METH_VARARGS, NULL}, // for internal usage.
     {"_resume", _resume, METH_VARARGS, NULL}, // for internal usage.
+    {"_wait_for_callbacks", _wait_for_callbacks, METH_NOARGS, NULL}, // for internal usage.
     {NULL, NULL, 0, NULL}      /* sentinel */
 };
 

yappi/yappi.py

@@ -1317,6 +1317,7 @@ def clear_stats():
     Clears all of the profile results.
     """
     _yappi._pause()
+    _yappi._wait_for_callbacks()
     try:
         _yappi.clear_stats()
     finally: