Skip to content

Commit 9b2f2fa

Browse files
henryiiihenryiii
committed
feat: expose the precompiled-mode sources to non-CMake builds
Adds pybind11.get_source_dir() / python -m pybind11 --srcdir, a srcdir variable in pybind11.pc, and Pybind11Extension(precompile=True) which compiles src/pybind11_combined.cpp into the extension and defines PYBIND11_PRECOMPILED (hard error if the sources are missing). Assisted-by: ClaudeCode:claude-fable-5
1 parent ee4122a commit 9b2f2fa

9 files changed

Lines changed: 181 additions & 63 deletions

File tree

‎CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -410,6 +410,7 @@ if(PYBIND11_INSTALL)
410410
endif()
411411
endif()
412412
join_paths(includedir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_INCLUDEDIR}")
413+
join_paths(srcdir_for_pc_file "\${prefix}" "${CMAKE_INSTALL_DATAROOTDIR}/pybind11/src")
413414
configure_file("${CMAKE_CURRENT_SOURCE_DIR}/tools/pybind11.pc.in"
414415
"${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc" @ONLY)
415416
install(FILES "${CMAKE_CURRENT_BINARY_DIR}/pybind11.pc"

‎include/pybind11/pybind11-inl.h‎

Lines changed: 50 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -60,10 +60,10 @@ PYBIND11_INLINE std::string replace_newlines_and_squash(const char *text) {
6060
}
6161

6262
PYBIND11_INLINE std::string generate_function_signature(const char *type_caster_name_field,
63-
detail::function_record *func_rec,
64-
const std::type_info *const *types,
65-
size_t &type_index,
66-
size_t &arg_index) {
63+
detail::function_record *func_rec,
64+
const std::type_info *const *types,
65+
size_t &type_index,
66+
size_t &arg_index) {
6767
std::string signature;
6868
bool is_starred = false;
6969
// `is_return_value.top()` is true if we are currently inside the return type of the
@@ -440,8 +440,9 @@ PYBIND11_INLINE const char *error_already_set::what() const noexcept {
440440
}
441441

442442
PYBIND11_NAMESPACE_BEGIN(detail)
443-
PYBIND11_INLINE function
444-
get_type_override(const void *this_ptr, const type_info *this_type, const char *name) {
443+
PYBIND11_INLINE function get_type_override(const void *this_ptr,
444+
const type_info *this_type,
445+
const char *name) {
445446
handle self = get_object_handle(this_ptr, this_type);
446447
if (!self) {
447448
return function();
@@ -542,11 +543,10 @@ cpp_function::make_function_record() {
542543
return unique_function_record(new detail::function_record());
543544
}
544545

545-
546546
PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&unique_rec,
547-
const char *text,
548-
const std::type_info *const *types,
549-
size_t args) {
547+
const char *text,
548+
const std::type_info *const *types,
549+
size_t args) {
550550
// Do NOT receive `unique_rec` by value. If this function fails to move out the unique_ptr,
551551
// we do not want this to destruct the pointer. `initialize` (the caller) still relies on
552552
// the pointee being alive after this call. Only move out if a `capsule` is going to keep
@@ -619,8 +619,7 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u
619619
if (PyCFunction_Check(rec->sibling.ptr())) {
620620
auto *self = PyCFunction_GET_SELF(rec->sibling.ptr());
621621
if (self == nullptr) {
622-
pybind11_fail(
623-
"initialize_generic: Unexpected nullptr from PyCFunction_GET_SELF");
622+
pybind11_fail("initialize_generic: Unexpected nullptr from PyCFunction_GET_SELF");
624623
}
625624
chain = detail::function_record_ptr_from_PyObject(self);
626625
if (chain && !chain->scope.is(rec->scope)) {
@@ -758,7 +757,6 @@ PYBIND11_INLINE void cpp_function::initialize_generic(unique_function_record &&u
758757
}
759758
}
760759

761-
762760
PYBIND11_INLINE void cpp_function::destruct(detail::function_record *rec, bool free_strings) {
763761
// If on Python 3.9, check the interpreter "MICRO" (patch) version.
764762
// If this is running on 3.9.0, we have to work around a bug.
@@ -804,7 +802,6 @@ PYBIND11_INLINE void cpp_function::destruct(detail::function_record *rec, bool f
804802
}
805803
}
806804

807-
808805
PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self,
809806
PyObject *const *args_in_arr,
810807
size_t nargsf,
@@ -820,8 +817,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self,
820817
overload */
821818
const auto n_args_in = static_cast<size_t>(PyVectorcall_NARGS(nargsf));
822819

823-
handle parent = n_args_in > 0 ? args_in_arr[0] : nullptr,
824-
result = PYBIND11_TRY_NEXT_OVERLOAD;
820+
handle parent = n_args_in > 0 ? args_in_arr[0] : nullptr, result = PYBIND11_TRY_NEXT_OVERLOAD;
825821

826822
auto self_value_and_holder = value_and_holder();
827823
if (overloads->is_constructor) {
@@ -852,8 +848,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self,
852848
std::vector<function_call> second_pass;
853849

854850
// However, if there are no overloads, we can just skip the no-convert pass entirely
855-
const bool overloaded
856-
= current_overload != nullptr && current_overload->next != nullptr;
851+
const bool overloaded = current_overload != nullptr && current_overload->next != nullptr;
857852

858853
for (; current_overload != nullptr; current_overload = current_overload->next) {
859854

@@ -1069,8 +1064,7 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self,
10691064
// set of all-false flags. If the call fails, we'll swap the flags back in for
10701065
// the conversion-allowed call below.
10711066
second_pass_convert = std::move(call.args_convert);
1072-
call.args_convert
1073-
= args_convert_vector<arg_vector_small_size>(func.nargs, false);
1067+
call.args_convert = args_convert_vector<arg_vector_small_size>(func.nargs, false);
10741068
}
10751069

10761070
// 6. Call the function.
@@ -1252,7 +1246,6 @@ PYBIND11_INLINE PyObject *cpp_function::dispatcher(PyObject *self,
12521246
return result.ptr();
12531247
}
12541248

1255-
12561249
PYBIND11_NAMESPACE_BEGIN(detail)
12571250

12581251
PYBIND11_INLINE void generic_type::initialize(const type_record &rec) {
@@ -1315,8 +1308,7 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) {
13151308
mark_parents_nonsimple(tinfo->type);
13161309
tinfo->simple_ancestors = false;
13171310
} else if (rec.bases.size() == 1) {
1318-
auto *parent_tinfo
1319-
= get_type_info(reinterpret_cast<PyTypeObject *>(rec.bases[0].ptr()));
1311+
auto *parent_tinfo = get_type_info(reinterpret_cast<PyTypeObject *>(rec.bases[0].ptr()));
13201312
assert(parent_tinfo != nullptr);
13211313
bool parent_simple_ancestors = parent_tinfo->simple_ancestors;
13221314
tinfo->simple_ancestors = parent_simple_ancestors;
@@ -1331,7 +1323,6 @@ PYBIND11_INLINE void generic_type::initialize(const type_record &rec) {
13311323
}
13321324
}
13331325

1334-
13351326
PYBIND11_INLINE void generic_type::mark_parents_nonsimple(PyTypeObject *value) {
13361327
auto t = reinterpret_borrow<tuple>(value->tp_bases);
13371328
for (handle h : t) {
@@ -1343,8 +1334,8 @@ PYBIND11_INLINE void generic_type::mark_parents_nonsimple(PyTypeObject *value) {
13431334
}
13441335
}
13451336

1346-
1347-
PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic, bool is_convertible) {
1337+
PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic,
1338+
bool is_convertible) {
13481339
m_base.attr("__entries") = dict();
13491340
auto property = handle(reinterpret_cast<PyObject *>(&PyProperty_Type));
13501341
auto static_property
@@ -1380,8 +1371,8 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic,
13801371
std::string docstring;
13811372
dict entries = arg.attr("__entries");
13821373
if ((reinterpret_cast<PyTypeObject *>(arg.ptr()))->tp_doc) {
1383-
docstring += std::string(
1384-
reinterpret_cast<PyTypeObject *>(arg.ptr())->tp_doc);
1374+
docstring
1375+
+= std::string(reinterpret_cast<PyTypeObject *>(arg.ptr())->tp_doc);
13851376
docstring += "\n\n";
13861377
}
13871378
docstring += "Members:";
@@ -1405,8 +1396,7 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic,
14051396

14061397
m_base.attr("__members__") = static_property(cpp_function(
14071398
[](handle arg) -> dict {
1408-
dict entries = arg.attr("__entries"),
1409-
m;
1399+
dict entries = arg.attr("__entries"), m;
14101400
for (auto kv : entries) {
14111401
m[kv.first] = kv.second[int_(0)];
14121402
}
@@ -1418,38 +1408,38 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::init(bool is_arithmetic,
14181408
"");
14191409

14201410
#define PYBIND11_ENUM_OP_STRICT(op, expr, strict_behavior) \
1421-
m_base.attr(op) = cpp_function( \
1422-
[](const object &a, const object &b) { \
1423-
if (!type::handle_of(a).is(type::handle_of(b))) \
1424-
strict_behavior; /* NOLINT(bugprone-macro-parentheses) */ \
1425-
return expr; \
1426-
}, \
1427-
name(op), \
1428-
is_method(m_base), \
1429-
arg("other"), \
1430-
pos_only())
1411+
m_base.attr(op) = cpp_function( \
1412+
[](const object &a, const object &b) { \
1413+
if (!type::handle_of(a).is(type::handle_of(b))) \
1414+
strict_behavior; /* NOLINT(bugprone-macro-parentheses) */ \
1415+
return expr; \
1416+
}, \
1417+
name(op), \
1418+
is_method(m_base), \
1419+
arg("other"), \
1420+
pos_only())
14311421

14321422
#define PYBIND11_ENUM_OP_CONV(op, expr) \
1433-
m_base.attr(op) = cpp_function( \
1434-
[](const object &a_, const object &b_) { \
1435-
int_ a(a_), b(b_); \
1436-
return expr; \
1437-
}, \
1438-
name(op), \
1439-
is_method(m_base), \
1440-
arg("other"), \
1441-
pos_only())
1423+
m_base.attr(op) = cpp_function( \
1424+
[](const object &a_, const object &b_) { \
1425+
int_ a(a_), b(b_); \
1426+
return expr; \
1427+
}, \
1428+
name(op), \
1429+
is_method(m_base), \
1430+
arg("other"), \
1431+
pos_only())
14421432

14431433
#define PYBIND11_ENUM_OP_CONV_LHS(op, expr) \
1444-
m_base.attr(op) = cpp_function( \
1445-
[](const object &a_, const object &b) { \
1446-
int_ a(a_); \
1447-
return expr; \
1448-
}, \
1449-
name(op), \
1450-
is_method(m_base), \
1451-
arg("other"), \
1452-
pos_only())
1434+
m_base.attr(op) = cpp_function( \
1435+
[](const object &a_, const object &b) { \
1436+
int_ a(a_); \
1437+
return expr; \
1438+
}, \
1439+
name(op), \
1440+
is_method(m_base), \
1441+
arg("other"), \
1442+
pos_only())
14531443

14541444
if (is_convertible) {
14551445
if (is_arithmetic) {
@@ -1476,8 +1466,8 @@ m_base.attr(op) = cpp_function(
14761466
pos_only());
14771467
}
14781468

1479-
1480-
PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::value(char const *name_, object value, const char *doc) {
1469+
PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void
1470+
enum_base::value(char const *name_, object value, const char *doc) {
14811471
dict entries = m_base.attr("__entries");
14821472
str name(name_);
14831473
if (entries.contains(name)) {
@@ -1490,14 +1480,12 @@ PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::value(char const *name_,
14901480
m_base.attr(std::move(name)) = std::move(value);
14911481
}
14921482

1493-
14941483
PYBIND11_NOINLINE_ATTR PYBIND11_INLINE void enum_base::export_values() {
14951484
dict entries = m_base.attr("__entries");
14961485
for (auto kv : entries) {
14971486
m_parent.attr(kv.first) = kv.second[int_(0)];
14981487
}
14991488
}
15001489

1501-
15021490
PYBIND11_NAMESPACE_END(detail)
15031491
PYBIND11_NAMESPACE_END(PYBIND11_NAMESPACE)

‎pybind11/__init__.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88

99

1010
from ._version import __version__, version_info
11-
from .commands import get_cmake_dir, get_include, get_pkgconfig_dir
11+
from .commands import get_cmake_dir, get_include, get_pkgconfig_dir, get_source_dir
1212

1313
__all__ = (
1414
"__version__",
1515
"get_cmake_dir",
1616
"get_include",
1717
"get_pkgconfig_dir",
18+
"get_source_dir",
1819
"version_info",
1920
)

‎pybind11/__main__.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
get_include_dirs,
1818
get_ldflags,
1919
get_pkgconfig_dir,
20+
get_source_dir,
2021
)
2122

2223

@@ -50,6 +51,12 @@ def main() -> None:
5051
action="store_true",
5152
help="Print the pkgconfig directory, ideal for setting $PKG_CONFIG_PATH.",
5253
)
54+
parser.add_argument(
55+
"--srcdir",
56+
action="store_true",
57+
help="Print the directory containing the library sources for the optional"
58+
" precompiled mode.",
59+
)
5360
parser.add_argument(
5461
"--extension-suffix",
5562
action="store_true",
@@ -101,6 +108,8 @@ def main() -> None:
101108
print(quote(get_cmake_dir()))
102109
if args.pkgconfigdir:
103110
print(quote(get_pkgconfig_dir()))
111+
if args.srcdir:
112+
print(quote(get_source_dir()))
104113
if args.extension_suffix:
105114
print(ext_suffix)
106115

‎pybind11/commands.py‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,24 @@ def get_include(user: bool = False) -> str: # noqa: ARG001
5252
return installed_path if os.path.exists(installed_path) else source_path
5353

5454

55+
def get_source_dir() -> str:
56+
"""
57+
Return the path to the pybind11 library sources, for the optional
58+
precompiled mode. Compile ``pybind11_combined.cpp`` (or the individual
59+
``.cpp`` files) with ``PYBIND11_PRECOMPILED`` defined, and define that
60+
macro for every translation unit that includes pybind11.
61+
"""
62+
installed_path = os.path.join(DIR, "share", "pybind11", "src")
63+
source_path = os.path.join(os.path.dirname(DIR), "src")
64+
if os.path.exists(installed_path):
65+
return installed_path
66+
if os.path.exists(source_path):
67+
return source_path
68+
69+
msg = "pybind11 library sources not found (pybind11 not installed?)"
70+
raise ImportError(msg)
71+
72+
5573
def get_cmake_dir() -> str:
5674
"""
5775
Return the path to the pybind11 CMake module directory.

‎pybind11/setup_helpers.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ class Pybind11Extension(_Extension):
108108
109109
If you want to add pybind11 headers manually, for example for an exact
110110
git checkout, then set ``include_pybind11=False``.
111+
112+
Set ``precompile=True`` to compile the pybind11 library sources into the
113+
extension (one extra translation unit) instead of instantiating everything
114+
inline in every file; this usually builds faster. Requires an installed
115+
pybind11 package that ships the library sources.
111116
"""
112117

113118
# flags are prepended, so that they can be further overridden, e.g. by
@@ -127,6 +132,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
127132
kwargs["language"] = "c++"
128133

129134
include_pybind11 = kwargs.pop("include_pybind11", True)
135+
precompile = kwargs.pop("precompile", False)
130136

131137
super().__init__(*args, **kwargs)
132138

@@ -143,6 +149,27 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
143149
except ModuleNotFoundError:
144150
pass
145151

152+
if precompile:
153+
# No silent fallback: failing to precompile would quietly rebuild
154+
# everything inline, so a missing source tree is an error.
155+
try:
156+
import pybind11
157+
158+
combined = os.path.join(
159+
pybind11.get_source_dir(), "pybind11_combined.cpp"
160+
)
161+
except (ImportError, AttributeError) as err:
162+
msg = (
163+
"precompile=True requires an installed pybind11 package "
164+
"that provides the library sources"
165+
)
166+
raise ValueError(msg) from err
167+
if not os.path.exists(combined):
168+
msg = f"pybind11 library sources not found: {combined}"
169+
raise ValueError(msg)
170+
self.sources.append(combined)
171+
self.define_macros.append(("PYBIND11_PRECOMPILED", None))
172+
146173
self.cxx_std = cxx_std
147174

148175
cflags = []

‎tests/extra_python_package/test_files.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
PKGCONFIG = """\
3434
prefix=${{pcfiledir}}/../../
3535
includedir=${{prefix}}/include
36+
srcdir=${{prefix}}/share/pybind11/src
3637
3738
Name: pybind11
3839
Description: Seamless operability between C++11 and Python

0 commit comments

Comments
 (0)