Skip to content

Commit 0e35313

Browse files
authored
gh-160689 : Link only the bootstrap modules into the build helper programs (#160537)
The build-helper programs (`Programs/_freeze_module` and `_bootstrap_python`) don't need to link extension modules that they don't need. This PR makes sure the helper programs get only the modules they need to do their job, regardless of what the build configuration sets for the final built runtime and interpreter. This follows the same pattern as what `PCbuild/_freeze_module.vcxproj` does with `PC/config_minimal.c`, and lets "extra" builtin modules (from `Setup.local` or `MODULE_BUILDTYPE=static`) depend on frozen modules or `getpath`.
1 parent 3acc941 commit 0e35313

7 files changed

Lines changed: 143 additions & 37 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,7 @@ Modules/Setup.config
102102
Modules/Setup.local
103103
Modules/Setup.stdlib
104104
Modules/config.c
105+
Modules/config_bootstrap.c
105106
Modules/ld_so_aix
106107
Programs/_freeze_module
107108
Programs/_testembed

‎Makefile.pre.in‎

Lines changed: 54 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,13 @@ MODSHARED_NAMES= _MODSHARED_NAMES_
2525
MODDISABLED_NAMES= _MODDISABLED_NAMES_
2626
MODOBJS= _MODOBJS_
2727
MODLIBS= _MODLIBS_
28+
# Objects and libraries of the bootstrap modules (Modules/Setup.bootstrap)
29+
# only. These are the only static modules linked into the build helper
30+
# programs (Programs/_freeze_module, _bootstrap_python), so that other
31+
# builtin modules (Modules/Setup.local, MODULE_BUILDTYPE=static) may depend
32+
# on products of those helpers (frozen modules, getpath).
33+
BOOTSTRAP_MODOBJS= _BOOTSTRAP_MODOBJS_
34+
BOOTSTRAP_MODLIBS= _BOOTSTRAP_MODLIBS_
2835

2936
# === Variables set by configure
3037
VERSION= @VERSION@
@@ -359,11 +366,14 @@ COVERAGE_REPORT_OPTIONS=--rc lcov_branch_coverage=1 --branch-coverage --title "C
359366

360367
##########################################################################
361368
# Modules
362-
MODULE_OBJS= \
363-
Modules/config.o \
369+
MODULE_OBJS_COMMON= \
364370
Modules/main.o \
365371
Modules/gcmodule.o
366372

373+
MODULE_OBJS= \
374+
Modules/config.o \
375+
$(MODULE_OBJS_COMMON)
376+
367377
IO_H= Modules/_io/_iomodule.h
368378

369379
IO_OBJS= \
@@ -614,6 +624,19 @@ LIBRARY_OBJS= \
614624
Modules/getpath.o \
615625
Python/frozen.o
616626

627+
# objects that get linked into the build helper programs
628+
# (Programs/_freeze_module, _bootstrap_python): the core, plus the bootstrap
629+
# modules only (see BOOTSTRAP_MODOBJS). Does not include frozen modules nor
630+
# getpath, which the helpers are used to build.
631+
BOOTSTRAP_LIBRARY_OBJS= \
632+
Modules/getbuildinfo_bootstrap.o \
633+
$(PARSER_OBJS) \
634+
$(OBJECT_OBJS) \
635+
$(PYTHON_OBJS) \
636+
Modules/config_bootstrap.o \
637+
$(MODULE_OBJS_COMMON) \
638+
$(BOOTSTRAP_MODOBJS)
639+
617640
LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
618641

619642
##########################################################################
@@ -1613,7 +1636,7 @@ Modules/Setup.bootstrap: $(srcdir)/Modules/Setup.bootstrap.in config.status
16131636
Modules/Setup.stdlib: $(srcdir)/Modules/Setup.stdlib.in config.status
16141637
./config.status $@
16151638

1616-
Makefile Modules/config.c: Makefile.pre \
1639+
Makefile Modules/config.c Modules/config_bootstrap.c: Makefile.pre \
16171640
$(srcdir)/Modules/config.c.in \
16181641
$(MAKESETUP) \
16191642
$(srcdir)/Modules/Setup \
@@ -1624,9 +1647,9 @@ Makefile Modules/config.c: Makefile.pre \
16241647
-s Modules \
16251648
Modules/Setup.local \
16261649
Modules/Setup.stdlib \
1627-
Modules/Setup.bootstrap \
1650+
-b Modules/Setup.bootstrap \
16281651
$(srcdir)/Modules/Setup
1629-
@mv config.c Modules
1652+
@mv config.c config_bootstrap.c Modules
16301653
@echo "The Makefile was updated, you may need to re-run make."
16311654

16321655
.PHONY: regen-test-frozenmain
@@ -1661,9 +1684,9 @@ BOOTSTRAP_HEADERS = \
16611684

16621685
Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)
16631686

1664-
_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
1665-
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
1666-
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)
1687+
_bootstrap_python: $(BOOTSTRAP_LIBRARY_OBJS) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
1688+
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(BOOTSTRAP_LIBRARY_OBJS) \
1689+
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(BOOTSTRAP_MODLIBS) $(SYSLIBS)
16671690
# Dummy pybuilddir.txt is needed for _bootstrap_python to be runnable
16681691
@echo "none" > ./pybuilddir.txt
16691692

@@ -1681,6 +1704,9 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
16811704
#
16821705
# Standard build process:
16831706
# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*().
1707+
# Only the bootstrap modules (Modules/Setup.bootstrap) are part of these
1708+
# objects (see BOOTSTRAP_LIBRARY_OBJS), so other builtin modules may
1709+
# depend on the output of the following steps.
16841710
# 2) build Programs/_freeze_module binary.
16851711
# 3) create frozen module headers for importlib and getpath.
16861712
# 4) build _bootstrap_python binary.
@@ -1753,8 +1779,8 @@ Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
17531779

17541780
Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
17551781

1756-
Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN)
1757-
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
1782+
Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(BOOTSTRAP_LIBRARY_OBJS)
1783+
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(BOOTSTRAP_LIBRARY_OBJS) $(LIBS) $(BOOTSTRAP_MODLIBS) $(SYSLIBS)
17581784

17591785
# We manually freeze getpath.py rather than through freeze_modules
17601786
Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
@@ -1907,11 +1933,24 @@ Modules/getbuildinfo.o: $(PARSER_OBJS) \
19071933
$(MODOBJS) \
19081934
$(DTRACE_OBJS) \
19091935
$(srcdir)/Modules/getbuildinfo.c
1910-
$(CC) -c $(PY_CORE_CFLAGS) \
1936+
$(CC) -c $(PY_CORE_CFLAGS) $(GETBUILDINFO_CFLAGS) \
1937+
-o $@ $(srcdir)/Modules/getbuildinfo.c
1938+
1939+
# Separate object for the build helper programs: it must not depend on
1940+
# $(MODOBJS), which may in turn depend on the helpers' output.
1941+
Modules/getbuildinfo_bootstrap.o: $(PARSER_OBJS) \
1942+
$(OBJECT_OBJS) \
1943+
$(PYTHON_OBJS) \
1944+
$(MODULE_OBJS_COMMON) \
1945+
$(BOOTSTRAP_MODOBJS) \
1946+
$(srcdir)/Modules/getbuildinfo.c
1947+
$(CC) -c $(PY_CORE_CFLAGS) $(GETBUILDINFO_CFLAGS) \
1948+
-o $@ $(srcdir)/Modules/getbuildinfo.c
1949+
1950+
GETBUILDINFO_CFLAGS= \
19111951
-DGITVERSION="\"`LC_ALL=C $(GITVERSION)`\"" \
19121952
-DGITTAG="\"`LC_ALL=C $(GITTAG)`\"" \
1913-
-DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\"" \
1914-
-o $@ $(srcdir)/Modules/getbuildinfo.c
1953+
-DGITBRANCH="\"`LC_ALL=C $(GITBRANCH)`\""
19151954

19161955
Modules/getpath.o: $(srcdir)/Modules/getpath.c Python/frozen_modules/getpath.h Makefile $(PYTHON_HEADERS)
19171956
$(CC) -c $(PY_CORE_CFLAGS) -DPYTHONPATH='"$(PYTHONPATH)"' \
@@ -2279,7 +2318,7 @@ regen-slots: Python/slots.toml
22792318
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_slots.py \
22802319
--generate-all
22812320

2282-
$(LIBRARY_OBJS) $(MODOBJS) Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS)
2321+
$(LIBRARY_OBJS) $(MODOBJS) Modules/config_bootstrap.o Programs/python.o $(PLATFORM_MAIN_OBJS): $(PYTHON_HEADERS)
22832322

22842323

22852324
######################################################################
@@ -3365,7 +3404,7 @@ clean: clean-profile clean-jit-stencils
33653404
clobber: clean
33663405
-rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \
33673406
tags TAGS \
3368-
config.cache config.log pyconfig.h Modules/config.c
3407+
config.cache config.log pyconfig.h Modules/config.c Modules/config_bootstrap.c
33693408
-rm -rf build platform
33703409
-rm -rf $(PYTHONFRAMEWORKDIR)
33713410
-rm -rf Platforms/Apple/iOS/Frameworks
Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
The build helper programs ``Programs/_freeze_module`` and
2+
``_bootstrap_python`` are now linked only with the modules of
3+
:file:`Modules/Setup.bootstrap`, instead of with every static module. Builtin
4+
modules added via :file:`Modules/Setup.local` or ``MODULE_BUILDTYPE=static``
5+
can now depend on frozen modules or on ``getpath``.

‎Modules/makesetup‎

Lines changed: 78 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,22 @@ set -e
44
# Convert templates into Makefile and config.c, based on the module
55
# definitions found in the file Setup.
66
#
7-
# Usage: makesetup [-s dir] [-c file] [-m file] [Setup] ... [-n [Setup] ...]
7+
# Usage: makesetup [-s dir] [-c file] [-m file] [Setup] ... [-b Setup] ...
8+
# [-n [Setup] ...]
89
#
910
# Options:
1011
# -s directory: alternative source directory (default .)
1112
# -l directory: library source directory (default derived from $0)
1213
# -c file: alternative config.c template (default $libdir/config.c.in)
13-
# -c -: don't write config.c
14+
# -c -: don't write config.c (nor config_bootstrap.c)
1415
# -m file: alternative Makefile template (default ./Makefile.pre)
1516
# -m -: don't write Makefile
1617
#
1718
# Remaining arguments are one or more Setup files (default ./Setup).
19+
# The Setup file following a -b option holds the *bootstrap* modules:
20+
# the static modules that the build helper programs (_freeze_module,
21+
# _bootstrap_python) are linked with. They are also regular modules of
22+
# the final build.
1823
# Setup files after a -n option are used for their variables, modules
1924
# and libraries but not for their .o files.
2025
#
@@ -28,6 +33,9 @@ set -e
2833
# + insert 'extern PyObject* PyInit_<module>(void);' before MARKER 1
2934
# + insert '{"<module>", PyInit_<module>},' before MARKER 2
3035
#
36+
# Copying config.c.in to config_bootstrap.c (only with a -b Setup file):
37+
# - same as config.c, but only for the modules of the -b Setup file
38+
#
3139
# Copying Makefile.pre to Makefile:
3240
# - insert an identifying comment at the start
3341
# - replace _MODBUILT_NAMES_ by the list of *static* and *shared* modules
@@ -37,6 +45,8 @@ set -e
3745
# - replace _MODOBJS_ by the list of objects from Setup (except for
3846
# Setup files after a -n option)
3947
# - replace _MODLIBS_ by the list of libraries from Setup
48+
# - replace _BOOTSTRAP_MODOBJS_ and _BOOTSTRAP_MODLIBS_ by the objects and
49+
# libraries of the -b Setup file
4050
# - for each object file mentioned in Setup, append a rule
4151
# '<file>.o: <file>.c; <build commands>' to the end of the Makefile
4252
# - for each module mentioned in Setup, append a rule
@@ -47,13 +57,14 @@ set -e
4757
# Loop over command line options
4858
usage='
4959
usage: makesetup [-s srcdir] [-l libdir] [-c config.c.in] [-m Makefile.pre]
50-
[Setup] ... [-n [Setup] ...]'
60+
[Setup] ... [-b Setup] ... [-n [Setup] ...]'
5161
srcdir='.'
5262
libdir=''
5363
config=''
5464
makepre=''
5565
noobjects=''
5666
doconfig=yes
67+
bootstrap=
5768
while :
5869
do
5970
case $1 in
@@ -93,7 +104,8 @@ do
93104
echo '' # Add a linebreak so we don't choke on files missing EOL.
94105
case $i in
95106
-n) echo '*noobjects*';;
96-
*) echo '*doconfig*'; cat "$i";;
107+
-b) echo '*bootstrap*';;
108+
*) echo '*doconfig*'; cat "$i"; echo ''; echo '*nobootstrap*';;
97109
esac
98110
done |
99111
sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
@@ -114,6 +126,10 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
114126
LIBS=
115127
LOCALLIBS=
116128
BASELIBS=
129+
BOOTMODS=
130+
BOOTOBJS=
131+
BOOTLIBS=
132+
havebootstrap=
117133
while read line
118134
do
119135
# to handle backslashes for sh's that don't automatically
@@ -136,6 +152,8 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
136152
noobjects=yes;
137153
continue;;
138154
'*doconfig*') doconfig=yes; continue;;
155+
'*bootstrap*') bootstrap=yes; havebootstrap=yes; continue;;
156+
'*nobootstrap*') bootstrap=; continue;;
139157
'*static*') doconfig=yes; continue;;
140158
'*noconfig*') doconfig=no; continue;;
141159
'*shared*') doconfig=no; continue;;
@@ -204,6 +222,18 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
204222
# Detected multiple rules for a module, first rule wins. This
205223
# allows users to disable modules in Setup.local.
206224
echo 1>&2 "maksetup: '$mod' was handled by previous rule."
225+
case $bootstrap in
226+
yes)
227+
# This module is in the bootstrap Setup file, but an
228+
# earlier Setup file (e.g. Setup.local) already defined
229+
# it, and that definition won. Give the build helpers
230+
# that earlier definition, so they use the same module
231+
# as the final interpreter.
232+
eval "BOOTMODS=\"\$BOOTMODS \${STATICMOD_$mod}\""
233+
eval "BOOTOBJS=\"\$BOOTOBJS \${STATICOBJS_$mod}\""
234+
eval "BOOTLIBS=\"\$BOOTLIBS \${STATICLIBS_$mod}\""
235+
;;
236+
esac
207237
continue 2;;
208238
esac
209239
CONFIGURED="$CONFIGURED,${mod},"
@@ -213,6 +243,12 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
213243
LIBS="$LIBS $libs"
214244
MODS="$MODS $mods"
215245
BUILT="$BUILT $mods"
246+
case $bootstrap in
247+
yes)
248+
BOOTLIBS="$BOOTLIBS $libs"
249+
BOOTMODS="$BOOTMODS $mods"
250+
;;
251+
esac
216252
;;
217253
no)
218254
BUILT="$BUILT $mods"
@@ -265,7 +301,21 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
265301
echo "$rule" >>$rulesf
266302
done
267303
case $doconfig in
268-
yes) OBJS="$OBJS $objs";;
304+
yes)
305+
OBJS="$OBJS $objs"
306+
case $bootstrap in
307+
yes) BOOTOBJS="$BOOTOBJS $objs";;
308+
esac
309+
# Record each static module's objects and libs by name, so
310+
# the bootstrap handling above can find a definition that
311+
# won over a later line in the bootstrap Setup file.
312+
for mod in $mods
313+
do
314+
eval "STATICMOD_$mod='$mod'"
315+
eval "STATICOBJS_$mod='$objs'"
316+
eval "STATICLIBS_$mod='$libs'"
317+
done
318+
;;
269319
esac
270320
for mod in $mods
271321
do
@@ -295,24 +345,31 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
295345
DEFS="BASEMODLIBS=$BASELIBS$NL$DEFS"
296346
DEFS="LOCALMODLIBS=$LOCALLIBS$NL$DEFS"
297347

298-
EXTDECLS=
299-
INITBITS=
300-
for mod in $MODS
301-
do
302-
EXTDECLS="${EXTDECLS}extern PyObject* PyInit_$mod(void);$NL"
303-
INITBITS="${INITBITS} {\"$mod\", PyInit_$mod},$NL"
304-
done
305-
306-
307-
case $config in
308-
-) ;;
309-
*) sed -e "
348+
# Generate a config.c from the template for the given modules
349+
genconfig() {
350+
out=$1; shift
351+
EXTDECLS=
352+
INITBITS=
353+
for mod
354+
do
355+
EXTDECLS="${EXTDECLS}extern PyObject* PyInit_$mod(void);$NL"
356+
INITBITS="${INITBITS} {\"$mod\", PyInit_$mod},$NL"
357+
done
358+
sed -e "
310359
1i$NL/* Generated automatically from $config by makesetup. */
311360
/MARKER 1/i$NL$EXTDECLS
312361
313362
/MARKER 2/i$NL$INITBITS
314363
315-
" $config >config.c
364+
" $config >$out
365+
}
366+
367+
case $config in
368+
-) ;;
369+
*) genconfig config.c $MODS
370+
case $havebootstrap in
371+
yes) genconfig config_bootstrap.c $BOOTMODS;;
372+
esac
316373
;;
317374
esac
318375

@@ -331,6 +388,9 @@ sed -e 's/[ ]*#.*//' -e '/^[ ]*$/d' |
331388
echo "s%_MODBUILT_NAMES_%$BUILT%" >>$sedf
332389
echo "s%_MODSHARED_NAMES_%$BUILT_SHARED%" >>$sedf
333390
echo "s%_MODDISABLED_NAMES_%$DISABLED%" >>$sedf
391+
# _BOOTSTRAP_* first: _MODOBJS_ is a substring of _BOOTSTRAP_MODOBJS_
392+
echo "s%_BOOTSTRAP_MODOBJS_%$BOOTOBJS%" >>$sedf
393+
echo "s%_BOOTSTRAP_MODLIBS_%$BOOTLIBS%" >>$sedf
334394
echo "s%_MODOBJS_%$OBJS%" >>$sedf
335395
echo "s%_MODLIBS_%$LIBS%" >>$sedf
336396
echo "/Definitions added by makesetup/r $sedr" >>$sedf

‎Tools/c-analyzer/cpython/ignored.tsv‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ Modules/cmathmodule.c - sinh_special_values -
314314
Modules/cmathmodule.c - sqrt_special_values -
315315
Modules/cmathmodule.c - tanh_special_values -
316316
Modules/config.c - _PyImport_Inittab -
317+
Modules/config_bootstrap.c - _PyImport_Inittab -
317318
Modules/faulthandler.c - faulthandler_handlers -
318319
Modules/getnameinfo.c - gni_afdl -
319320
Modules/hmacmodule.c - py_hmac_static_hinfo -

‎configure‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)