Skip to content

Commit 1cbaa50

Browse files
gvanrossumgvanrossum
authored
bpo-45696: Deep-freeze selected modules (GH-29118)
This gains 10% or more in startup time for `python -c pass` on UNIX-ish systems. The Makefile.pre.in generating code builds on Eric's work for bpo-45020, but the .c file generator is new. Windows version TBD.
1 parent fc9b622 commit 1cbaa50

File tree

16 files changed

+808
-56
lines changed

16 files changed

+808
-56
lines changed

‎.gitattributes‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ Modules/clinic/*.h linguist-generated=true
4646
Objects/clinic/*.h linguist-generated=true
4747
PC/clinic/*.h linguist-generated=true
4848
Python/clinic/*.h linguist-generated=true
49+
Python/deepfreeze/*.c linguist-generated=true
4950
Python/frozen_modules/*.h linguist-generated=true
5051
Python/frozen_modules/MANIFEST linguist-generated=true
5152
Include/internal/pycore_ast.h linguist-generated=true

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ Lib/distutils/command/*.pdb
5959
Lib/lib2to3/*.pickle
6060
Lib/test/data/*
6161
!Lib/test/data/README
62+
/_bootstrap_python
6263
/Makefile
6364
/Makefile.pre
6465
Mac/Makefile
@@ -115,6 +116,7 @@ Tools/unicode/data/
115116
/platform
116117
/profile-clean-stamp
117118
/profile-run-stamp
119+
/Python/deepfreeze/*.c
118120
/pybuilddir.txt
119121
/pyconfig.h
120122
/python-config

‎Doc/library/ctypes.rst‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1087,7 +1087,9 @@ size, we show only how this table can be read with :mod:`ctypes`::
10871087
>>> class struct_frozen(Structure):
10881088
... _fields_ = [("name", c_char_p),
10891089
... ("code", POINTER(c_ubyte)),
1090-
... ("size", c_int)]
1090+
... ("size", c_int),
1091+
... ("get_code", POINTER(c_ubyte)), # Function pointer
1092+
... ]
10911093
...
10921094
>>>
10931095

‎Include/cpython/import.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ struct _frozen {
3232
const char *name; /* ASCII encoded string */
3333
const unsigned char *code;
3434
int size;
35+
PyObject *(*get_code)(void);
3536
};
3637

3738
/* Embedding apps may change this pointer to point to their favorite

‎Lib/ctypes/test/test_values.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,9 @@ def test_frozentable(self):
5353
class struct_frozen(Structure):
5454
_fields_ = [("name", c_char_p),
5555
("code", POINTER(c_ubyte)),
56-
("size", c_int)]
56+
("size", c_int),
57+
("get_code", POINTER(c_ubyte)), # Function ptr
58+
]
5759
FrozenTable = POINTER(struct_frozen)
5860

5961
modules = []

‎Makefile.pre.in‎

Lines changed: 182 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ LIBOBJS= @LIBOBJS@
259259

260260
PYTHON= python$(EXE)
261261
BUILDPYTHON= python$(BUILDEXE)
262+
BOOTSTRAP= _bootstrap_python
262263

263264
PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
264265
UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
@@ -448,6 +449,30 @@ OBJECT_OBJS= \
448449
Objects/unionobject.o \
449450
Objects/weakrefobject.o
450451

452+
# DEEPFREEZE_OBJS is auto-generated by Tools/scripts/freeze_modules.py.
453+
DEEPFREEZE_OBJS = \
454+
Python/deepfreeze/importlib._bootstrap.o \
455+
Python/deepfreeze/importlib._bootstrap_external.o \
456+
Python/deepfreeze/zipimport.o \
457+
Python/deepfreeze/abc.o \
458+
Python/deepfreeze/codecs.o \
459+
Python/deepfreeze/io.o \
460+
Python/deepfreeze/_collections_abc.o \
461+
Python/deepfreeze/_sitebuiltins.o \
462+
Python/deepfreeze/genericpath.o \
463+
Python/deepfreeze/ntpath.o \
464+
Python/deepfreeze/posixpath.o \
465+
Python/deepfreeze/os.o \
466+
Python/deepfreeze/site.o \
467+
Python/deepfreeze/stat.o \
468+
Python/deepfreeze/__hello__.o \
469+
Python/deepfreeze/__phello__.o \
470+
Python/deepfreeze/__phello__.ham.o \
471+
Python/deepfreeze/__phello__.ham.eggs.o \
472+
Python/deepfreeze/__phello__.spam.o \
473+
Python/deepfreeze/frozen_only.o
474+
# End DEEPFREEZE_OBJS
475+
451476
##########################################################################
452477
# objects that get linked into the Python library
453478
LIBRARY_OBJS_OMIT_FROZEN= \
@@ -460,6 +485,7 @@ LIBRARY_OBJS_OMIT_FROZEN= \
460485

461486
LIBRARY_OBJS= \
462487
$(LIBRARY_OBJS_OMIT_FROZEN) \
488+
$(DEEPFREEZE_OBJS) \
463489
Python/frozen.o
464490

465491
##########################################################################
@@ -602,9 +628,9 @@ platform: $(BUILDPYTHON) pybuilddir.txt
602628
# problems by creating a dummy pybuilddir.txt just to allow interpreter
603629
# initialization to succeed. It will be overwritten by generate-posix-vars
604630
# or removed in case of failure.
605-
pybuilddir.txt: $(BUILDPYTHON)
631+
pybuilddir.txt: $(BOOTSTRAP)
606632
@echo "none" > ./pybuilddir.txt
607-
$(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\
633+
./$(BOOTSTRAP) -S -m sysconfig --generate-posix-vars ;\
608634
if test $$? -ne 0 ; then \
609635
echo "generate-posix-vars failed" ; \
610636
rm -f ./pybuilddir.txt ; \
@@ -738,6 +764,158 @@ regen-test-frozenmain: $(BUILDPYTHON)
738764
Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
739765
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
740766

767+
############################################################################
768+
# "Bootstrap Python" used to run deepfreeze.py
769+
770+
BOOTSTRAP_HEADERS = \
771+
Python/frozen_modules/importlib._bootstrap.h \
772+
Python/frozen_modules/importlib._bootstrap_external.h \
773+
Python/frozen_modules/zipimport.h
774+
775+
Python/bootstrap_frozen.o: Python/bootstrap_frozen.c Include/cpython/import.h $(BOOTSTRAP_HEADERS)
776+
777+
$(BOOTSTRAP): $(LIBRARY_OBJS_OMIT_FROZEN) \
778+
Python/bootstrap_frozen.o Programs/python.o
779+
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
780+
Python/bootstrap_frozen.o \
781+
Programs/python.o \
782+
$(LIBS) $(MODLIBS) $(SYSLIBS)
783+
784+
############################################################################
785+
# Deepfreeze targets
786+
787+
.PHONY: regen-deepfreeze
788+
regen-deepfreeze: $(DEEPFREEZE_OBJS)
789+
790+
DEEPFREEZE_DEPS = \
791+
$(BOOTSTRAP) \
792+
pybuilddir.txt \
793+
$(srcdir)/Tools/scripts/deepfreeze.py
794+
795+
# BEGIN: deepfreeze modules
796+
797+
Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Lib/importlib/_bootstrap.py $(DEEPFREEZE_DEPS)
798+
@echo "Deepfreezing Python/deepfreeze/importlib._bootstrap.c from Lib/importlib/_bootstrap.py"
799+
@./$(BOOTSTRAP) \
800+
$(srcdir)/Tools/scripts/deepfreeze.py \
801+
$(srcdir)/Lib/importlib/_bootstrap.py -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
802+
803+
Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Lib/importlib/_bootstrap_external.py $(DEEPFREEZE_DEPS)
804+
@echo "Deepfreezing Python/deepfreeze/importlib._bootstrap_external.c from Lib/importlib/_bootstrap_external.py"
805+
@./$(BOOTSTRAP) \
806+
$(srcdir)/Tools/scripts/deepfreeze.py \
807+
$(srcdir)/Lib/importlib/_bootstrap_external.py -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
808+
809+
Python/deepfreeze/zipimport.c: $(srcdir)/Lib/zipimport.py $(DEEPFREEZE_DEPS)
810+
@echo "Deepfreezing Python/deepfreeze/zipimport.c from Lib/zipimport.py"
811+
@./$(BOOTSTRAP) \
812+
$(srcdir)/Tools/scripts/deepfreeze.py \
813+
$(srcdir)/Lib/zipimport.py -m zipimport -o Python/deepfreeze/zipimport.c
814+
815+
Python/deepfreeze/abc.c: $(srcdir)/Lib/abc.py $(DEEPFREEZE_DEPS)
816+
@echo "Deepfreezing Python/deepfreeze/abc.c from Lib/abc.py"
817+
@./$(BOOTSTRAP) \
818+
$(srcdir)/Tools/scripts/deepfreeze.py \
819+
$(srcdir)/Lib/abc.py -m abc -o Python/deepfreeze/abc.c
820+
821+
Python/deepfreeze/codecs.c: $(srcdir)/Lib/codecs.py $(DEEPFREEZE_DEPS)
822+
@echo "Deepfreezing Python/deepfreeze/codecs.c from Lib/codecs.py"
823+
@./$(BOOTSTRAP) \
824+
$(srcdir)/Tools/scripts/deepfreeze.py \
825+
$(srcdir)/Lib/codecs.py -m codecs -o Python/deepfreeze/codecs.c
826+
827+
Python/deepfreeze/io.c: $(srcdir)/Lib/io.py $(DEEPFREEZE_DEPS)
828+
@echo "Deepfreezing Python/deepfreeze/io.c from Lib/io.py"
829+
@./$(BOOTSTRAP) \
830+
$(srcdir)/Tools/scripts/deepfreeze.py \
831+
$(srcdir)/Lib/io.py -m io -o Python/deepfreeze/io.c
832+
833+
Python/deepfreeze/_collections_abc.c: $(srcdir)/Lib/_collections_abc.py $(DEEPFREEZE_DEPS)
834+
@echo "Deepfreezing Python/deepfreeze/_collections_abc.c from Lib/_collections_abc.py"
835+
@./$(BOOTSTRAP) \
836+
$(srcdir)/Tools/scripts/deepfreeze.py \
837+
$(srcdir)/Lib/_collections_abc.py -m _collections_abc -o Python/deepfreeze/_collections_abc.c
838+
839+
Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Lib/_sitebuiltins.py $(DEEPFREEZE_DEPS)
840+
@echo "Deepfreezing Python/deepfreeze/_sitebuiltins.c from Lib/_sitebuiltins.py"
841+
@./$(BOOTSTRAP) \
842+
$(srcdir)/Tools/scripts/deepfreeze.py \
843+
$(srcdir)/Lib/_sitebuiltins.py -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
844+
845+
Python/deepfreeze/genericpath.c: $(srcdir)/Lib/genericpath.py $(DEEPFREEZE_DEPS)
846+
@echo "Deepfreezing Python/deepfreeze/genericpath.c from Lib/genericpath.py"
847+
@./$(BOOTSTRAP) \
848+
$(srcdir)/Tools/scripts/deepfreeze.py \
849+
$(srcdir)/Lib/genericpath.py -m genericpath -o Python/deepfreeze/genericpath.c
850+
851+
Python/deepfreeze/ntpath.c: $(srcdir)/Lib/ntpath.py $(DEEPFREEZE_DEPS)
852+
@echo "Deepfreezing Python/deepfreeze/ntpath.c from Lib/ntpath.py"
853+
@./$(BOOTSTRAP) \
854+
$(srcdir)/Tools/scripts/deepfreeze.py \
855+
$(srcdir)/Lib/ntpath.py -m ntpath -o Python/deepfreeze/ntpath.c
856+
857+
Python/deepfreeze/posixpath.c: $(srcdir)/Lib/posixpath.py $(DEEPFREEZE_DEPS)
858+
@echo "Deepfreezing Python/deepfreeze/posixpath.c from Lib/posixpath.py"
859+
@./$(BOOTSTRAP) \
860+
$(srcdir)/Tools/scripts/deepfreeze.py \
861+
$(srcdir)/Lib/posixpath.py -m posixpath -o Python/deepfreeze/posixpath.c
862+
863+
Python/deepfreeze/os.c: $(srcdir)/Lib/os.py $(DEEPFREEZE_DEPS)
864+
@echo "Deepfreezing Python/deepfreeze/os.c from Lib/os.py"
865+
@./$(BOOTSTRAP) \
866+
$(srcdir)/Tools/scripts/deepfreeze.py \
867+
$(srcdir)/Lib/os.py -m os -o Python/deepfreeze/os.c
868+
869+
Python/deepfreeze/site.c: $(srcdir)/Lib/site.py $(DEEPFREEZE_DEPS)
870+
@echo "Deepfreezing Python/deepfreeze/site.c from Lib/site.py"
871+
@./$(BOOTSTRAP) \
872+
$(srcdir)/Tools/scripts/deepfreeze.py \
873+
$(srcdir)/Lib/site.py -m site -o Python/deepfreeze/site.c
874+
875+
Python/deepfreeze/stat.c: $(srcdir)/Lib/stat.py $(DEEPFREEZE_DEPS)
876+
@echo "Deepfreezing Python/deepfreeze/stat.c from Lib/stat.py"
877+
@./$(BOOTSTRAP) \
878+
$(srcdir)/Tools/scripts/deepfreeze.py \
879+
$(srcdir)/Lib/stat.py -m stat -o Python/deepfreeze/stat.c
880+
881+
Python/deepfreeze/__hello__.c: $(srcdir)/Lib/__hello__.py $(DEEPFREEZE_DEPS)
882+
@echo "Deepfreezing Python/deepfreeze/__hello__.c from Lib/__hello__.py"
883+
@./$(BOOTSTRAP) \
884+
$(srcdir)/Tools/scripts/deepfreeze.py \
885+
$(srcdir)/Lib/__hello__.py -m __hello__ -o Python/deepfreeze/__hello__.c
886+
887+
Python/deepfreeze/__phello__.c: $(srcdir)/Lib/__phello__/__init__.py $(DEEPFREEZE_DEPS)
888+
@echo "Deepfreezing Python/deepfreeze/__phello__.c from Lib/__phello__/__init__.py"
889+
@./$(BOOTSTRAP) \
890+
$(srcdir)/Tools/scripts/deepfreeze.py \
891+
$(srcdir)/Lib/__phello__/__init__.py -m __phello__ -o Python/deepfreeze/__phello__.c
892+
893+
Python/deepfreeze/__phello__.ham.c: $(srcdir)/Lib/__phello__/ham/__init__.py $(DEEPFREEZE_DEPS)
894+
@echo "Deepfreezing Python/deepfreeze/__phello__.ham.c from Lib/__phello__/ham/__init__.py"
895+
@./$(BOOTSTRAP) \
896+
$(srcdir)/Tools/scripts/deepfreeze.py \
897+
$(srcdir)/Lib/__phello__/ham/__init__.py -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
898+
899+
Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Lib/__phello__/ham/eggs.py $(DEEPFREEZE_DEPS)
900+
@echo "Deepfreezing Python/deepfreeze/__phello__.ham.eggs.c from Lib/__phello__/ham/eggs.py"
901+
@./$(BOOTSTRAP) \
902+
$(srcdir)/Tools/scripts/deepfreeze.py \
903+
$(srcdir)/Lib/__phello__/ham/eggs.py -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
904+
905+
Python/deepfreeze/__phello__.spam.c: $(srcdir)/Lib/__phello__/spam.py $(DEEPFREEZE_DEPS)
906+
@echo "Deepfreezing Python/deepfreeze/__phello__.spam.c from Lib/__phello__/spam.py"
907+
@./$(BOOTSTRAP) \
908+
$(srcdir)/Tools/scripts/deepfreeze.py \
909+
$(srcdir)/Lib/__phello__/spam.py -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
910+
911+
Python/deepfreeze/frozen_only.c: $(srcdir)/Tools/freeze/flag.py $(DEEPFREEZE_DEPS)
912+
@echo "Deepfreezing Python/deepfreeze/frozen_only.c from Tools/freeze/flag.py"
913+
@./$(BOOTSTRAP) \
914+
$(srcdir)/Tools/scripts/deepfreeze.py \
915+
$(srcdir)/Tools/freeze/flag.py -m frozen_only -o Python/deepfreeze/frozen_only.c
916+
917+
# END: deepfreeze modules
918+
741919
############################################################################
742920
# frozen modules (including importlib)
743921

@@ -2017,7 +2195,8 @@ clean-retain-profile: pycremoval
20172195
find build -name '*.py[co]' -exec rm -f {} ';' || true
20182196
-rm -f pybuilddir.txt
20192197
-rm -f Lib/lib2to3/*Grammar*.pickle
2020-
-rm -f Programs/_testembed Programs/_freeze_module
2198+
-rm -f Programs/_testembed Programs/_freeze_module $(BOOTSTRAP)
2199+
-rm -f Python/deepfreeze/*.[co]
20212200
-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
20222201
-rm -f Include/pydtrace_probes.h
20232202
-rm -f profile-gen-stamp

‎Misc/NEWS.d/next/Build/2021-11-03-00-19-50.bpo-45696.eKs46f.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Skip the marshal step for frozen modules by generating C code that produces a set of ready-to-use code objects. This speeds up startup time by another 10% or more.

‎Python/bootstrap_frozen.c‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
2+
/* Frozen modules bootstrap */
3+
4+
/* This file is linked with "bootstrap Python"
5+
which is used (only) to run Tools/scripts/deepfreeze.py. */
6+
7+
#include "Python.h"
8+
#include "pycore_import.h"
9+
10+
/* Includes for frozen modules: */
11+
#include "frozen_modules/importlib._bootstrap.h"
12+
#include "frozen_modules/importlib._bootstrap_external.h"
13+
#include "frozen_modules/zipimport.h"
14+
/* End includes */
15+
16+
/* Note that a negative size indicates a package. */
17+
18+
static const struct _frozen bootstrap_modules[] = {
19+
{"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
20+
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
21+
{"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
22+
{0, 0, 0} /* bootstrap sentinel */
23+
};
24+
static const struct _frozen stdlib_modules[] = {
25+
{0, 0, 0} /* stdlib sentinel */
26+
};
27+
static const struct _frozen test_modules[] = {
28+
{0, 0, 0} /* test sentinel */
29+
};
30+
const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
31+
const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
32+
const struct _frozen *_PyImport_FrozenTest = test_modules;
33+
34+
static const struct _module_alias aliases[] = {
35+
{"_frozen_importlib", "importlib._bootstrap"},
36+
{"_frozen_importlib_external", "importlib._bootstrap_external"},
37+
{0, 0} /* aliases sentinel */
38+
};
39+
const struct _module_alias *_PyImport_FrozenAliases = aliases;
40+
41+
42+
/* Embedding apps may change this pointer to point to their favorite
43+
collection of frozen modules: */
44+
45+
const struct _frozen *PyImport_FrozenModules = NULL;

‎Python/deepfreeze/README.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
This directory contains the generated .c files for all the deep-frozen
2+
modules. Python/frozen.c depends on these files.
3+
4+
None of these files are committed into the repo.
5+
6+
See Tools/scripts/freeze_modules.py for more info.

0 commit comments

Comments
 (0)