Skip to content

Commit eb483c4

Browse files
tirantiran
authored
bpo-45949: Pure Python freeze module for cross builds (GH-29899)
1 parent a62be77 commit eb483c4

File tree

9 files changed

+205
-115
lines changed

9 files changed

+205
-115
lines changed

‎Doc/using/configure.rst‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -518,9 +518,8 @@ Cross Compiling Options
518518

519519
Cross compiling, also known as cross building, can be used to build Python
520520
for another CPU architecture or platform. Cross compiling requires a Python
521-
interpreter and the :program:`_freeze_module` binary from another build. The
522-
version of the build Python and :program:`_freeze_module` command must be
523-
the same as the cross compiled host Python.
521+
interpreter for the build platform. The version of the build Python must match
522+
the version of the cross compiled host Python.
524523

525524
.. cmdoption:: --build=BUILD
526525

@@ -530,13 +529,7 @@ the same as the cross compiled host Python.
530529

531530
cross-compile to build programs to run on HOST (target platform)
532531

533-
.. cmdoption:: --with-freeze-module=Programs/_freeze_module
534-
535-
path to ``_freeze_module`` binary for cross compiling.
536-
537-
.. versionadded:: 3.11
538-
539-
.. cmdoption:: --with-build-python=python3.xx
532+
.. cmdoption:: --with-build-python=path/to/python
540533

541534
path to build ``python`` binary for cross compiling
542535

@@ -559,7 +552,6 @@ Cross compiling example::
559552
CONFIG_SITE=config.site-aarch64 ../configure \
560553
--build=x86_64-pc-linux-gnu \
561554
--host=aarch64-unknown-linux-gnu \
562-
--with-freeze-module=../x86_64/Programs/_freeze_module \
563555
--with-build-python=../x86_64/python
564556

565557

‎Makefile.pre.in‎

Lines changed: 65 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -283,14 +283,19 @@ BUILDPYTHON= python$(BUILDEXE)
283283
PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
284284
UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
285285
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
286-
# Standard builds use _bootstrap_python for freezing, cross compiling
287-
# uses build Python, which must have the same version and bytecode,
288-
PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
286+
287+
# Normal builds use Programs/_freeze_module.c for bootstrapping and
288+
# ./_bootstrap_python Programs/_freeze_module.py for remaining modules
289+
# Cross builds use an external "build Python" for all modules.
290+
PYTHON_FOR_FREEZE=@PYTHON_FOR_FREEZE@
291+
FREEZE_MODULE_BOOTSTRAP=@FREEZE_MODULE_BOOTSTRAP@
292+
FREEZE_MODULE_BOOTSTRAP_DEPS=@FREEZE_MODULE_BOOTSTRAP_DEPS@
293+
FREEZE_MODULE=@FREEZE_MODULE@
294+
FREEZE_MODULE_DEPS=@FREEZE_MODULE_DEPS@
295+
289296
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
290297
BUILD_GNU_TYPE= @build@
291298
HOST_GNU_TYPE= @host@
292-
# Allow developers to override freeze_module command for cross building (bpo-45886)
293-
FREEZE_MODULE?=@FREEZE_MODULE@
294299

295300
# Tcl and Tk config info from --with-tcltk-includes and -libs options
296301
TCLTK_INCLUDES= @TCLTK_INCLUDES@
@@ -967,7 +972,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
967972
.PHONY: regen-deepfreeze
968973
regen-deepfreeze: $(DEEPFREEZE_OBJS)
969974

970-
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python
975+
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS)
971976

972977
# BEGIN: deepfreeze modules
973978

@@ -1044,6 +1049,30 @@ Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZ
10441049

10451050
############################################################################
10461051
# frozen modules (including importlib)
1052+
#
1053+
# Freezing is a multi step process. It works differently for standard builds
1054+
# and cross builds. Standard builds use Programs/_freeze_module and
1055+
# _bootstrap_python for freezing and deepfreezing, so users can build Python
1056+
# without an existing Python installation. Cross builds cannot execute
1057+
# compiled binaries and therefore rely on an external build Python
1058+
# interpreter. The build interpreter must have same version and same bytecode
1059+
# as the host (target) binary.
1060+
#
1061+
# Standard build process:
1062+
# 1) compile minimal core objects for Py_Compile*() and PyMarshal_Write*().
1063+
# 2) build Programs/_freeze_module binary.
1064+
# 3) create frozen module headers for importlib and getpath.
1065+
# 4) build _bootstrap_python binary.
1066+
# 5) create remaining frozen module headers with
1067+
# ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python
1068+
# script is used to test the cross compile code path.
1069+
# 6) deepfreeze modules with _bootstrap_python
1070+
#
1071+
# Cross compile process:
1072+
# 1) create all frozen module headers with external build Python and
1073+
# Programs/_freeze_module.py script.
1074+
# 2) deepfreeze modules with external build Python.
1075+
#
10471076

10481077
# FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
10491078
FROZEN_FILES_IN = \
@@ -1104,83 +1133,83 @@ Modules/getpath_noop.o: $(srcdir)/Modules/getpath_noop.c Makefile
11041133
Programs/_freeze_module: Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN)
11051134
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o Modules/getpath_noop.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
11061135

1136+
# We manually freeze getpath.py rather than through freeze_modules
1137+
Python/frozen_modules/getpath.h: Modules/getpath.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
1138+
$(FREEZE_MODULE_BOOTSTRAP) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
1139+
11071140
# BEGIN: freezing modules
11081141

1109-
Python/frozen_modules/importlib._bootstrap.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap.py
1110-
$(FREEZE_MODULE) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
1142+
Python/frozen_modules/importlib._bootstrap.h: Lib/importlib/_bootstrap.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
1143+
$(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py Python/frozen_modules/importlib._bootstrap.h
11111144

1112-
Python/frozen_modules/importlib._bootstrap_external.h: $(FREEZE_MODULE) Lib/importlib/_bootstrap_external.py
1113-
$(FREEZE_MODULE) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
1145+
Python/frozen_modules/importlib._bootstrap_external.h: Lib/importlib/_bootstrap_external.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
1146+
$(FREEZE_MODULE_BOOTSTRAP) importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py Python/frozen_modules/importlib._bootstrap_external.h
11141147

1115-
Python/frozen_modules/zipimport.h: $(FREEZE_MODULE) Lib/zipimport.py
1116-
$(FREEZE_MODULE) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
1148+
Python/frozen_modules/zipimport.h: Lib/zipimport.py $(FREEZE_MODULE_BOOTSTRAP_DEPS)
1149+
$(FREEZE_MODULE_BOOTSTRAP) zipimport $(srcdir)/Lib/zipimport.py Python/frozen_modules/zipimport.h
11171150

1118-
Python/frozen_modules/abc.h: $(FREEZE_MODULE) Lib/abc.py
1151+
Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS)
11191152
$(FREEZE_MODULE) abc $(srcdir)/Lib/abc.py Python/frozen_modules/abc.h
11201153

1121-
Python/frozen_modules/codecs.h: $(FREEZE_MODULE) Lib/codecs.py
1154+
Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS)
11221155
$(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h
11231156

1124-
Python/frozen_modules/io.h: $(FREEZE_MODULE) Lib/io.py
1157+
Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS)
11251158
$(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h
11261159

1127-
Python/frozen_modules/_collections_abc.h: $(FREEZE_MODULE) Lib/_collections_abc.py
1160+
Python/frozen_modules/_collections_abc.h: Lib/_collections_abc.py $(FREEZE_MODULE_DEPS)
11281161
$(FREEZE_MODULE) _collections_abc $(srcdir)/Lib/_collections_abc.py Python/frozen_modules/_collections_abc.h
11291162

1130-
Python/frozen_modules/_sitebuiltins.h: $(FREEZE_MODULE) Lib/_sitebuiltins.py
1163+
Python/frozen_modules/_sitebuiltins.h: Lib/_sitebuiltins.py $(FREEZE_MODULE_DEPS)
11311164
$(FREEZE_MODULE) _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py Python/frozen_modules/_sitebuiltins.h
11321165

1133-
Python/frozen_modules/genericpath.h: $(FREEZE_MODULE) Lib/genericpath.py
1166+
Python/frozen_modules/genericpath.h: Lib/genericpath.py $(FREEZE_MODULE_DEPS)
11341167
$(FREEZE_MODULE) genericpath $(srcdir)/Lib/genericpath.py Python/frozen_modules/genericpath.h
11351168

1136-
Python/frozen_modules/ntpath.h: $(FREEZE_MODULE) Lib/ntpath.py
1169+
Python/frozen_modules/ntpath.h: Lib/ntpath.py $(FREEZE_MODULE_DEPS)
11371170
$(FREEZE_MODULE) ntpath $(srcdir)/Lib/ntpath.py Python/frozen_modules/ntpath.h
11381171

1139-
Python/frozen_modules/posixpath.h: $(FREEZE_MODULE) Lib/posixpath.py
1172+
Python/frozen_modules/posixpath.h: Lib/posixpath.py $(FREEZE_MODULE_DEPS)
11401173
$(FREEZE_MODULE) posixpath $(srcdir)/Lib/posixpath.py Python/frozen_modules/posixpath.h
11411174

1142-
Python/frozen_modules/os.h: $(FREEZE_MODULE) Lib/os.py
1175+
Python/frozen_modules/os.h: Lib/os.py $(FREEZE_MODULE_DEPS)
11431176
$(FREEZE_MODULE) os $(srcdir)/Lib/os.py Python/frozen_modules/os.h
11441177

1145-
Python/frozen_modules/site.h: $(FREEZE_MODULE) Lib/site.py
1178+
Python/frozen_modules/site.h: Lib/site.py $(FREEZE_MODULE_DEPS)
11461179
$(FREEZE_MODULE) site $(srcdir)/Lib/site.py Python/frozen_modules/site.h
11471180

1148-
Python/frozen_modules/stat.h: $(FREEZE_MODULE) Lib/stat.py
1181+
Python/frozen_modules/stat.h: Lib/stat.py $(FREEZE_MODULE_DEPS)
11491182
$(FREEZE_MODULE) stat $(srcdir)/Lib/stat.py Python/frozen_modules/stat.h
11501183

1151-
Python/frozen_modules/importlib.util.h: $(FREEZE_MODULE) Lib/importlib/util.py
1184+
Python/frozen_modules/importlib.util.h: Lib/importlib/util.py $(FREEZE_MODULE_DEPS)
11521185
$(FREEZE_MODULE) importlib.util $(srcdir)/Lib/importlib/util.py Python/frozen_modules/importlib.util.h
11531186

1154-
Python/frozen_modules/importlib.machinery.h: $(FREEZE_MODULE) Lib/importlib/machinery.py
1187+
Python/frozen_modules/importlib.machinery.h: Lib/importlib/machinery.py $(FREEZE_MODULE_DEPS)
11551188
$(FREEZE_MODULE) importlib.machinery $(srcdir)/Lib/importlib/machinery.py Python/frozen_modules/importlib.machinery.h
11561189

1157-
Python/frozen_modules/runpy.h: $(FREEZE_MODULE) Lib/runpy.py
1190+
Python/frozen_modules/runpy.h: Lib/runpy.py $(FREEZE_MODULE_DEPS)
11581191
$(FREEZE_MODULE) runpy $(srcdir)/Lib/runpy.py Python/frozen_modules/runpy.h
11591192

1160-
Python/frozen_modules/__hello__.h: $(FREEZE_MODULE) Lib/__hello__.py
1193+
Python/frozen_modules/__hello__.h: Lib/__hello__.py $(FREEZE_MODULE_DEPS)
11611194
$(FREEZE_MODULE) __hello__ $(srcdir)/Lib/__hello__.py Python/frozen_modules/__hello__.h
11621195

1163-
Python/frozen_modules/__phello__.h: $(FREEZE_MODULE) Lib/__phello__/__init__.py
1196+
Python/frozen_modules/__phello__.h: Lib/__phello__/__init__.py $(FREEZE_MODULE_DEPS)
11641197
$(FREEZE_MODULE) __phello__ $(srcdir)/Lib/__phello__/__init__.py Python/frozen_modules/__phello__.h
11651198

1166-
Python/frozen_modules/__phello__.ham.h: $(FREEZE_MODULE) Lib/__phello__/ham/__init__.py
1199+
Python/frozen_modules/__phello__.ham.h: Lib/__phello__/ham/__init__.py $(FREEZE_MODULE_DEPS)
11671200
$(FREEZE_MODULE) __phello__.ham $(srcdir)/Lib/__phello__/ham/__init__.py Python/frozen_modules/__phello__.ham.h
11681201

1169-
Python/frozen_modules/__phello__.ham.eggs.h: $(FREEZE_MODULE) Lib/__phello__/ham/eggs.py
1202+
Python/frozen_modules/__phello__.ham.eggs.h: Lib/__phello__/ham/eggs.py $(FREEZE_MODULE_DEPS)
11701203
$(FREEZE_MODULE) __phello__.ham.eggs $(srcdir)/Lib/__phello__/ham/eggs.py Python/frozen_modules/__phello__.ham.eggs.h
11711204

1172-
Python/frozen_modules/__phello__.spam.h: $(FREEZE_MODULE) Lib/__phello__/spam.py
1205+
Python/frozen_modules/__phello__.spam.h: Lib/__phello__/spam.py $(FREEZE_MODULE_DEPS)
11731206
$(FREEZE_MODULE) __phello__.spam $(srcdir)/Lib/__phello__/spam.py Python/frozen_modules/__phello__.spam.h
11741207

1175-
Python/frozen_modules/frozen_only.h: $(FREEZE_MODULE) Tools/freeze/flag.py
1208+
Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS)
11761209
$(FREEZE_MODULE) frozen_only $(srcdir)/Tools/freeze/flag.py Python/frozen_modules/frozen_only.h
11771210

11781211
# END: freezing modules
11791212

1180-
# We manually freeze getpath.py rather than through freeze_modules
1181-
Python/frozen_modules/getpath.h: $(FREEZE_MODULE) Modules/getpath.py
1182-
$(FREEZE_MODULE) getpath $(srcdir)/Modules/getpath.py Python/frozen_modules/getpath.h
1183-
11841213
Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
11851214

11861215
.PHONY: regen-frozen

‎Misc/NEWS.d/next/Build/2021-12-02-23-21-18.bpo-45949.OTSo9X.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Use pure Python ``freeze_module`` for all but importlib bootstrap files.
2+
``--with-freeze-module`` :program:`configure` option is no longer needed for
3+
cross builds.

‎Programs/_freeze_module.c‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55
This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen".
66
77
See Python/frozen.c for more info.
8+
9+
Keep this file in sync with Programs/_freeze_module.py.
810
*/
911

1012
#include <Python.h>

‎Programs/_freeze_module.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
"""Python implementation of Programs/_freeze_module.c
2+
3+
The pure Python implementation uses same functions and arguments as the C
4+
implementation.
5+
6+
The generated byte code is slightly different because
7+
compile() sets the PyCF_SOURCE_IS_UTF8 flag and objects have a
8+
reference count > 1. Marshal adds the `FLAG_REF` flag and creates a
9+
reference `hashtable`.
10+
"""
11+
12+
import marshal
13+
import sys
14+
15+
header = "/* Auto-generated by Programs/_freeze_module.py */"
16+
17+
18+
def read_text(inpath: str) -> bytes:
19+
with open(inpath, "rb") as f:
20+
return f.read()
21+
22+
23+
def compile_and_marshal(name: str, text: bytes) -> bytes:
24+
filename = f"<frozen {name}>"
25+
# exec == Py_file_input
26+
code = compile(text, filename, "exec", optimize=0, dont_inherit=True)
27+
return marshal.dumps(code)
28+
29+
30+
def get_varname(name: str, prefix: str) -> str:
31+
return f"{prefix}{name.replace('.', '_')}"
32+
33+
34+
def write_code(outfile, marshalled: bytes, varname: str) -> None:
35+
data_size = len(marshalled)
36+
37+
outfile.write(f"const unsigned char {varname}[] = {{\n")
38+
39+
for n in range(0, data_size, 16):
40+
outfile.write(" ")
41+
outfile.write(",".join(str(i) for i in marshalled[n : n + 16]))
42+
outfile.write(",\n")
43+
outfile.write("};\n")
44+
45+
46+
def write_frozen(outpath: str, inpath: str, name: str, marshalled: bytes) -> None:
47+
with open(outpath, "w") as outfile:
48+
outfile.write(header)
49+
outfile.write("\n")
50+
arrayname = get_varname(name, "_Py_M__")
51+
write_code(outfile, marshalled, arrayname)
52+
53+
54+
def main():
55+
if len(sys.argv) != 4:
56+
sys.exit("need to specify the name, input and output paths\n")
57+
58+
name = sys.argv[1]
59+
inpath = sys.argv[2]
60+
outpath = sys.argv[3]
61+
62+
text = read_text(inpath)
63+
marshalled = compile_and_marshal(name, text)
64+
write_frozen(outpath, inpath, name, marshalled)
65+
66+
67+
if __name__ == "__main__":
68+
main()

‎Tools/scripts/deepfreeze.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -393,13 +393,14 @@ def generate(self, name: str, obj: object) -> str:
393393
}
394394
"""
395395

396-
FROZEN_COMMENT = "/* Auto-generated by Programs/_freeze_module.c */"
396+
FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */"
397+
FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */"
397398

398399
FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*"
399400

400401

401402
def is_frozen_header(source: str) -> bool:
402-
return source.startswith(FROZEN_COMMENT)
403+
return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY))
403404

404405

405406
def decode_frozen_data(source: str) -> types.CodeType:

‎Tools/scripts/freeze_modules.py‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -264,6 +264,10 @@ def ispkg(self):
264264
else:
265265
return os.path.basename(self.pyfile) == '__init__.py'
266266

267+
@property
268+
def isbootstrap(self):
269+
return self.id in BOOTSTRAP
270+
267271

268272
def resolve_frozen_file(frozenid, destdir):
269273
"""Return the filename corresponding to the given frozen ID.
@@ -476,7 +480,7 @@ def regen_frozen(modules):
476480
indent = ' '
477481
lastsection = None
478482
for mod in modules:
479-
if mod.frozenid in BOOTSTRAP:
483+
if mod.isbootstrap:
480484
lines = bootstraplines
481485
elif mod.section == TESTS_SECTION:
482486
lines = testlines
@@ -585,10 +589,17 @@ def regen_makefile(modules):
585589
pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
586590
pyfiles.append(f'\t\t{pyfile} \\')
587591

588-
freeze = (f'$(FREEZE_MODULE) {src.frozenid} '
589-
f'$(srcdir)/{pyfile} {frozen_header}')
592+
if src.isbootstrap:
593+
freezecmd = '$(FREEZE_MODULE_BOOTSTRAP)'
594+
freezedep = '$(FREEZE_MODULE_BOOTSTRAP_DEPS)'
595+
else:
596+
freezecmd = '$(FREEZE_MODULE)'
597+
freezedep = '$(FREEZE_MODULE_DEPS)'
598+
599+
freeze = (f'{freezecmd} {src.frozenid} '
600+
f'$(srcdir)/{pyfile} {frozen_header}')
590601
rules.extend([
591-
f'{frozen_header}: $(FREEZE_MODULE) {pyfile}',
602+
f'{frozen_header}: {pyfile} {freezedep}',
592603
f'\t{freeze}',
593604
'',
594605
])

0 commit comments

Comments
 (0)