From 6d6f312062bd0a1088ab6d64b06af47c881fdd8c Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 11 Jan 2022 18:30:12 +0530 Subject: [PATCH 01/15] improve it --- Tools/scripts/deepfreeze.py | 72 +++++++++++++------------------------ 1 file changed, 24 insertions(+), 48 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 002d680e10c2f6..1f0c0ab1c68a39 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -4,13 +4,10 @@ extension modules are not available. """ import argparse -import ast import builtins import collections import contextlib import os -import re -import sys import time import types from typing import Dict, FrozenSet, Tuple, TextIO @@ -35,6 +32,9 @@ def make_string_literal(b: bytes) -> str: return "".join(res) +def normalize_filename(modname: str)->str: + return modname.replace(".", "_").replace("/", "_") + CO_FAST_LOCAL = 0x20 CO_FAST_CELL = 0x40 CO_FAST_FREE = 0x80 @@ -104,8 +104,7 @@ def removesuffix(base: str, suffix: str) -> str: class Printer: - def __init__(self, file: TextIO): - self.level = 0 + def __init__(self, file: TextIO) -> None: self.file = file self.cache: Dict[Tuple[type, object], str] = {} self.hits, self.misses = 0, 0 @@ -344,6 +343,14 @@ def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: self.write("// TODO: The above tuple should be a frozenset") return ret + def generate_file(self, filename: str, code: object)-> None: + filename = normalize_filename(filename) + self.generate(f"{filename}_toplevel", code) + with self.block(f"static void {filename}_do_patchups(void)"): + for p in self.patchups: + self.write(p) + self.write(EPILOGUE.replace("%%NAME%%", filename)) + def generate(self, name: str, obj: object) -> str: # Use repr() in the key to distinguish -0.0 from +0.0 key = (type(obj), obj, repr(obj)) @@ -352,7 +359,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, types.CodeType) or isinstance(obj, umarshal.Code): + if isinstance(obj, (types.CodeType, umarshal.Code)) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -388,54 +395,26 @@ def generate(self, name: str, obj: object) -> str: PyObject * _Py_get_%%NAME%%_toplevel(void) { - do_patchups(); - return (PyObject *) &toplevel; + %%NAME%%_do_patchups(); + return (PyObject *) &%%NAME%%_toplevel; } """ -FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" -FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */" - -FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" - - -def is_frozen_header(source: str) -> bool: - return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY)) - -def decode_frozen_data(source: str) -> types.CodeType: - lines = source.splitlines() - while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None: - del lines[0] - while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: - del lines[-1] - values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip()) - data = bytes(values) - return umarshal.loads(data) - - -def generate(source: str, filename: str, modname: str, file: TextIO) -> None: - if is_frozen_header(source): - code = decode_frozen_data(source) - else: - code = compile(source, filename, "exec") - printer = Printer(file) - printer.generate("toplevel", code) - printer.write("") - with printer.block("static void do_patchups(void)"): - for p in printer.patchups: - printer.write(p) - here = os.path.dirname(__file__) - printer.write(EPILOGUE.replace("%%NAME%%", modname.replace(".", "_"))) +def generate(files: str, output: TextIO) -> None: + printer = Printer(output) + for file in files: + with open(file, "r", encoding="utf8") as fd: + code = compile(fd.read(),f'', "exec") + printer.generate_file(file, code) if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") parser = argparse.ArgumentParser() -parser.add_argument("-m", "--module", help="Defaults to basename(file)") parser.add_argument("-o", "--output", help="Defaults to MODULE.c") parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") -parser.add_argument("file", help="Input file (required)") +parser.add_argument("files", nargs='+', help="Input files (required)") @contextlib.contextmanager @@ -453,13 +432,10 @@ def main() -> None: global verbose args = parser.parse_args() verbose = args.verbose - with open(args.file, encoding="utf-8") as f: - source = f.read() - modname = args.module or removesuffix(os.path.basename(args.file), ".py") - output = args.output or modname + ".c" + output = args.output or 'deepfreeze.c' with open(output, "w", encoding="utf-8") as file: with report_time("generate"): - generate(source, f"", modname, file) + generate(args.files, file) if verbose: print(f"Wrote {os.path.getsize(output)} bytes to {output}") From 04a8a8bc8706ed86fb1698825bb0841de06d83a5 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 12 Jan 2022 11:29:05 +0530 Subject: [PATCH 02/15] deepfreeze for linux --- Makefile.pre.in | 98 +-------------------------------- Tools/scripts/deepfreeze.py | 28 +++++----- Tools/scripts/freeze_modules.py | 26 ++------- 3 files changed, 21 insertions(+), 131 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index fbd4c3a23fd81b..4eaba776cefaad 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -479,32 +479,8 @@ OBJECT_OBJS= \ Objects/unionobject.o \ Objects/weakrefobject.o -# DEEPFREEZE_OBJS is auto-generated by Tools/scripts/freeze_modules.py. DEEPFREEZE_OBJS = \ - Python/deepfreeze/importlib._bootstrap.o \ - Python/deepfreeze/importlib._bootstrap_external.o \ - Python/deepfreeze/zipimport.o \ - Python/deepfreeze/abc.o \ - Python/deepfreeze/codecs.o \ - Python/deepfreeze/io.o \ - Python/deepfreeze/_collections_abc.o \ - Python/deepfreeze/_sitebuiltins.o \ - Python/deepfreeze/genericpath.o \ - Python/deepfreeze/ntpath.o \ - Python/deepfreeze/posixpath.o \ - Python/deepfreeze/os.o \ - Python/deepfreeze/site.o \ - Python/deepfreeze/stat.o \ - Python/deepfreeze/importlib.util.o \ - Python/deepfreeze/importlib.machinery.o \ - Python/deepfreeze/runpy.o \ - Python/deepfreeze/__hello__.o \ - Python/deepfreeze/__phello__.o \ - Python/deepfreeze/__phello__.ham.o \ - Python/deepfreeze/__phello__.ham.eggs.o \ - Python/deepfreeze/__phello__.spam.o \ - Python/deepfreeze/frozen_only.o -# End DEEPFREEZE_OBJS + Python/deepfreeze/deepfreeze.o ########################################################################## # objects that get linked into the Python library @@ -992,76 +968,8 @@ regen-deepfreeze: $(DEEPFREEZE_OBJS) DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) # BEGIN: deepfreeze modules - -Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c - -Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c - -Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c - -Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c - -Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c - -Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c - -Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c - -Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c - -Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c - -Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c - -Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c - -Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c - -Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c - -Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c - -Python/deepfreeze/importlib.util.c: Python/frozen_modules/importlib.util.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib.util.h -m importlib.util -o Python/deepfreeze/importlib.util.c - -Python/deepfreeze/importlib.machinery.c: Python/frozen_modules/importlib.machinery.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib.machinery.h -m importlib.machinery -o Python/deepfreeze/importlib.machinery.c - -Python/deepfreeze/runpy.c: Python/frozen_modules/runpy.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/runpy.h -m runpy -o Python/deepfreeze/runpy.c - -Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c - -Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c - -Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c - -Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c - -Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c - -Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c - +Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Lib/importlib/_bootstrap.py importlib._bootstrap -i Lib/importlib/_bootstrap_external.py importlib._bootstrap_external -i Lib/zipimport.py zipimport -i Lib/abc.py abc -i Lib/codecs.py codecs -i Lib/io.py io -i Lib/_collections_abc.py _collections_abc -i Lib/_sitebuiltins.py _sitebuiltins -i Lib/genericpath.py genericpath -i Lib/ntpath.py ntpath -i Lib/posixpath.py posixpath -i Lib/os.py os -i Lib/site.py site -i Lib/stat.py stat -i Lib/importlib/util.py importlib.util -i Lib/importlib/machinery.py importlib.machinery -i Lib/runpy.py runpy -i Lib/__hello__.py __hello__ -i Lib/__phello__/__init__.py __phello__ -i Lib/__phello__/ham/__init__.py __phello__.ham -i Lib/__phello__/ham/eggs.py __phello__.ham.eggs -i Lib/__phello__/spam.py __phello__.spam -i Tools/freeze/flag.py frozen_only -o Python/deepfreeze/deepfreeze.c # END: deepfreeze modules ############################################################################ diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 1f0c0ab1c68a39..5aea4536a89b5b 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -32,9 +32,6 @@ def make_string_literal(b: bytes) -> str: return "".join(res) -def normalize_filename(modname: str)->str: - return modname.replace(".", "_").replace("/", "_") - CO_FAST_LOCAL = 0x20 CO_FAST_CELL = 0x40 CO_FAST_FREE = 0x80 @@ -106,7 +103,8 @@ class Printer: def __init__(self, file: TextIO) -> None: self.file = file - self.cache: Dict[Tuple[type, object], str] = {} + self.level = 0 + self.cache: Dict[tuple[object, object, str], str] = {} self.hits, self.misses = 0, 0 self.patchups: list[str] = [] self.write('#include "Python.h"') @@ -343,13 +341,13 @@ def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: self.write("// TODO: The above tuple should be a frozenset") return ret - def generate_file(self, filename: str, code: object)-> None: - filename = normalize_filename(filename) - self.generate(f"{filename}_toplevel", code) - with self.block(f"static void {filename}_do_patchups(void)"): + def generate_file(self, module: str, code: object)-> None: + module = module.replace(".", "_") + self.generate(f"{module}_toplevel", code) + with self.block(f"static void {module}_do_patchups(void)"): for p in self.patchups: self.write(p) - self.write(EPILOGUE.replace("%%NAME%%", filename)) + self.write(EPILOGUE.replace("%%NAME%%", module)) def generate(self, name: str, obj: object) -> str: # Use repr() in the key to distinguish -0.0 from +0.0 @@ -401,12 +399,12 @@ def generate(self, name: str, obj: object) -> str: """ -def generate(files: str, output: TextIO) -> None: +def generate(input: tuple[str,str], output: TextIO) -> None: printer = Printer(output) - for file in files: + for file, modname in input: with open(file, "r", encoding="utf8") as fd: - code = compile(fd.read(),f'', "exec") - printer.generate_file(file, code) + code = compile(fd.read(), f'', "exec") + printer.generate_file(modname, code) if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") @@ -414,7 +412,7 @@ def generate(files: str, output: TextIO) -> None: parser = argparse.ArgumentParser() parser.add_argument("-o", "--output", help="Defaults to MODULE.c") parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") -parser.add_argument("files", nargs='+', help="Input files (required)") +parser.add_argument("-i","--input", nargs=2, action='append', help="Input files (required)", metavar=('file', 'module')) @contextlib.contextmanager @@ -435,7 +433,7 @@ def main() -> None: output = args.output or 'deepfreeze.c' with open(output, "w", encoding="utf-8") as file: with report_time("generate"): - generate(args.files, file) + generate(args.input, file) if verbose: print(f"Wrote {os.path.getsize(output)} bytes to {output}") diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index cbe8bf1ce60cdc..567c5c89174c24 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -575,16 +575,12 @@ def regen_frozen(modules): def regen_makefile(modules): pyfiles = [] frozenfiles = [] - deepfreezefiles = [] rules = [''] - deepfreezerules = [''] + deepfreezerule = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)", + "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) - deepfreeze_header = relpath_for_posix_display(src.deepfreezefile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') - cfile = deepfreeze_header[:-2] + ".c" - ofile = deepfreeze_header[:-2] + ".o" - deepfreezefiles.append(f"\t\t{ofile} \\") pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') @@ -603,15 +599,10 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)') - deepfreezerules.append( - f"\t$(PYTHON_FOR_FREEZE) " - f"$(srcdir)/Tools/scripts/deepfreeze.py " - f"{frozen_header} -m {src.frozenid} -o {cfile}") - deepfreezerules.append('') + deepfreezerule[-1] += f" -i {pyfile} {src.frozenid}" + deepfreezerule[-1] += ' -o Python/deepfreeze/deepfreeze.c' pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") - deepfreezefiles[-1] = deepfreezefiles[-1].rstrip(" \\") print(f'# Updating {os.path.relpath(MAKEFILE)}') with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile): @@ -630,13 +621,6 @@ def regen_makefile(modules): frozenfiles, MAKEFILE, ) - lines = replace_block( - lines, - "DEEPFREEZE_OBJS =", - "# End DEEPFREEZE_OBJS", - deepfreezefiles, - MAKEFILE, - ) lines = replace_block( lines, "# BEGIN: freezing modules", @@ -648,7 +632,7 @@ def regen_makefile(modules): lines, "# BEGIN: deepfreeze modules", "# END: deepfreeze modules", - deepfreezerules, + deepfreezerule, MAKEFILE, ) outfile.writelines(lines) From c010a02d734474d1e9572fbc44f563256780c81c Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 12 Jan 2022 18:29:33 +0530 Subject: [PATCH 03/15] try to make it work on Windows --- PCbuild/_freeze_module.vcxproj | 65 +++------------------------------ PCbuild/pythoncore.vcxproj | 24 +----------- Tools/scripts/deepfreeze.py | 2 +- Tools/scripts/freeze_modules.py | 20 +++++++--- 4 files changed, 22 insertions(+), 89 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 59519cade26705..5d668d54d7c258 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -241,162 +241,116 @@ importlib._bootstrap $(IntDir)importlib._bootstrap.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap.h - $(IntDir)importlib._bootstrap.g.c - $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap.c importlib._bootstrap_external $(IntDir)importlib._bootstrap_external.g.h $(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h - $(IntDir)importlib._bootstrap_external.g.c - $(PySourcePath)Python\deepfreeze\df.importlib._bootstrap_external.c zipimport $(IntDir)zipimport.g.h $(PySourcePath)Python\frozen_modules\zipimport.h - $(IntDir)zipimport.g.c - $(PySourcePath)Python\deepfreeze\df.zipimport.c abc $(IntDir)abc.g.h $(PySourcePath)Python\frozen_modules\abc.h - $(IntDir)abc.g.c - $(PySourcePath)Python\deepfreeze\df.abc.c codecs $(IntDir)codecs.g.h $(PySourcePath)Python\frozen_modules\codecs.h - $(IntDir)codecs.g.c - $(PySourcePath)Python\deepfreeze\df.codecs.c io $(IntDir)io.g.h $(PySourcePath)Python\frozen_modules\io.h - $(IntDir)io.g.c - $(PySourcePath)Python\deepfreeze\df.io.c _collections_abc $(IntDir)_collections_abc.g.h $(PySourcePath)Python\frozen_modules\_collections_abc.h - $(IntDir)_collections_abc.g.c - $(PySourcePath)Python\deepfreeze\df._collections_abc.c _sitebuiltins $(IntDir)_sitebuiltins.g.h $(PySourcePath)Python\frozen_modules\_sitebuiltins.h - $(IntDir)_sitebuiltins.g.c - $(PySourcePath)Python\deepfreeze\df._sitebuiltins.c genericpath $(IntDir)genericpath.g.h $(PySourcePath)Python\frozen_modules\genericpath.h - $(IntDir)genericpath.g.c - $(PySourcePath)Python\deepfreeze\df.genericpath.c ntpath $(IntDir)ntpath.g.h $(PySourcePath)Python\frozen_modules\ntpath.h - $(IntDir)ntpath.g.c - $(PySourcePath)Python\deepfreeze\df.ntpath.c posixpath $(IntDir)posixpath.g.h $(PySourcePath)Python\frozen_modules\posixpath.h - $(IntDir)posixpath.g.c - $(PySourcePath)Python\deepfreeze\df.posixpath.c os $(IntDir)os.g.h $(PySourcePath)Python\frozen_modules\os.h - $(IntDir)os.g.c - $(PySourcePath)Python\deepfreeze\df.os.c site $(IntDir)site.g.h $(PySourcePath)Python\frozen_modules\site.h - $(IntDir)site.g.c - $(PySourcePath)Python\deepfreeze\df.site.c stat $(IntDir)stat.g.h $(PySourcePath)Python\frozen_modules\stat.h - $(IntDir)stat.g.c - $(PySourcePath)Python\deepfreeze\df.stat.c importlib.util $(IntDir)importlib.util.g.h $(PySourcePath)Python\frozen_modules\importlib.util.h - $(IntDir)importlib.util.g.c - $(PySourcePath)Python\deepfreeze\df.importlib.util.c importlib.machinery $(IntDir)importlib.machinery.g.h $(PySourcePath)Python\frozen_modules\importlib.machinery.h - $(IntDir)importlib.machinery.g.c - $(PySourcePath)Python\deepfreeze\df.importlib.machinery.c runpy $(IntDir)runpy.g.h $(PySourcePath)Python\frozen_modules\runpy.h - $(IntDir)runpy.g.c - $(PySourcePath)Python\deepfreeze\df.runpy.c __hello__ $(IntDir)__hello__.g.h $(PySourcePath)Python\frozen_modules\__hello__.h - $(IntDir)__hello__.g.c - $(PySourcePath)Python\deepfreeze\df.__hello__.c __phello__ $(IntDir)__phello__.g.h $(PySourcePath)Python\frozen_modules\__phello__.h - $(IntDir)__phello__.g.c - $(PySourcePath)Python\deepfreeze\df.__phello__.c __phello__.ham $(IntDir)__phello__.ham.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.h - $(IntDir)__phello__.ham.g.c - $(PySourcePath)Python\deepfreeze\df.__phello__.ham.c __phello__.ham.eggs $(IntDir)__phello__.ham.eggs.g.h $(PySourcePath)Python\frozen_modules\__phello__.ham.eggs.h - $(IntDir)__phello__.ham.eggs.g.c - $(PySourcePath)Python\deepfreeze\df.__phello__.ham.eggs.c __phello__.spam $(IntDir)__phello__.spam.g.h $(PySourcePath)Python\frozen_modules\__phello__.spam.h - $(IntDir)__phello__.spam.g.c - $(PySourcePath)Python\deepfreeze\df.__phello__.spam.c frozen_only $(IntDir)frozen_only.g.h $(PySourcePath)Python\frozen_modules\frozen_only.h - $(IntDir)frozen_only.g.c - $(PySourcePath)Python\deepfreeze\df.frozen_only.c @@ -424,11 +378,11 @@ Condition="'@(_UpdatedGetPath)' != ''" Importance="high" /> - + - + Condition="!Exists(%(None.OutFile)) or (Exists(%(None.IntFile)) and '$([System.IO.File]::ReadAllText(%(None.OutFile)).Replace(` `, ` `))' != '$([System.IO.File]::ReadAllText(%(None.IntFile)).Replace(` `, ` `))')"> @@ -439,16 +393,9 @@ AfterTargets="_RebuildFrozen" DependsOnTargets="FindPythonForBuild" Condition="$(Configuration) != 'PGUpdate'"> - - - - - - - + + + diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 83ae2f08749aa8..493dd7a09fe0a0 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -527,29 +527,7 @@ - - - - - - - - - - - - - - - - - - - - - - - + diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 5aea4536a89b5b..c9ebd92bde4505 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -431,7 +431,7 @@ def main() -> None: args = parser.parse_args() verbose = args.verbose output = args.output or 'deepfreeze.c' - with open(output, "w", encoding="utf-8") as file: + with open(output, "w+", encoding="utf-8") as file: with report_time("generate"): generate(args.input, file) if verbose: diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 567c5c89174c24..e2ab7a7ab23e04 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -642,26 +642,24 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] + deepfreezerules = ["\t') projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') projlines.append(f' $(PySourcePath){header}') - projlines.append(f' $(IntDir){deepintfile}') - projlines.append(f' $(PySourcePath){deepoutfile}') projlines.append(f' ') filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') + deepfreezerules[-1] += f' "-i" "$(PySourcePath){pyfile}" "{src.frozenid}"' + deepfreezerules[-1] += ' "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' - corelines.append(f' ') + corelines.append(f' ') print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}') with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): @@ -674,6 +672,16 @@ def regen_pcbuild(modules): PCBUILD_PROJECT, ) outfile.writelines(lines) + with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile): + lines = infile.readlines() + lines = replace_block( + lines, + '', + '', + deepfreezerules, + PCBUILD_PROJECT, + ) + outfile.writelines(lines) print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}') with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile): lines = infile.readlines() From a254084fea9891db60e688dc6b83a2c6397e5c0b Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 12 Jan 2022 18:47:45 +0530 Subject: [PATCH 04/15] seems to work --- PCbuild/_freeze_module.vcxproj | 2 +- Tools/scripts/deepfreeze.py | 28 +++++++++++++++++++++++++++- Tools/scripts/freeze_modules.py | 2 +- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 5d668d54d7c258..1e40978d2397bd 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -394,7 +394,7 @@ DependsOnTargets="FindPythonForBuild" Condition="$(Configuration) != 'PGUpdate'"> - + diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index c9ebd92bde4505..2d2287c50a7132 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -8,6 +8,8 @@ import collections import contextlib import os +import re +import ast import time import types from typing import Dict, FrozenSet, Tuple, TextIO @@ -398,12 +400,36 @@ def generate(self, name: str, obj: object) -> str: } """ +FROZEN_COMMENT_C = "/* Auto-generated by Programs/_freeze_module.c */" +FROZEN_COMMENT_PY = "/* Auto-generated by Programs/_freeze_module.py */" + +FROZEN_DATA_LINE = r"\s*(\d+,\s*)+\s*" + + +def is_frozen_header(source: str) -> bool: + return source.startswith((FROZEN_COMMENT_C, FROZEN_COMMENT_PY)) + + +def decode_frozen_data(source: str) -> types.CodeType: + lines = source.splitlines() + while lines and re.match(FROZEN_DATA_LINE, lines[0]) is None: + del lines[0] + while lines and re.match(FROZEN_DATA_LINE, lines[-1]) is None: + del lines[-1] + values: Tuple[int, ...] = ast.literal_eval("".join(lines).strip()) + data = bytes(values) + return umarshal.loads(data) + def generate(input: tuple[str,str], output: TextIO) -> None: printer = Printer(output) for file, modname in input: with open(file, "r", encoding="utf8") as fd: - code = compile(fd.read(), f'', "exec") + source = fd.read() + if is_frozen_header(source): + code = decode_frozen_data(source) + else: + code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) if verbose: print(f"Cache hits: {printer.hits}, misses: {printer.misses}") diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index e2ab7a7ab23e04..38463eb24f22d9 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -656,7 +656,7 @@ def regen_pcbuild(modules): filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezerules[-1] += f' "-i" "$(PySourcePath){pyfile}" "{src.frozenid}"' + deepfreezerules[-1] += f' "-i" "$(PySourcePath){header}" "{src.frozenid}"' deepfreezerules[-1] += ' "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' corelines.append(f' ') From 6209461450403db0c58c2ea05f6411d1eed99db3 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 12 Jan 2022 19:02:33 +0530 Subject: [PATCH 05/15] fix outdated info --- Tools/scripts/deepfreeze.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 2d2287c50a7132..a49be7c8f71c21 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -436,9 +436,10 @@ def generate(input: tuple[str,str], output: TextIO) -> None: parser = argparse.ArgumentParser() -parser.add_argument("-o", "--output", help="Defaults to MODULE.c") +parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c") parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") -parser.add_argument("-i","--input", nargs=2, action='append', help="Input files (required)", metavar=('file', 'module')) +parser.add_argument("-i","--input", nargs=2, action='append', help="Input file and module name (required)", + metavar=('file', 'module')) @contextlib.contextmanager From 2140b72639c0f0a8ceeff5657c4c54bceb03f02f Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 13 Jan 2022 09:26:12 +0530 Subject: [PATCH 06/15] fix linux --- Makefile.pre.in | 2 +- Tools/scripts/freeze_modules.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 4eaba776cefaad..efac59da61dae6 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -969,7 +969,7 @@ DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) # BEGIN: deepfreeze modules Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Lib/importlib/_bootstrap.py importlib._bootstrap -i Lib/importlib/_bootstrap_external.py importlib._bootstrap_external -i Lib/zipimport.py zipimport -i Lib/abc.py abc -i Lib/codecs.py codecs -i Lib/io.py io -i Lib/_collections_abc.py _collections_abc -i Lib/_sitebuiltins.py _sitebuiltins -i Lib/genericpath.py genericpath -i Lib/ntpath.py ntpath -i Lib/posixpath.py posixpath -i Lib/os.py os -i Lib/site.py site -i Lib/stat.py stat -i Lib/importlib/util.py importlib.util -i Lib/importlib/machinery.py importlib.machinery -i Lib/runpy.py runpy -i Lib/__hello__.py __hello__ -i Lib/__phello__/__init__.py __phello__ -i Lib/__phello__/ham/__init__.py __phello__.ham -i Lib/__phello__/ham/eggs.py __phello__.ham.eggs -i Lib/__phello__/spam.py __phello__.spam -i Tools/freeze/flag.py frozen_only -o Python/deepfreeze/deepfreeze.c + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Python/frozen_modules/importlib._bootstrap.h importlib._bootstrap -i Python/frozen_modules/importlib._bootstrap_external.h importlib._bootstrap_external -i Python/frozen_modules/zipimport.h zipimport -i Python/frozen_modules/abc.h abc -i Python/frozen_modules/codecs.h codecs -i Python/frozen_modules/io.h io -i Python/frozen_modules/_collections_abc.h _collections_abc -i Python/frozen_modules/_sitebuiltins.h _sitebuiltins -i Python/frozen_modules/genericpath.h genericpath -i Python/frozen_modules/ntpath.h ntpath -i Python/frozen_modules/posixpath.h posixpath -i Python/frozen_modules/os.h os -i Python/frozen_modules/site.h site -i Python/frozen_modules/stat.h stat -i Python/frozen_modules/importlib.util.h importlib.util -i Python/frozen_modules/importlib.machinery.h importlib.machinery -i Python/frozen_modules/runpy.h runpy -i Python/frozen_modules/__hello__.h __hello__ -i Python/frozen_modules/__phello__.h __phello__ -i Python/frozen_modules/__phello__.ham.h __phello__.ham -i Python/frozen_modules/__phello__.ham.eggs.h __phello__.ham.eggs -i Python/frozen_modules/__phello__.spam.h __phello__.spam -i Python/frozen_modules/frozen_only.h frozen_only -o Python/deepfreeze/deepfreeze.c # END: deepfreeze modules ############################################################################ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 38463eb24f22d9..99395b6ec1ce37 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -599,7 +599,7 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerule[-1] += f" -i {pyfile} {src.frozenid}" + deepfreezerule[-1] += f" -i {frozen_header} {src.frozenid}" deepfreezerule[-1] += ' -o Python/deepfreeze/deepfreeze.c' pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") @@ -642,7 +642,7 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] - deepfreezerules = ["\t Date: Thu, 13 Jan 2022 13:48:12 +0530 Subject: [PATCH 07/15] fix patchups not clearing --- Tools/scripts/deepfreeze.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index a49be7c8f71c21..5d7487b5a83533 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -349,6 +349,7 @@ def generate_file(self, module: str, code: object)-> None: with self.block(f"static void {module}_do_patchups(void)"): for p in self.patchups: self.write(p) + self.patchups.clear() self.write(EPILOGUE.replace("%%NAME%%", module)) def generate(self, name: str, obj: object) -> str: From a0dcf7c611c6e0323ca8ebc7e6f2b351aaf0cf38 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 13 Jan 2022 14:38:58 +0530 Subject: [PATCH 08/15] better formatting --- Makefile.pre.in | 1 + Tools/scripts/freeze_modules.py | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index efac59da61dae6..14d2f03e7db713 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -970,6 +970,7 @@ DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) # BEGIN: deepfreeze modules Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Python/frozen_modules/importlib._bootstrap.h importlib._bootstrap -i Python/frozen_modules/importlib._bootstrap_external.h importlib._bootstrap_external -i Python/frozen_modules/zipimport.h zipimport -i Python/frozen_modules/abc.h abc -i Python/frozen_modules/codecs.h codecs -i Python/frozen_modules/io.h io -i Python/frozen_modules/_collections_abc.h _collections_abc -i Python/frozen_modules/_sitebuiltins.h _sitebuiltins -i Python/frozen_modules/genericpath.h genericpath -i Python/frozen_modules/ntpath.h ntpath -i Python/frozen_modules/posixpath.h posixpath -i Python/frozen_modules/os.h os -i Python/frozen_modules/site.h site -i Python/frozen_modules/stat.h stat -i Python/frozen_modules/importlib.util.h importlib.util -i Python/frozen_modules/importlib.machinery.h importlib.machinery -i Python/frozen_modules/runpy.h runpy -i Python/frozen_modules/__hello__.h __hello__ -i Python/frozen_modules/__phello__.h __phello__ -i Python/frozen_modules/__phello__.ham.h __phello__.ham -i Python/frozen_modules/__phello__.ham.eggs.h __phello__.ham.eggs -i Python/frozen_modules/__phello__.spam.h __phello__.spam -i Python/frozen_modules/frozen_only.h frozen_only -o Python/deepfreeze/deepfreeze.c + # END: deepfreeze modules ############################################################################ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 99395b6ec1ce37..37df66ebe7928b 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -576,7 +576,7 @@ def regen_makefile(modules): pyfiles = [] frozenfiles = [] rules = [''] - deepfreezerule = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)", + deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)", "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) @@ -599,8 +599,9 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerule[-1] += f" -i {frozen_header} {src.frozenid}" - deepfreezerule[-1] += ' -o Python/deepfreeze/deepfreeze.c' + deepfreezerules[-1] += f" -i {frozen_header} {src.frozenid}" + deepfreezerules[-1] += ' -o Python/deepfreeze/deepfreeze.c' + deepfreezerules.append('') pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") @@ -632,7 +633,7 @@ def regen_makefile(modules): lines, "# BEGIN: deepfreeze modules", "# END: deepfreeze modules", - deepfreezerule, + deepfreezerules, MAKEFILE, ) outfile.writelines(lines) From d66570cf1e0ae0bdc440a28ccf659175df3dc0ce Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Tue, 18 Jan 2022 10:28:41 +0000 Subject: [PATCH 09/15] fix unix deps in Makefile --- Makefile.pre.in | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 14d2f03e7db713..fa522e073da93d 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -959,19 +959,6 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) -############################################################################ -# Deepfreeze targets - -.PHONY: regen-deepfreeze -regen-deepfreeze: $(DEEPFREEZE_OBJS) - -DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) - -# BEGIN: deepfreeze modules -Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Python/frozen_modules/importlib._bootstrap.h importlib._bootstrap -i Python/frozen_modules/importlib._bootstrap_external.h importlib._bootstrap_external -i Python/frozen_modules/zipimport.h zipimport -i Python/frozen_modules/abc.h abc -i Python/frozen_modules/codecs.h codecs -i Python/frozen_modules/io.h io -i Python/frozen_modules/_collections_abc.h _collections_abc -i Python/frozen_modules/_sitebuiltins.h _sitebuiltins -i Python/frozen_modules/genericpath.h genericpath -i Python/frozen_modules/ntpath.h ntpath -i Python/frozen_modules/posixpath.h posixpath -i Python/frozen_modules/os.h os -i Python/frozen_modules/site.h site -i Python/frozen_modules/stat.h stat -i Python/frozen_modules/importlib.util.h importlib.util -i Python/frozen_modules/importlib.machinery.h importlib.machinery -i Python/frozen_modules/runpy.h runpy -i Python/frozen_modules/__hello__.h __hello__ -i Python/frozen_modules/__phello__.h __phello__ -i Python/frozen_modules/__phello__.ham.h __phello__.ham -i Python/frozen_modules/__phello__.ham.eggs.h __phello__.ham.eggs -i Python/frozen_modules/__phello__.spam.h __phello__.spam -i Python/frozen_modules/frozen_only.h frozen_only -o Python/deepfreeze/deepfreeze.c - -# END: deepfreeze modules ############################################################################ # frozen modules (including importlib) @@ -1143,6 +1130,20 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN) $(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py @echo "The Makefile was updated, you may need to re-run make." +############################################################################ +# Deepfreeze targets + +.PHONY: regen-deepfreeze +regen-deepfreeze: $(DEEPFREEZE_OBJS) + +DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT) + +# BEGIN: deepfreeze modules +Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Python/frozen_modules/importlib._bootstrap.h importlib._bootstrap -i Python/frozen_modules/importlib._bootstrap_external.h importlib._bootstrap_external -i Python/frozen_modules/zipimport.h zipimport -i Python/frozen_modules/abc.h abc -i Python/frozen_modules/codecs.h codecs -i Python/frozen_modules/io.h io -i Python/frozen_modules/_collections_abc.h _collections_abc -i Python/frozen_modules/_sitebuiltins.h _sitebuiltins -i Python/frozen_modules/genericpath.h genericpath -i Python/frozen_modules/ntpath.h ntpath -i Python/frozen_modules/posixpath.h posixpath -i Python/frozen_modules/os.h os -i Python/frozen_modules/site.h site -i Python/frozen_modules/stat.h stat -i Python/frozen_modules/importlib.util.h importlib.util -i Python/frozen_modules/importlib.machinery.h importlib.machinery -i Python/frozen_modules/runpy.h runpy -i Python/frozen_modules/__hello__.h __hello__ -i Python/frozen_modules/__phello__.h __phello__ -i Python/frozen_modules/__phello__.ham.h __phello__.ham -i Python/frozen_modules/__phello__.ham.eggs.h __phello__.ham.eggs -i Python/frozen_modules/__phello__.spam.h __phello__.spam -i Python/frozen_modules/frozen_only.h frozen_only -o Python/deepfreeze/deepfreeze.c + +# END: deepfreeze modules + # We keep this renamed target around for folks with muscle memory. .PHONY: regen-importlib regen-importlib: regen-frozen From 863e0eeff5ef0d6951d7c1d81fb334cf5f40a3f3 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:03:25 +0530 Subject: [PATCH 10/15] code review round 1 --- Makefile.pre.in | 3 +-- Tools/scripts/deepfreeze.py | 14 +++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index fa522e073da93d..7385325c83cf3b 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -479,8 +479,7 @@ OBJECT_OBJS= \ Objects/unionobject.o \ Objects/weakrefobject.o -DEEPFREEZE_OBJS = \ - Python/deepfreeze/deepfreeze.o +DEEPFREEZE_OBJS = Python/deepfreeze/deepfreeze.o ########################################################################## # objects that get linked into the Python library diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 5d7487b5a83533..9ec2658da3a8e8 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -4,15 +4,15 @@ extension modules are not available. """ import argparse +import ast import builtins import collections import contextlib import os import re -import ast import time import types -from typing import Dict, FrozenSet, Tuple, TextIO +from typing import Dict, FrozenSet, TextIO, Tuple import umarshal @@ -105,8 +105,8 @@ class Printer: def __init__(self, file: TextIO) -> None: self.file = file - self.level = 0 self.cache: Dict[tuple[object, object, str], str] = {} + self.level = 0 self.hits, self.misses = 0, 0 self.patchups: list[str] = [] self.write('#include "Python.h"') @@ -349,7 +349,7 @@ def generate_file(self, module: str, code: object)-> None: with self.block(f"static void {module}_do_patchups(void)"): for p in self.patchups: self.write(p) - self.patchups.clear() + self.patchups.clear() self.write(EPILOGUE.replace("%%NAME%%", module)) def generate(self, name: str, obj: object) -> str: @@ -422,7 +422,7 @@ def decode_frozen_data(source: str) -> types.CodeType: return umarshal.loads(data) -def generate(input: tuple[str,str], output: TextIO) -> None: +def generate(input: tuple[str, str], output: TextIO) -> None: printer = Printer(output) for file, modname in input: with open(file, "r", encoding="utf8") as fd: @@ -458,8 +458,8 @@ def main() -> None: global verbose args = parser.parse_args() verbose = args.verbose - output = args.output or 'deepfreeze.c' - with open(output, "w+", encoding="utf-8") as file: + output = args.output or "deepfreeze.c" + with open(output, "w", encoding="utf-8") as file: with report_time("generate"): generate(args.input, file) if verbose: From ba2dfe714da8384c4745bc217d78eb5e567468ff Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 19 Jan 2022 04:36:15 +0000 Subject: [PATCH 11/15] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Misc/NEWS.d/next/Build/2022-01-19-04-36-15.bpo-46429.y0OtVL.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Build/2022-01-19-04-36-15.bpo-46429.y0OtVL.rst diff --git a/Misc/NEWS.d/next/Build/2022-01-19-04-36-15.bpo-46429.y0OtVL.rst b/Misc/NEWS.d/next/Build/2022-01-19-04-36-15.bpo-46429.y0OtVL.rst new file mode 100644 index 00000000000000..c983d9637fc897 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2022-01-19-04-36-15.bpo-46429.y0OtVL.rst @@ -0,0 +1 @@ +Merge all deep-frozen files into one for space savings. Patch by Kumar Aditya. \ No newline at end of file From ab644b9f97b5d4614beda2ca06d3753c0b243652 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:08:29 +0530 Subject: [PATCH 12/15] remove level change --- Tools/scripts/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 9ec2658da3a8e8..0581bd137ef3fe 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -104,9 +104,9 @@ def removesuffix(base: str, suffix: str) -> str: class Printer: def __init__(self, file: TextIO) -> None: + self.level = 0 self.file = file self.cache: Dict[tuple[object, object, str], str] = {} - self.level = 0 self.hits, self.misses = 0, 0 self.patchups: list[str] = [] self.write('#include "Python.h"') From acdd65546d3f4e69d243ab34282cd1d38c64aef0 Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 19 Jan 2022 10:22:35 +0530 Subject: [PATCH 13/15] chnage -i flag to just args --- Makefile.pre.in | 26 +++++++++++++++++++++++++- PCbuild/_freeze_module.vcxproj | 2 +- Tools/scripts/deepfreeze.py | 15 +++++++-------- Tools/scripts/freeze_modules.py | 8 ++++---- 4 files changed, 37 insertions(+), 14 deletions(-) diff --git a/Makefile.pre.in b/Makefile.pre.in index 7385325c83cf3b..68fbfcb84d4a29 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -1139,7 +1139,31 @@ DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FR # BEGIN: deepfreeze modules Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS) - $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py -i Python/frozen_modules/importlib._bootstrap.h importlib._bootstrap -i Python/frozen_modules/importlib._bootstrap_external.h importlib._bootstrap_external -i Python/frozen_modules/zipimport.h zipimport -i Python/frozen_modules/abc.h abc -i Python/frozen_modules/codecs.h codecs -i Python/frozen_modules/io.h io -i Python/frozen_modules/_collections_abc.h _collections_abc -i Python/frozen_modules/_sitebuiltins.h _sitebuiltins -i Python/frozen_modules/genericpath.h genericpath -i Python/frozen_modules/ntpath.h ntpath -i Python/frozen_modules/posixpath.h posixpath -i Python/frozen_modules/os.h os -i Python/frozen_modules/site.h site -i Python/frozen_modules/stat.h stat -i Python/frozen_modules/importlib.util.h importlib.util -i Python/frozen_modules/importlib.machinery.h importlib.machinery -i Python/frozen_modules/runpy.h runpy -i Python/frozen_modules/__hello__.h __hello__ -i Python/frozen_modules/__phello__.h __phello__ -i Python/frozen_modules/__phello__.ham.h __phello__.ham -i Python/frozen_modules/__phello__.ham.eggs.h __phello__.ham.eggs -i Python/frozen_modules/__phello__.spam.h __phello__.spam -i Python/frozen_modules/frozen_only.h frozen_only -o Python/deepfreeze/deepfreeze.c + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \ + Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \ + Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \ + Python/frozen_modules/zipimport.h:zipimport \ + Python/frozen_modules/abc.h:abc \ + Python/frozen_modules/codecs.h:codecs \ + Python/frozen_modules/io.h:io \ + Python/frozen_modules/_collections_abc.h:_collections_abc \ + Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \ + Python/frozen_modules/genericpath.h:genericpath \ + Python/frozen_modules/ntpath.h:ntpath \ + Python/frozen_modules/posixpath.h:posixpath \ + Python/frozen_modules/os.h:os \ + Python/frozen_modules/site.h:site \ + Python/frozen_modules/stat.h:stat \ + Python/frozen_modules/importlib.util.h:importlib.util \ + Python/frozen_modules/importlib.machinery.h:importlib.machinery \ + Python/frozen_modules/runpy.h:runpy \ + Python/frozen_modules/__hello__.h:__hello__ \ + Python/frozen_modules/__phello__.h:__phello__ \ + Python/frozen_modules/__phello__.ham.h:__phello__.ham \ + Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \ + Python/frozen_modules/__phello__.spam.h:__phello__.spam \ + Python/frozen_modules/frozen_only.h:frozen_only \ + -o Python/deepfreeze/deepfreeze.c # END: deepfreeze modules diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 1e40978d2397bd..e3cc7c79091a23 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -394,7 +394,7 @@ DependsOnTargets="FindPythonForBuild" Condition="$(Configuration) != 'PGUpdate'"> - + diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 0581bd137ef3fe..1d38882e3f01b0 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -422,9 +422,10 @@ def decode_frozen_data(source: str) -> types.CodeType: return umarshal.loads(data) -def generate(input: tuple[str, str], output: TextIO) -> None: +def generate(args: list[str], output: TextIO) -> None: printer = Printer(output) - for file, modname in input: + for arg in args: + file, modname = arg.rsplit(':', 1) with open(file, "r", encoding="utf8") as fd: source = fd.read() if is_frozen_header(source): @@ -437,11 +438,9 @@ def generate(input: tuple[str, str], output: TextIO) -> None: parser = argparse.ArgumentParser() -parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c") +parser.add_argument("-o", "--output", help="Defaults to deepfreeze.c", default="deepfreeze.c") parser.add_argument("-v", "--verbose", action="store_true", help="Print diagnostics") -parser.add_argument("-i","--input", nargs=2, action='append', help="Input file and module name (required)", - metavar=('file', 'module')) - +parser.add_argument('args', nargs="+", help="Input file and module name (required) in file:modname format") @contextlib.contextmanager def report_time(label: str): @@ -458,10 +457,10 @@ def main() -> None: global verbose args = parser.parse_args() verbose = args.verbose - output = args.output or "deepfreeze.c" + output = args.output with open(output, "w", encoding="utf-8") as file: with report_time("generate"): - generate(args.input, file) + generate(args.args, file) if verbose: print(f"Wrote {os.path.getsize(output)} bytes to {output}") diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index 37df66ebe7928b..b18582c50c75bb 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -577,7 +577,7 @@ def regen_makefile(modules): frozenfiles = [] rules = [''] deepfreezerules = ["Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)", - "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py"] + "\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \\"] for src in _iter_sources(modules): frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{frozen_header} \\') @@ -599,8 +599,8 @@ def regen_makefile(modules): f'\t{freeze}', '', ]) - deepfreezerules[-1] += f" -i {frozen_header} {src.frozenid}" - deepfreezerules[-1] += ' -o Python/deepfreeze/deepfreeze.c' + deepfreezerules.append(f"\t{frozen_header}:{src.frozenid} \\") + deepfreezerules.append('\t-o Python/deepfreeze/deepfreeze.c') deepfreezerules.append('') pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\") @@ -657,7 +657,7 @@ def regen_pcbuild(modules): filterlines.append(f' ') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezerules[-1] += f' "-i" "$(PySourcePath){header}" "{src.frozenid}"' + deepfreezerules[-1] += f' "$(PySourcePath){header}:{src.frozenid}"' deepfreezerules[-1] += ' "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' corelines.append(f' ') From d82eb9744f710c14fbf8c68e5d087ac8855430ec Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Wed, 19 Jan 2022 16:09:46 +0530 Subject: [PATCH 14/15] fix type annotations --- Tools/scripts/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/scripts/deepfreeze.py b/Tools/scripts/deepfreeze.py index 1d38882e3f01b0..d4ca02cb7367e1 100644 --- a/Tools/scripts/deepfreeze.py +++ b/Tools/scripts/deepfreeze.py @@ -106,7 +106,7 @@ class Printer: def __init__(self, file: TextIO) -> None: self.level = 0 self.file = file - self.cache: Dict[tuple[object, object, str], str] = {} + self.cache: Dict[tuple[type, object, str], str] = {} self.hits, self.misses = 0, 0 self.patchups: list[str] = [] self.write('#include "Python.h"') From ffe3b88e4ec785f0c9711a143d0dd89a1927c24a Mon Sep 17 00:00:00 2001 From: Kumar Aditya <59607654+kumaraditya303@users.noreply.github.com> Date: Thu, 20 Jan 2022 11:53:35 +0530 Subject: [PATCH 15/15] multiline MSBuild --- PCbuild/_freeze_module.vcxproj | 26 +++++++++++++++++++++++++- Tools/scripts/freeze_modules.py | 6 +++--- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index e3cc7c79091a23..0a74f5850a1e8e 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -394,7 +394,31 @@ DependsOnTargets="FindPythonForBuild" Condition="$(Configuration) != 'PGUpdate'"> - + diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index b18582c50c75bb..6d10758b5285cc 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -643,7 +643,7 @@ def regen_pcbuild(modules): projlines = [] filterlines = [] corelines = [] - deepfreezerules = ['\t') filterlines.append(' Python Files') filterlines.append(' ') - deepfreezerules[-1] += f' "$(PySourcePath){header}:{src.frozenid}"' - deepfreezerules[-1] += ' "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' + deepfreezerules.append(f'\t\t "$(PySourcePath){header}:{src.frozenid}" ^') + deepfreezerules.append('\t\t "-o" "$(PySourcePath)Python\\deepfreeze\\deepfreeze.c"\'/>' ) corelines.append(f' ')