bpo-23404: make touch becomes make regen-all (#1405) (#1461)

Doc/library/sysconfig.rst

@@ -255,7 +255,6 @@ You can use :mod:`sysconfig` as a script with Python's *-m* option:
             AIX_GENUINE_CPLUSPLUS = "0"
             AR = "ar"
             ARFLAGS = "rc"
-            ASDLGEN = "./Parser/asdl_c.py"
             ...
 
 This call will print in the standard output the information returned by

Lib/distutils/sysconfig.py

@@ -97,7 +97,7 @@ def get_python_inc(plat_specific=0, prefix=None):
             if plat_specific:
                 return base
             if _sys_home:
-                incdir = os.path.join(_sys_home, get_config_var('AST_H_DIR'))
+                incdir = os.path.join(_sys_home, 'Include')
             else:
                 incdir = os.path.join(get_config_var('srcdir'), 'Include')
             return os.path.normpath(incdir)

Mac/BuildScript/build-installer.py

@@ -1168,9 +1168,6 @@ def buildPython():
         shellQuote(WORKDIR)[1:-1],
         shellQuote(WORKDIR)[1:-1]))
 
-    print("Running make touch")
-    runCommand("make touch")
-
     print("Running make")
     runCommand("make")
 

Makefile.pre.in

@@ -224,7 +224,7 @@ LIBOBJS=	@LIBOBJS@
 PYTHON=		python$(EXE)
 BUILDPYTHON=	python$(BUILDEXE)
 
-PYTHON_FOR_GEN=@PYTHON_FOR_GEN@
+PYTHON_FOR_REGEN=@PYTHON_FOR_REGEN@
 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
 _PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
 BUILD_GNU_TYPE=	@build@
@@ -271,11 +271,6 @@ IO_OBJS=	\
 		Modules/_io/stringio.o
 
 ##########################################################################
-# Grammar
-GRAMMAR_H=	Include/graminit.h
-GRAMMAR_C=	Python/graminit.c
-GRAMMAR_INPUT=	$(srcdir)/Grammar/Grammar
-
 
 LIBFFI_INCLUDEDIR=	@LIBFFI_INCLUDEDIR@
 
@@ -337,38 +332,9 @@ PARSER_HEADERS= \
 PGENSRCS=	$(PSRCS) $(PGSRCS)
 PGENOBJS=	$(POBJS) $(PGOBJS)
 
-##########################################################################
-# opcode.h generation
-OPCODE_H_DIR= 	$(srcdir)/Include
-OPCODE_H_SCRIPT= $(srcdir)/Tools/scripts/generate_opcode_h.py
-OPCODE_H=	$(OPCODE_H_DIR)/opcode.h
-OPCODE_H_GEN=	$(PYTHON_FOR_GEN) $(OPCODE_H_SCRIPT) $(srcdir)/Lib/opcode.py $(OPCODE_H)
-#
-##########################################################################
-# AST
-AST_H_DIR=	Include
-AST_H=		$(AST_H_DIR)/Python-ast.h
-AST_C_DIR=	Python
-AST_C=		$(AST_C_DIR)/Python-ast.c
-AST_ASDL=	$(srcdir)/Parser/Python.asdl
-
-ASDLGEN_FILES=	$(srcdir)/Parser/asdl.py $(srcdir)/Parser/asdl_c.py
-# Note that a build now requires Python to exist before the build starts.
-# Use "hg touch" to fix up screwed up file mtimes in a checkout.
-ASDLGEN=	$(PYTHON_FOR_GEN) $(srcdir)/Parser/asdl_c.py
-
 ##########################################################################
 # Python
 
-OPCODETARGETS_H= \
-		Python/opcode_targets.h
-
-OPCODETARGETGEN= \
-		$(srcdir)/Python/makeopcodetargets.py
-
-OPCODETARGETGEN_FILES= \
-		$(OPCODETARGETGEN) $(srcdir)/Lib/opcode.py
-
 PYTHON_OBJS=	\
 		Python/_warnings.o \
 		Python/Python-ast.o \
@@ -551,7 +517,8 @@ coverage-lcov:
 	@echo "lcov report at $(COVERAGE_REPORT)/index.html"
 	@echo
 
-coverage-report:
+# Force regeneration of parser and importlib
+coverage-report: regen-grammar regen-importlib
 	: # force rebuilding of parser and importlib
 	@touch $(GRAMMAR_INPUT)
 	@touch $(srcdir)/Lib/importlib/_bootstrap.py
@@ -721,14 +688,24 @@ Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile
 Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
 	$(LINKCC) $(PY_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) $(LDLAST)
 
-Python/importlib_external.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap_external.py Programs/_freeze_importlib
+.PHONY: regen-importlib
+regen-importlib: Programs/_freeze_importlib
+	# Regenerate Python/importlib_external.h
+	# from Lib/importlib/_bootstrap_external.py using _freeze_importlib
 	./Programs/_freeze_importlib \
-	    $(srcdir)/Lib/importlib/_bootstrap_external.py Python/importlib_external.h
-
-Python/importlib.h: @GENERATED_COMMENT@ $(srcdir)/Lib/importlib/_bootstrap.py Programs/_freeze_importlib
+	    $(srcdir)/Lib/importlib/_bootstrap_external.py \
+	    $(srcdir)/Python/importlib_external.h
+	# Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py
+	# using _freeze_importlib
 	./Programs/_freeze_importlib \
-	    $(srcdir)/Lib/importlib/_bootstrap.py Python/importlib.h
+	    $(srcdir)/Lib/importlib/_bootstrap.py \
+	    $(srcdir)/Python/importlib.h
+
+
+############################################################################
+# Regenerate all generated files
 
+regen-all: regen-opcode regen-opcode-targets regen-typeslots regen-grammar regen-ast regen-importlib
 
 ############################################################################
 # Special rules for object files
@@ -787,15 +764,18 @@ Python/sysmodule.o: $(srcdir)/Python/sysmodule.c Makefile
 
 $(IO_OBJS): $(IO_H)
 
-$(GRAMMAR_H): @GENERATED_COMMENT@ $(GRAMMAR_INPUT) $(PGEN)
-	@$(MKDIR_P) Include
-	$(PGEN) $(GRAMMAR_INPUT) $(GRAMMAR_H) $(GRAMMAR_C)
-$(GRAMMAR_C): @GENERATED_COMMENT@ $(GRAMMAR_H)
-	touch $(GRAMMAR_C)
-
 $(PGEN): $(PGENOBJS)
 		$(CC) $(OPT) $(PY_LDFLAGS) $(PGENOBJS) $(LIBS) -o $(PGEN)
 
+.PHONY: regen-grammar
+regen-grammar: $(PGEN)
+	# Regenerate Include/graminit.h and Python/graminit.c
+	# from Grammar/Grammar using pgen
+	@$(MKDIR_P) Include
+	$(PGEN) $(srcdir)/Grammar/Grammar \
+		$(srcdir)/Include/graminit.h \
+		$(srcdir)/Python/graminit.c
+
 Parser/grammar.o:	$(srcdir)/Parser/grammar.c \
 				$(srcdir)/Include/token.h \
 				$(srcdir)/Include/grammar.h
@@ -807,18 +787,28 @@ Parser/printgrammar.o: $(srcdir)/Parser/printgrammar.c
 
 Parser/pgenmain.o:	$(srcdir)/Include/parsetok.h
 
-$(AST_H): $(AST_ASDL) $(ASDLGEN_FILES)
-	$(MKDIR_P) $(AST_H_DIR)
-	$(ASDLGEN) -h $(AST_H_DIR) $(AST_ASDL)
-
-$(AST_C): $(AST_H) $(AST_ASDL) $(ASDLGEN_FILES)
-	$(MKDIR_P) $(AST_C_DIR)
-	$(ASDLGEN) -c $(AST_C_DIR) $(AST_ASDL)
-
-$(OPCODE_H): $(srcdir)/Lib/opcode.py $(OPCODE_H_SCRIPT)
-	$(OPCODE_H_GEN)
-
-Python/compile.o Python/symtable.o Python/ast.o: $(GRAMMAR_H) $(AST_H)
+.PHONY=regen-ast
+regen-ast:
+	# Regenerate Include/Python-ast.h using Parser/asdl_c.py -h
+	$(MKDIR_P) $(srcdir)/Include
+	$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
+		-h $(srcdir)/Include \
+		$(srcdir)/Parser/Python.asdl
+	# Regenerate Python/Python-ast.c using Parser/asdl_c.py -c
+	$(MKDIR_P) $(srcdir)/Python
+	$(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \
+		-c $(srcdir)/Python \
+		$(srcdir)/Parser/Python.asdl
+
+.PHONY: regen-opcode
+regen-opcode:
+	# Regenerate Include/opcode.h from Lib/opcode.py
+	# using Tools/scripts/generate_opcode_h.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
+		$(srcdir)/Lib/opcode.py \
+		$(srcdir)/Include/opcode.h
+
+Python/compile.o Python/symtable.o Python/ast.o: $(srcdir)/Include/graminit.h $(srcdir)/Include/Python-ast.h
 
 Python/getplatform.o: $(srcdir)/Python/getplatform.c
 		$(CC) -c $(PY_CORE_CFLAGS) -DPLATFORM='"$(MACHDEP)"' -o $@ $(srcdir)/Python/getplatform.c
@@ -867,16 +857,26 @@ Objects/unicodeobject.o: $(srcdir)/Objects/unicodeobject.c $(UNICODE_DEPS)
 Objects/dictobject.o: $(srcdir)/Objects/stringlib/eq.h
 Objects/setobject.o: $(srcdir)/Objects/stringlib/eq.h
 
-$(OPCODETARGETS_H): $(OPCODETARGETGEN_FILES)
-	$(PYTHON_FOR_GEN) $(OPCODETARGETGEN) $(OPCODETARGETS_H)
+.PHONY: regen-opcode-targets
+regen-opcode-targets:
+	# Regenerate Python/opcode_targets.h from Lib/opcode.py
+	# using Python/makeopcodetargets.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Python/makeopcodetargets.py \
+		$(srcdir)/Python/opcode_targets.h
 
-Python/ceval.o: $(OPCODETARGETS_H) $(srcdir)/Python/ceval_gil.h
+Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h
 
-Python/frozen.o: Python/importlib.h Python/importlib_external.h
+Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h
 
 Objects/typeobject.o: Objects/typeslots.inc
-Objects/typeslots.inc: $(srcdir)/Include/typeslots.h $(srcdir)/Objects/typeslots.py
-	$(PYTHON_FOR_GEN) $(srcdir)/Objects/typeslots.py < $(srcdir)/Include/typeslots.h Objects/typeslots.inc
+
+.PHONY: regen-typeslots
+regen-typeslots:
+	# Regenerate Objects/typeslots.inc from Include/typeslotsh
+	# using Objects/typeslots.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Objects/typeslots.py \
+		< $(srcdir)/Include/typeslots.h \
+		$(srcdir)/Objects/typeslots.inc
 
 ############################################################################
 # Header files
@@ -929,7 +929,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/node.h \
 		$(srcdir)/Include/object.h \
 		$(srcdir)/Include/objimpl.h \
-		$(OPCODE_H) \
+		$(srcdir)/Include/opcode.h \
 		$(srcdir)/Include/osdefs.h \
 		$(srcdir)/Include/patchlevel.h \
 		$(srcdir)/Include/pgen.h \
@@ -970,7 +970,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/weakrefobject.h \
 		pyconfig.h \
 		$(PARSER_HEADERS) \
-		$(AST_H)
+		$(srcdir)/Include/Python-ast.h
 
 $(LIBRARY_OBJS) $(MODOBJS) Programs/python.o: $(PYTHON_HEADERS)
 
@@ -1551,9 +1551,12 @@ recheck:
 	$(SHELL) config.status --recheck
 	$(SHELL) config.status
 
-# Rebuild the configure script from configure.ac; also rebuild pyconfig.h.in
+# Regenerate configure and pyconfig.h.in
+.PHONY: autoconf
 autoconf:
+	# Regenerate the configure script from configure.ac using autoconf
 	(cd $(srcdir); autoconf -Wall)
+	# Regenerate pyconfig.h.in from configure.ac using autoheader
 	(cd $(srcdir); autoheader -Wall)
 
 # Create a tags file for vi
@@ -1570,14 +1573,6 @@ TAGS::
 	etags Include/*.h; \
 	for i in $(SRCDIRS); do etags -a $$i/*.[ch]; done
 
-# This fixes up the mtimes of checked-in generated files, assuming that they
-# only *appear* to be outdated because of checkout order.
-# This is run while preparing a source release tarball, and can be run manually
-# to avoid bootstrap issues.
-touch:
-	cd $(srcdir); \
-	hg --config extensions.touch=Tools/hg/hgtouch.py touch -v
-
 # Sanitation targets -- clean leaves libraries, executables and tags
 # files, which clobber removes as well
 pycremoval:
@@ -1695,7 +1690,7 @@ Python/thread.o: @THREADHEADERS@
 .PHONY: maninstall libinstall inclinstall libainstall sharedinstall
 .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure
 .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools
-.PHONY: frameworkaltinstallunixtools recheck autoconf clean clobber distclean
+.PHONY: frameworkaltinstallunixtools recheck clean clobber distclean
 .PHONY: smelly funny patchcheck touch altmaninstall commoninstall
 .PHONY: gdbhooks
 

Misc/NEWS

@@ -203,6 +203,10 @@ Documentation
 Build
 -----
 
+- bpo-23404: Don't regenerate generated files based on file modification time
+  anymore: the action is now explicit. Replace ``make touch`` with
+  ``make regen-all``.
+
 - bpo-29643: Fix ``--enable-optimization`` didn't work.
 
 Tests