From e22807e5a1b3296b2d2080320688ee985782294f Mon Sep 17 00:00:00 2001 From: Marc Herbert Date: Thu, 7 Feb 2019 13:39:59 -0800 Subject: [PATCH] doc: fix incremental build by fixing doxygen output mtime Doxygen doesn't support incremental builds. It could be ok because it doesn't take much time in this codebase. However it's not because it makes old output look new which has a cascade effect on sphinx: https://sourceforge.net/p/doxygen/mailman/message/36580807/ Make doxygen artificially smarter by saving and restoring modify timestamps on the filesystem for doxygen output files that haven't changed. On my system this brings down the time to run an incremental "make htmldocs" from 75s down to 8s (cmake -DKCONFIG_TURBO_MODE=1 used in both cases) This optimization speeds-up non-doxygen documentation work only. Signed-off-by: Marc Herbert --- doc/CMakeLists.txt | 20 ++++- doc/scripts/restore_modification_times.py | 104 ++++++++++++++++++++++ 2 files changed, 122 insertions(+), 2 deletions(-) create mode 100755 doc/scripts/restore_modification_times.py diff --git a/doc/CMakeLists.txt b/doc/CMakeLists.txt index 5cdfa9727ee76..e186800fea254 100644 --- a/doc/CMakeLists.txt +++ b/doc/CMakeLists.txt @@ -122,6 +122,22 @@ add_custom_target( -P ${ZEPHYR_BASE}/cmake/util/execute_process.cmake ) +# Doxygen doesn't support incremental builds. +# It could be ok because it's pretty fast. +# But it's not because it has a cascade effect on sphinx: +# https://sourceforge.net/p/doxygen/mailman/message/36580807/ +# For now this optimization speeds-up non-doxygen documentation work +# only (by one order of magnitude). +add_custom_target( + doxy_real_modified_times + COMMAND ${CMAKE_COMMAND} -E env + ${PYTHON_EXECUTABLE} scripts/restore_modification_times.py + --loglevel ERROR _build/doxygen/xml + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} +) + +add_dependencies(doxy_real_modified_times doxy) + add_custom_target( pristine COMMAND ${CMAKE_COMMAND} -P ${ZEPHYR_BASE}/cmake/pristine.cmake @@ -259,14 +275,14 @@ endif() # # Dependencies and final targets # -add_dependencies(html content doxy kconfig) +add_dependencies(html content doxy_real_modified_times kconfig) add_custom_target( htmldocs ) add_dependencies(htmldocs html) -add_dependencies(latex content doxy kconfig) +add_dependencies(latex content doxy_real_modified_times kconfig) add_custom_target( latexdocs diff --git a/doc/scripts/restore_modification_times.py b/doc/scripts/restore_modification_times.py new file mode 100755 index 0000000000000..8fc5c7b329a75 --- /dev/null +++ b/doc/scripts/restore_modification_times.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2019, Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 + + +import argparse +import shutil +import os +import sys +import filecmp +import logging + + +def main(): + parser = argparse.ArgumentParser( + description="""Maintains a shadow copy of generated files. Restores their previous + modification times when their content hasn't changed. This stops + build tools assuming generated files have changed when their + content has not. Doxygen for instance doesn't support + incremental builds and regenerates (XML,...) files which seem + new even when they haven't changed at all. This breaks + incremental builds for tools processing its output further. + Skips: %s.""" % filecmp.DEFAULT_IGNORES + ) + + parser.add_argument("newer", help="Location of the generated files monitored") + parser.add_argument("shadow_dir", help="backup location", nargs='?') + parser.add_argument("-l", "--loglevel", help="python logging level", + default="ERROR") + + args = parser.parse_args() + + # At the INFO level, running twice back to back should print nothing + # the second time. + logging.basicConfig(level=getattr(logging, args.loglevel)) + + # Strip any trailing slash + args.newer = os.path.normpath(args.newer) + + if args.shadow_dir is None: + args.shadow_dir = args.newer + "_shadow_files_stats" + + os.makedirs(args.shadow_dir, exist_ok=True) + + save_filestats_restore_mtimes(filecmp.dircmp(args.newer, args.shadow_dir)) + + +def save_filestats_restore_mtimes(dcmp): + "left = newer, right = shadow backup" + + for same_f in dcmp.same_files: + restore_older_mtime(os.path.join(dcmp.left, same_f), + os.path.join(dcmp.right, same_f)) + + for name in dcmp.left_only + dcmp.diff_files: + logging.info("Saving new object(s) to %s ", + os.path.join(dcmp.right, name)) + rsync(os.path.join(dcmp.left, name), + os.path.join(dcmp.right, name)) + + for name in dcmp.right_only: + obsolete = os.path.join(dcmp.right, name) + if os.path.isdir(obsolete) and not os.path.islink(obsolete): + logging.info("Cleaning up dir %s ", obsolete) + shutil.rmtree(obsolete) + else: + logging.info("Cleaning up file or link %s ", obsolete) + os.remove(obsolete) + + for sub_dcmp in dcmp.subdirs.values(): + logging.debug("Recursing into %s", sub_dcmp.left) + save_filestats_restore_mtimes(sub_dcmp) + + +def restore_older_mtime(newer, shadow): + newer_stat = os.lstat(newer) + newer_mtime = newer_stat.st_mtime_ns + shadow_mtime = os.lstat(shadow).st_mtime_ns + if shadow_mtime == newer_mtime: + logging.debug("Nothing to do for %s ", newer) + return + if shadow_mtime < newer_mtime: + logging.debug("Restoring mtime of unchanged %s ", newer) + os.utime(newer, ns=(newer_stat.st_atime_ns, shadow_mtime)) + return + if shadow_mtime > newer_mtime: + logging.error("Newer modified time on shadow file %s!", shadow) + sys.exit("Corrupted shadow, aborting.") + + +def rsync(src, dest): + if os.path.islink(src): + linkto = os.readlink(src) + os.symlink(linkto, dest) + elif os.path.isdir(src): + shutil.copytree(src, dest, symlinks=True) + else: + shutil.copy2(src, dest) + + +if __name__ == "__main__": + main()