Start Python formatting review with pre-commit and isort

This patch draws on Cyber Domain Ontology deployments from the noted
PRs.

References:
* casework/CASE-Utilities-Python#37
* Cyber-Domain-Ontology/CDO-Shapes-Example#1

Signed-off-by: Alex Nelson <[email protected]>

Author: ajnelson-nist

.github/workflows/continuous-integration.yml

@@ -37,6 +37,11 @@ jobs:
           sudo apt update
           sudo apt install --yes libxml2-utils
 
+      - name: Pre-commit Checks
+        run: |
+          pip -q install pre-commit
+          pre-commit run --all-files
+
       - name: Make check
         run: make check
 

.gitignore

@@ -11,3 +11,4 @@ python/demo.dfxml
 .pytest_cache
 *.egg-info
 *.log
+.venv-pre-commit

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)

Makefile

@@ -18,11 +18,18 @@ ifeq ($(shell basename $(SHELL)),sh)
 SHELL := $(shell which /bin/bash 2>/dev/null || which /usr/local/bin/bash)
 endif
 
-all:
+PYTHON3 ?= python3
+ifeq ($(PYTHON3),)
+$(error python3 not found)
+endif
+
+all: \
+  .venv-pre-commit/var/.pre-commit-built.log
 
 .PHONY: \
   check-mypy \
-  check-supply-chain
+  check-supply-chain \
+  check-supply-chain-pre-commit
 
 .git_submodule_init.done.log: .gitmodules
 	# Confirm dfxml_schema has been checked out at least once.
@@ -31,28 +38,86 @@ all:
 	test -r dependencies/dfxml_schema/dfxml.xsd
 	touch $@
 
+# This virtual environment is meant to be built once and then persist, even through 'make clean'.
+# If a recipe is written to remove this flag file, it should first run `pre-commit uninstall`.
+.venv-pre-commit/var/.pre-commit-built.log:
+	rm -rf .venv-pre-commit
+	test -r .pre-commit-config.yaml \
+	  || (echo "ERROR:Makefile:pre-commit is expected to install for this repository, but .pre-commit-config.yaml does not seem to exist." >&2 ; exit 1)
+	$(PYTHON3) -m venv \
+	  .venv-pre-commit
+	source .venv-pre-commit/bin/activate \
+	  && pip install \
+	    --upgrade \
+	    pip \
+	    setuptools \
+	    wheel
+	source .venv-pre-commit/bin/activate \
+	  && pip install \
+	    pre-commit
+	source .venv-pre-commit/bin/activate \
+	  && pre-commit install
+	mkdir -p \
+	  .venv-pre-commit/var
+	touch $@
+
 clean:
 	find . -name '*~' -exec rm {} \;
 	$(MAKE) \
 	  --directory tests \
 	  clean
 
 check: \
-  check-mypy 
+  .git_submodule_init.done.log \
+  .venv-pre-commit/var/.pre-commit-built.log
 	$(MAKE) \
+	  PYTHON3=$(PYTHON3) \
 	  SHELL=$(SHELL) \
 	  --directory tests \
 	  check
 
 check-mypy: \
   .git_submodule_init.done.log
 	$(MAKE) \
+	  PYTHON3=$(PYTHON3) \
 	  SHELL=$(SHELL) \
 	  --directory tests \
 	  check-mypy
 
 check-supply-chain: \
+  check-supply-chain-pre-commit \
   check-mypy
 
+# Update pre-commit configuration and use the updated config file to
+# review code.  Only have Make exit if 'pre-commit run' modifies files.
+check-supply-chain-pre-commit: \
+  .venv-pre-commit/var/.pre-commit-built.log
+	source .venv-pre-commit/bin/activate \
+	  && pre-commit autoupdate
+	git diff \
+	  --exit-code \
+	  .pre-commit-config.yaml \
+	  || ( \
+	      source .venv-pre-commit/bin/activate \
+	        && pre-commit run \
+	          --all-files \
+	          --config .pre-commit-config.yaml \
+	    ) \
+	    || git diff \
+	      --stat \
+	      --exit-code \
+	      || ( \
+	          echo \
+	            "WARNING:Makefile:pre-commit configuration can be updated.  It appears the updated would change file formatting." \
+	            >&2 \
+	            ; exit 1 \
+                )
+	@git diff \
+	  --exit-code \
+	  .pre-commit-config.yaml \
+	  || echo \
+	    "INFO:Makefile:pre-commit configuration can be updated.  It appears the update would not change file formatting." \
+	    >&2
+
 check-tools:
 	(cd tests/misc_object_tests;make check)

tests/Makefile

@@ -18,7 +18,7 @@ endif
 
 top_srcdir := $(shell cd .. ; pwd)
 
-PYTHON3 ?= $(shell which python3.9 2>/dev/null || which python3.8 2>/dev/null || which python3.7 2>/dev/null || which python3.6 2>/dev/null || which python3)
+PYTHON3 ?= python3
 ifeq ($(PYTHON3),)
 $(error python3 not found)
 endif