Merge branch 'main' into bump_dfxml_version_references

Author: ajnelson-nist

.github/workflows/continuous-integration.yml

@@ -12,7 +12,9 @@ jobs:
     strategy:
       matrix:
         os: ['ubuntu-latest', 'macos-latest']
-        python-version: [3.8, 3.9]
+        python-version:
+          - '3.8'
+          - '3.12'
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -35,6 +37,11 @@ jobs:
           sudo apt update
           sudo apt install --yes libxml2-utils
 
+      - name: Pre-commit Checks
+        run: |
+          pip -q install pre-commit
+          pre-commit run --all-files
+
       - name: Make check
         run: make check
 

.github/workflows/supply-chain.yml

@@ -29,7 +29,7 @@ jobs:
       matrix:
         python-version:
           - '3.8'
-          - '3.9'
+          - '3.12'
 
     steps:
     - uses: actions/checkout@v4

.gitignore

@@ -11,3 +11,4 @@ python/demo.dfxml
 .pytest_cache
 *.egg-info
 *.log
+.venv-pre-commit

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+  - repo: https://github.com/pycqa/isort
+    rev: 5.13.2
+    hooks:
+      - id: isort
+        name: isort (python)

CONTRIBUTE.md

@@ -1,6 +1,25 @@
 # Contributing to DFXML's Python code base
 
 
+## Pre-commit
+
+This project uses [the `pre-commit` tool](https://pre-commit.com/) for linting.
+
+`pre-commit` hooks into Git's commit machinery to run a set of linters and static analyzers over each change. To install `pre-commit` into Git's hooks, run one (not both) of the following sets of commands:
+
+```bash
+pip install pre-commit
+pre-commit --version
+pre-commit install
+```
+
+Or:
+
+```bash
+make
+```
+
+
 ## Installable tools versus in-place scripts
 
 The [`dfxml/bin/`](dfxml/bin/) directory contains scripts for interacting with DFXML.  Some of the tools are installed in the command-line `$PATH` when the `dfxml` package is installed.

Makefile

@@ -18,11 +18,18 @@ ifeq ($(shell basename $(SHELL)),sh)
 SHELL := $(shell which /bin/bash 2>/dev/null || which /usr/local/bin/bash)
 endif
 
-all:
+PYTHON3 ?= python3
+ifeq ($(PYTHON3),)
+$(error python3 not found)
+endif
+
+all: \
+  .venv-pre-commit/var/.pre-commit-built.log
 
 .PHONY: \
   check-mypy \
-  check-supply-chain
+  check-supply-chain \
+  check-supply-chain-pre-commit
 
 .git_submodule_init.done.log: .gitmodules
 	# Confirm dfxml_schema has been checked out at least once.
@@ -31,28 +38,86 @@ all:
 	test -r dependencies/dfxml_schema/dfxml.xsd
 	touch $@
 
+# This virtual environment is meant to be built once and then persist, even through 'make clean'.
+# If a recipe is written to remove this flag file, it should first run `pre-commit uninstall`.
+.venv-pre-commit/var/.pre-commit-built.log:
+	rm -rf .venv-pre-commit
+	test -r .pre-commit-config.yaml \
+	  || (echo "ERROR:Makefile:pre-commit is expected to install for this repository, but .pre-commit-config.yaml does not seem to exist." >&2 ; exit 1)
+	$(PYTHON3) -m venv \
+	  .venv-pre-commit
+	source .venv-pre-commit/bin/activate \
+	  && pip install \
+	    --upgrade \
+	    pip \
+	    setuptools \
+	    wheel
+	source .venv-pre-commit/bin/activate \
+	  && pip install \
+	    pre-commit
+	source .venv-pre-commit/bin/activate \
+	  && pre-commit install
+	mkdir -p \
+	  .venv-pre-commit/var
+	touch $@
+
 clean:
 	find . -name '*~' -exec rm {} \;
 	$(MAKE) \
 	  --directory tests \
 	  clean
 
 check: \
-  check-mypy 
+  .git_submodule_init.done.log \
+  .venv-pre-commit/var/.pre-commit-built.log
 	$(MAKE) \
+	  PYTHON3=$(PYTHON3) \
 	  SHELL=$(SHELL) \
 	  --directory tests \
 	  check
 
 check-mypy: \
   .git_submodule_init.done.log
 	$(MAKE) \
+	  PYTHON3=$(PYTHON3) \
 	  SHELL=$(SHELL) \
 	  --directory tests \
 	  check-mypy
 
 check-supply-chain: \
+  check-supply-chain-pre-commit \
   check-mypy
 
+# Update pre-commit configuration and use the updated config file to
+# review code.  Only have Make exit if 'pre-commit run' modifies files.
+check-supply-chain-pre-commit: \
+  .venv-pre-commit/var/.pre-commit-built.log
+	source .venv-pre-commit/bin/activate \
+	  && pre-commit autoupdate
+	git diff \
+	  --exit-code \
+	  .pre-commit-config.yaml \
+	  || ( \
+	      source .venv-pre-commit/bin/activate \
+	        && pre-commit run \
+	          --all-files \
+	          --config .pre-commit-config.yaml \
+	    ) \
+	    || git diff \
+	      --stat \
+	      --exit-code \
+	      || ( \
+	          echo \
+	            "WARNING:Makefile:pre-commit configuration can be updated.  It appears the updated would change file formatting." \
+	            >&2 \
+	            ; exit 1 \
+                )
+	@git diff \
+	  --exit-code \
+	  .pre-commit-config.yaml \
+	  || echo \
+	    "INFO:Makefile:pre-commit configuration can be updated.  It appears the update would not change file formatting." \
+	    >&2
+
 check-tools:
 	(cd tests/misc_object_tests;make check)

demos/demo_fiwalk_diskimage.py

@@ -6,10 +6,12 @@
 if the required buffer size exceeds available RAM!
 """
 
-import sys
 import io
+import sys
+
 from dfxml import fiwalk
 
+
 def writeDfxml(imageFile: str, outFile: str) -> None:
     """Generate filesystem metadata for disk image and and write resulting
     dfxml to file"""

demos/demo_mac_timeline.py

@@ -7,7 +7,6 @@
 sys.path.append( os.path.join(os.path.dirname(__file__), ".."))
 import dfxml
 
-
 timeline = []
 
 def process(fi):

demos/demo_mac_timeline_iter.py

@@ -22,6 +22,7 @@
 sys.path.append( os.path.join(os.path.dirname(__file__), ".."))
 import dfxml
 
+
 def main():
     if len(sys.argv) < 2:
         print("Usage: {} <filename.xml>".format(sys.argv[0]))

demos/demo_piecewise.py

@@ -9,10 +9,11 @@
 import sys
 
 sys.path.append( os.path.join(os.path.dirname(__file__), ".."))
-import dfxml
-
+import collections
+import math
+import sys
 
-import math,sys,collections
+import dfxml
 
 
 class SectorCorrelator: