simplify CI scripts

Author: rmorshea

.github/workflows/test.yml

@@ -11,31 +11,28 @@ on:
     - cron: "0 0 * * *"
 
 jobs:
-  test-python:
+  test-python-coverage:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: nanasess/setup-chromedriver@master
       - uses: actions/setup-node@v2-beta
         with:
-          node-version: "14.x"
+          node-version: "14"
       - name: Use Latest Python
         uses: actions/setup-python@v2
         with:
-          python-version: 3.9
-      - name: Install latest NPM
-        run: |
-          npm install -g [email protected]
-          npm --version
+          python-version: "3.10"
       - name: Install Python Dependencies
         run: pip install -r requirements/test-run.txt
       - name: Run Tests
-        run: nox -s test -- --headless
-  test-python-versions:
+        env: {"CI": "true"}
+        run: nox -s test_python_suite -- --headless
+  test-python-environments:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
         os: [ubuntu-latest, macos-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2
@@ -47,60 +44,36 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Install latest NPM
-        run: |
-          npm install -g [email protected]
-          npm --version
       - name: Install Python Dependencies
         run: pip install -r requirements/test-run.txt
       - name: Run Tests
-        run: nox -s test_short -- --headless --no-cov
-  test-javascript:
+        env: {"CI": "true"}
+        run: nox -s test_python -- --headless --no-cov
+  test-docs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2
+      - uses: actions/setup-node@v2-beta
         with:
           node-version: "14"
-      - name: Install latest NPM
-        run: |
-          npm install -g [email protected]
-          npm --version
-      - name: Test Javascript
-        working-directory: ./src/client
-        run: |
-          npm install
-          npm test
-          npm run build
-  test-documentation-image:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out src from Git
-        uses: actions/checkout@v2
-      - name: Get history and tags for SCM versioning to work
-        run: |
-          git fetch --prune --unshallow
-          git fetch --depth=1 origin +refs/tags/*:refs/tags/*
-      - name: Build Docker Image
-        run: docker build . --file docs/Dockerfile
-  test-build-package:
+      - name: Use Latest Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+      - name: Install Python Dependencies
+        run: pip install -r requirements/test-run.txt
+      - name: Run Tests
+        env: {"CI": "true"}
+        run: nox -s test_docs
+  test-javascript:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v2-beta
         with:
           node-version: "14"
-      - name: Use Latest Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.9
-      - name: Install latest NPM
-        run: |
-          npm install -g [email protected]
-          npm --version
       - name: Install Python Dependencies
-        run: |
-          pip install --upgrade pip
-          pip install -r requirements/build-pkg.txt
-      - name: Test Build Creation
-        run: python setup.py bdist_wheel sdist
+        run: pip install -r requirements/test-run.txt
+      - name: Run Tests
+        env: {"CI": "true"}
+        run: nox -s test_javascript

docs/source/developing-idom/contributor-guide.rst

@@ -94,6 +94,9 @@ In order to develop IDOM locally you'll first need to install the following:
     *   - NPM >= 7.13
         - https://docs.npmjs.com/try-the-latest-stable-version-of-npm
 
+    *   - Docker
+        - https://docs.docker.com/get-docker/
+
 .. note::
 
     NodeJS distributes a version of NPM, but you'll want to get the latest

noxfile.py

@@ -4,7 +4,7 @@
 import os
 import re
 from pathlib import Path
-from typing import Any, Callable
+from typing import Any, Callable, TypeVar
 
 import nox
 from nox.sessions import Session
@@ -13,22 +13,56 @@
 ROOT = Path(__file__).parent
 SRC = ROOT / "src"
 POSARGS_PATTERN = re.compile(r"^(\w+)\[(.+)\]$")
+TRUE_VALUES = {"true", "True", "TRUE", "1"}
 
 
-def apply_standard_pip_upgrades(
-    function: Callable[[Session], Any]
-) -> Callable[[Session], Any]:
-    @functools.wraps(function)
-    def wrapper(session: Session) -> None:
-        session.install("--upgrade", "pip", "setuptools", "wheel")
-        return function(session)
+_Return = TypeVar("_Return")
 
-    return wrapper
+
+def do_first(
+    first_session_func: Callable[[Session], None]
+) -> Callable[[Callable[[Session], _Return]], Callable[[Session], _Return]]:
+    """Decorator for functions defining session actions that should happen first
+
+    >>> @do_first
+    >>> def setup(session):
+    >>>     ...  # do some setup
+    >>>
+    >>> @setup
+    >>> def the_actual_session(session):
+    >>>     ...  # so actual work
+
+    This makes it quick an easy to define common setup actions.
+    """
+
+    def setup(
+        second_session_func: Callable[[Session], _Return]
+    ) -> Callable[[Session], _Return]:
+        @functools.wraps(second_session_func)
+        def wrapper(session: Session) -> Any:
+            first_session_func(session)
+            return second_session_func(session)
+
+        return wrapper
+
+    return setup
+
+
+@do_first
+def apply_standard_pip_upgrades(session: Session) -> None:
+    session.install("--upgrade", "pip")
+
+
+@do_first
+def install_latest_npm_in_ci(session: Session) -> None:
+    if os.environ.get("CI") in TRUE_VALUES:
+        session.run("npm", "install", "-g", "npm@latest")
 
 
 @nox.session(reuse_venv=True)
 @apply_standard_pip_upgrades
 def format(session: Session) -> None:
+    """Auto format Python and Javascript code"""
     # format Python
     install_requirements_file(session, "check-style")
     session.run("black", ".")
@@ -58,6 +92,7 @@ def example(session: Session) -> None:
 
 
 @nox.session(reuse_venv=True)
+@install_latest_npm_in_ci
 @apply_standard_pip_upgrades
 def docs(session: Session) -> None:
     """Build and display documentation in the browser (automatically reloads on change)"""
@@ -115,23 +150,31 @@ def docs_in_docker(session: Session) -> None:
 def test(session: Session) -> None:
     """Run the complete test suite"""
     session.notify("test_python", posargs=session.posargs)
-    session.notify("test_types")
-    session.notify("test_style")
     session.notify("test_docs")
     session.notify("test_javascript")
 
 
 @nox.session
-def test_short(session: Session) -> None:
-    """Run a shortened version of the test suite"""
-    session.notify("test_python", posargs=session.posargs)
-    session.notify("test_docs")
-    session.notify("test_javascript")
+def test_python(session: Session) -> None:
+    """Run all Python checks"""
+    session.notify("test_python_suite", posargs=session.posargs)
+    session.notify("test_python_types")
+    session.notify("test_python_style")
+    session.notify("test_python_build")
+
+
+@nox.session
+def test_javascript(session: Session) -> None:
+    """Run all Javascript checks"""
+    session.notify("test_javascript_suite")
+    session.notify("test_javascript_build")
+    session.notify("test_javascript_style")
 
 
 @nox.session
+@install_latest_npm_in_ci
 @apply_standard_pip_upgrades
-def test_python(session: Session) -> None:
+def test_python_suite(session: Session) -> None:
     """Run the Python-based test suite"""
     session.env["IDOM_DEBUG_MODE"] = "1"
     install_requirements_file(session, "test-env")
@@ -149,8 +192,8 @@ def test_python(session: Session) -> None:
 
 @nox.session
 @apply_standard_pip_upgrades
-def test_types(session: Session) -> None:
-    """Perform a static type analysis of the codebase"""
+def test_python_types(session: Session) -> None:
+    """Perform a static type analysis of the Python codebase"""
     install_requirements_file(session, "check-types")
     install_requirements_file(session, "pkg-deps")
     install_requirements_file(session, "pkg-extras")
@@ -159,8 +202,8 @@ def test_types(session: Session) -> None:
 
 @nox.session
 @apply_standard_pip_upgrades
-def test_style(session: Session) -> None:
-    """Check that style guidelines are being followed"""
+def test_python_style(session: Session) -> None:
+    """Check that Python style guidelines are being followed"""
     install_requirements_file(session, "check-style")
     session.run("flake8", "src/idom", "tests", "docs")
     black_default_exclude = r"\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist"
@@ -176,6 +219,15 @@ def test_style(session: Session) -> None:
 
 @nox.session
 @apply_standard_pip_upgrades
+def test_python_build(session: Session) -> None:
+    """Test whether the Python package can be build for distribution"""
+    install_requirements_file(session, "build-pkg")
+    session.run("python", "setup.py", "bdist_wheel", "sdist")
+
+
+@nox.session
+@install_latest_npm_in_ci
+@apply_standard_pip_upgrades
 def test_docs(session: Session) -> None:
     """Verify that the docs build and that doctests pass"""
     install_requirements_file(session, "build-docs")
@@ -192,19 +244,47 @@ def test_docs(session: Session) -> None:
         "docs/build",
     )
     session.run("sphinx-build", "-b", "doctest", "docs/source", "docs/build")
+    # ensure docker image build works too
+    session.run("docker", "build", ".", "--file", "docs/Dockerfile", external=True)
 
 
-@nox.session
-def test_javascript(session: Session) -> None:
-    """Run the Javascript-based test suite and ensure it bundles succesfully"""
+@do_first
+@install_latest_npm_in_ci
+def setup_client_env(session: Session) -> None:
     session.chdir(SRC / "client")
     session.run("npm", "install", external=True)
+
+
+@nox.session
+@setup_client_env
+def test_javascript_suite(session: Session) -> None:
+    """Run the Javascript-based test suite and ensure it bundles succesfully"""
+    session.run("npm", "run", "test", external=True)
+
+
+@nox.session
+@setup_client_env
+def test_javascript_build(session: Session) -> None:
+    """Run the Javascript-based test suite and ensure it bundles succesfully"""
     session.run("npm", "run", "test", external=True)
+
+
+@nox.session
+@setup_client_env
+def test_javascript_style(session: Session) -> None:
+    """Check that Javascript style guidelines are being followed"""
+    session.run("npm", "run", "check-format", external=True)
+
+
+@nox.session
+def build_js(session: Session) -> None:
+    """Build javascript client code"""
+    session.chdir(SRC / "client")
     session.run("npm", "run", "build", external=True)
 
 
 @nox.session
-def tag(session: Session):
+def tag(session: Session) -> None:
     """Create a new git tag"""
     try:
         session.run(
@@ -265,12 +345,6 @@ def update_version(session: Session) -> None:
     session.install("-e", ".")
 
 
-@nox.session
-def build_js(session: Session) -> None:
-    session.chdir(SRC / "client")
-    session.run("npm", "run", "build", external=True)
-
-
 @nox.session(reuse_venv=True)
 def latest_pull_requests(session: Session) -> None:
     """A basic script for outputing changelog info"""

src/client/package.json

@@ -12,7 +12,8 @@
     "build": "snowpack build",
     "format": "npm --workspaces run format",
     "publish": "npm --workspaces publish",
-    "test": "npm --workspaces test"
+    "test": "npm --workspaces test",
+    "check-format": "npm --workspaces run check-format"
   },
   "version": "0.33.3",
   "workspaces": [

src/client/packages/idom-app-react/package.json

@@ -18,6 +18,7 @@
   },
   "scripts": {
     "format": "prettier --write ./src",
+    "check-format": "prettier --check ./src",
     "test": "echo 'no tests'"
   },
   "version": "0.33.3"

src/client/packages/idom-client-react/package.json

@@ -26,6 +26,7 @@
   },
   "scripts": {
     "format": "prettier --write ./src",
+    "check-format": "prettier --check ./src",
     "test": "uvu tests"
   },
   "type": "module",