From 2e9f3e6b13b5bd8f6d858082875cbd3d1661c575 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 7 Nov 2025 08:22:19 -0500 Subject: [PATCH 1/6] Use with-connect instead of bespoke integration test setup --- .github/workflows/ci.yaml | 79 ++++++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 86b818d4..bcc3cee6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,8 +11,8 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v7 - run: uv python install - run: make dev - run: make lint @@ -31,43 +31,60 @@ jobs: - "3.12" - "3.13" steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v7 - run: uv python install ${{ matrix.python-version }} - run: make dev - run: make test - setup-integration-test: - runs-on: ubuntu-latest - outputs: - versions: ${{ steps.versions.outputs.versions }} - steps: - - uses: actions/checkout@v4 - - id: versions - working-directory: ./integration - # The `jq` command is "output compact, raw input, slurp, split on new lines, and remove the last element". This results in a JSON array of Connect versions (e.g., ["2025.01.0", "2024.12.0"]). - run: | - versions=$(make print-versions | jq -c -Rs 'split("\n") | .[:-1]') - echo "versions=$versions" >> "$GITHUB_OUTPUT" - integration-test: runs-on: ubuntu-latest - needs: setup-integration-test strategy: fail-fast: false matrix: - CONNECT_VERSION: ${{ fromJson(needs.setup-integration-test.outputs.versions) }} + CONNECT_VERSION: + - 2025.10.0 + - 2025.09.1 + - 2025.07.0 + - 2025.06.0 + - 2025.05.0 + - 2025.04.0 + - 2025.03.0 + - 2025.02.0 + - 2025.01.0 + - 2024.12.0 + - 2024.11.0 + - 2024.09.0 + - 2024.08.0 + - 2024.06.0 + - 2024.05.0 + - 2024.04.1 + - 2024.04.0 + - 2024.03.0 + - 2024.02.0 + - 2024.01.0 + - 2023.12.0 + - 2023.10.0 + - 2023.09.0 + - 2023.07.0 + - 2023.06.0 + - 2023.05.0 + - 2023.01.1 + - 2023.01.0 + - 2022.12.0 + - 2022.11.0 steps: - - uses: actions/checkout@v4 - - uses: docker/setup-buildx-action@v3 - - name: Write Posit Connect license to disk - run: echo "$CONNECT_LICENSE" > ./integration/license.lic - env: - CONNECT_LICENSE: ${{ secrets.CONNECT_LICENSE }} - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v7 - run: uv python install - - run: make -C ./integration ${{ matrix.CONNECT_VERSION }} - - uses: actions/upload-artifact@v4 + - name: Run integration tests + uses: posit-dev/with-connect@main + with: + version: ${{ matrix.CONNECT_VERSION }} + license: ${{ secrets.CONNECT_LICENSE }} + command: + uv run pytest -s --junit-xml=./reports/${{ matrix.CONNECT_VERSION }}.xml + - uses: actions/upload-artifact@v5 if: always() with: name: ${{ matrix.CONNECT_VERSION }} - Integration Test Report @@ -81,7 +98,7 @@ jobs: pull-requests: write if: always() steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v6 with: path: artifacts - uses: EnricoMi/publish-unit-test-result-action@v2 @@ -94,8 +111,8 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: astral-sh/setup-uv@v6 + - uses: actions/checkout@v5 + - uses: astral-sh/setup-uv@v7 - run: uv python install - run: make dev - run: make build From 6be152ada1070d2e2dc4b2d60d9ae0997e199d08 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 7 Nov 2025 08:33:26 -0500 Subject: [PATCH 2/6] Fix junit path --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bcc3cee6..423a09d0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -83,7 +83,7 @@ jobs: version: ${{ matrix.CONNECT_VERSION }} license: ${{ secrets.CONNECT_LICENSE }} command: - uv run pytest -s --junit-xml=./reports/${{ matrix.CONNECT_VERSION }}.xml + uv run pytest -s --junit-xml=./integration/reports/${{ matrix.CONNECT_VERSION }}.xml - uses: actions/upload-artifact@v5 if: always() with: From fd37a5cc54ddbcb1142d885db61f7516dfbedc15 Mon Sep 17 00:00:00 2001 From: Neal Richardson Date: Fri, 7 Nov 2025 08:34:35 -0500 Subject: [PATCH 3/6] Delete integration Makefile and compose --- integration/Makefile | 146 --------------------------------------- integration/compose.yaml | 46 ------------ 2 files changed, 192 deletions(-) delete mode 100644 integration/Makefile delete mode 100644 integration/compose.yaml diff --git a/integration/Makefile b/integration/Makefile deleted file mode 100644 index 0bdbd62c..00000000 --- a/integration/Makefile +++ /dev/null @@ -1,146 +0,0 @@ -include ../vars.mk - -# Docker settings -DOCKER_COMPOSE ?= docker compose -DOCKER_CONNECT_IMAGE ?= rstudio/rstudio-connect -DOCKER_PROJECT_IMAGE_TAG ?= $(PROJECT_NAME):latest - -# Connect settings -CONNECT_BOOTSTRAP_SECRETKEY ?= $(shell head -c 32 /dev/random | base64) - -# pytest settings -PYTEST_ARGS ?= "-s" - -.DEFAULT_GOAL := latest - -.PHONY: $(CONNECT_VERSIONS) \ - all \ - build \ - down \ - down-% \ - latest \ - test \ - up \ - up-% \ - help - -# Versions -CONNECT_VERSIONS := \ - 2025.07.0 \ - 2025.06.0 \ - 2025.05.0 \ - 2025.04.0 \ - 2025.03.0 \ - 2025.02.0 \ - 2025.01.0 \ - 2024.12.0 \ - 2024.11.0 \ - 2024.09.0 \ - 2024.08.0 \ - 2024.06.0 \ - 2024.05.0 \ - 2024.04.1 \ - 2024.04.0 \ - 2024.03.0 \ - 2024.02.0 \ - 2024.01.0 \ - 2023.12.0 \ - 2023.10.0 \ - 2023.09.0 \ - 2023.07.0 \ - 2023.06.0 \ - 2023.05.0 \ - 2023.01.1 \ - 2023.01.0 \ - 2022.12.0 \ - 2022.11.0 - -clean: - rm -rf logs reports - find . -type d -empty -delete - -# Run test suite for a specific Connect version. -$(CONNECT_VERSIONS): %: down-% up-% - -# Run test suite against all Connect versions. -all: $(CONNECT_VERSIONS:%=%) preview - -# Run test suite against latest Connect version. -latest: - $(MAKE) $(firstword $(CONNECT_VERSIONS)) - -# Run test suite against preview Connect version. -preview: - $(MAKE) \ - DOCKER_CONNECT_IMAGE=rstudio/rstudio-connect-preview \ - DOCKER_CONNECT_IMAGE_TAG=dev-jammy-daily \ - down-preview up-preview - -# Build Dockerfile -build: - make -C .. $(UV_LOCK) - docker build -t $(DOCKER_PROJECT_IMAGE_TAG) .. - -# Tear down resources. -down: $(CONNECT_VERSIONS:%=down-%) -down-%: DOCKER_CONNECT_IMAGE_TAG=jammy-$* -down-%: CONNECT_VERSION=$* -down-%: - CONNECT_BOOTSTRAP_SECRETKEY=$(CONNECT_BOOTSTRAP_SECRETKEY) \ - CONNECT_VERSION=$* \ - DOCKER_CONNECT_IMAGE_TAG=$(DOCKER_CONNECT_IMAGE_TAG) \ - DOCKER_CONNECT_IMAGE=$(DOCKER_CONNECT_IMAGE) \ - DOCKER_PROJECT_IMAGE_TAG=$(DOCKER_PROJECT_IMAGE_TAG) \ - PYTEST_ARGS="$(PYTEST_ARGS)" \ - $(DOCKER_COMPOSE) -p $(PROJECT_NAME)-$(subst .,-,$(CONNECT_VERSION)) down -v - -# Create, start, and run Docker Compose. -up: $(CONNECT_VERSIONS:%=up-%) -up-%: CONNECT_VERSION=$* -up-%: DOCKER_CONNECT_IMAGE_TAG=jammy-$* -up-%: build - CONNECT_BOOTSTRAP_SECRETKEY=$(CONNECT_BOOTSTRAP_SECRETKEY) \ - CONNECT_VERSION=$* \ - DOCKER_CONNECT_IMAGE_TAG=$(DOCKER_CONNECT_IMAGE_TAG) \ - DOCKER_CONNECT_IMAGE=$(DOCKER_CONNECT_IMAGE) \ - DOCKER_PROJECT_IMAGE_TAG=$(DOCKER_PROJECT_IMAGE_TAG) \ - PYTEST_ARGS="$(PYTEST_ARGS)" \ - $(DOCKER_COMPOSE) -p $(PROJECT_NAME)-$(subst .,-,$(CONNECT_VERSION)) up -V --abort-on-container-exit --no-build - -# Show available versions -print-versions: - @printf "%s\n" $(strip $(CONNECT_VERSIONS)) - -# Show help message. -help: - @echo "Makefile Targets:" - @echo " all (default) Run test suite for all Connect versions." - @echo " latest Run test suite for latest Connect version." - @echo " preview Run test suite for preview Connect version." - @echo " Run test suite for the specified Connect version. (e.g., make 2024.05.0)" - @echo " up Start Docker Compose for all Connect versions." - @echo " down Tear down Docker resources for all Connect versions." - @echo " clean Clean up the project directory." - @echo " print-versions Show the available Connect versions." - @echo " help Show this help message." - @echo - @echo "Common Usage:" - @echo " make -j 4 Run test suite in parallel for all Connect versions." - @echo " make latest Run test suite for latest Connect version." - @echo " make preview Run test suite for preview Connect version." - @echo " make 2024.05.0 Run test suite for specific Connect version." - @echo - @echo "Environment Variables:" - @echo " DOCKER_COMPOSE Command to invoke Docker Compose. Default: docker compose" - @echo " DOCKER_CONNECT_IMAGE Docker image name for Connect. Default: rstudio/rstudio-connect" - @echo " DOCKER_PROJECT_IMAGE_TAG Docker image name and tag for the project image. Default: $(PROJECT_NAME):latest" - @echo " PYTEST_ARGS Arguments to pass to pytest. Default: \"-s\"" - -# Run tests. -test: - mkdir -p logs - set -o pipefail; \ - CONNECT_VERSION=${CONNECT_VERSION} \ - CONNECT_API_KEY="$(shell $(UV) run rsconnect bootstrap -i -s http://connect:3939 --raw)" \ - $(UV) run pytest $(PYTEST_ARGS) --junit-xml=./reports/$(CONNECT_VERSION).xml | \ - tee ./logs/$(CONNECT_VERSION).log; diff --git a/integration/compose.yaml b/integration/compose.yaml deleted file mode 100644 index e52ec59c..00000000 --- a/integration/compose.yaml +++ /dev/null @@ -1,46 +0,0 @@ -services: - tests: - image: ${DOCKER_PROJECT_IMAGE_TAG} - # Run integration test suite. - # - # Target is relative to the ./integration directory, not the project root - # directory. The execution base directory is determined by the 'WORKDIR' - # in the Dockerfile. - command: make -C ./integration test - environment: - - CONNECT_BOOTSTRAP_SECRETKEY=${CONNECT_BOOTSTRAP_SECRETKEY} - # Port 3939 is the default port for Connect - - CONNECT_SERVER=http://connect:3939 - - CONNECT_VERSION=${CONNECT_VERSION} - - PYTEST_ARGS=${PYTEST_ARGS} - volumes: - - .:/sdk/integration - depends_on: - connect: - condition: service_healthy - networks: - - test - connect: - image: ${DOCKER_CONNECT_IMAGE}:${DOCKER_CONNECT_IMAGE_TAG} - pull_policy: always - environment: - - CONNECT_BOOTSTRAP_ENABLED=true - - CONNECT_BOOTSTRAP_SECRETKEY=${CONNECT_BOOTSTRAP_SECRETKEY} - - CONNECT_APPLICATIONS_PACKAGEAUDITINGENABLED=true - - CONNECT_TENSORFLOW_ENABLED=false - networks: - - test - privileged: true - volumes: - - /var/lib/rstudio-connect - - ./license.lic:/var/lib/rstudio-connect/rstudio-connect.lic:ro - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:3939"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 30s - -networks: - test: - driver: bridge From cc58adeb24441e150e054b0379291406e0dfb507 Mon Sep 17 00:00:00 2001 From: tdstein Date: Mon, 10 Nov 2025 10:56:08 -0500 Subject: [PATCH 4/6] feat: Add integration Makefile with dynamic port allocation and simplify CI workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new Makefile for integration tests that: - Dynamically allocates available ports using Python socket binding - Parses CONNECT_VERSION correctly from Make target patterns - Supports running tests for individual or all Connect versions - Includes comprehensive help documentation Update CI workflow to: - Extract Connect versions dynamically from the Makefile - Simplify the integration test matrix setup - Eliminate duplicate version list maintenance 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .github/workflows/ci.yaml | 48 ++++++------------ integration/Makefile | 100 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+), 32 deletions(-) create mode 100644 integration/Makefile diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 423a09d0..ef6dca1f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -37,42 +37,26 @@ jobs: - run: make dev - run: make test + setup-integration-test: + runs-on: ubuntu-latest + outputs: + versions: ${{ steps.versions.outputs.versions }} + steps: + - uses: actions/checkout@v4 + - id: versions + working-directory: ./integration + # The `jq` command is "output compact, raw input, slurp, split on new lines, and remove the last element". This results in a JSON array of Connect versions (e.g., ["2025.01.0", "2024.12.0"]). + run: | + versions=$(make print-versions | jq -c -Rs 'split("\n") | .[:-1]') + echo "versions=$versions" >> "$GITHUB_OUTPUT" + integration-test: runs-on: ubuntu-latest + needs: setup-integration-test strategy: fail-fast: false matrix: - CONNECT_VERSION: - - 2025.10.0 - - 2025.09.1 - - 2025.07.0 - - 2025.06.0 - - 2025.05.0 - - 2025.04.0 - - 2025.03.0 - - 2025.02.0 - - 2025.01.0 - - 2024.12.0 - - 2024.11.0 - - 2024.09.0 - - 2024.08.0 - - 2024.06.0 - - 2024.05.0 - - 2024.04.1 - - 2024.04.0 - - 2024.03.0 - - 2024.02.0 - - 2024.01.0 - - 2023.12.0 - - 2023.10.0 - - 2023.09.0 - - 2023.07.0 - - 2023.06.0 - - 2023.05.0 - - 2023.01.1 - - 2023.01.0 - - 2022.12.0 - - 2022.11.0 + CONNECT_VERSION: ${{ fromJson(needs.setup-integration-test.outputs.versions) }} steps: - uses: actions/checkout@v5 - uses: astral-sh/setup-uv@v7 @@ -83,7 +67,7 @@ jobs: version: ${{ matrix.CONNECT_VERSION }} license: ${{ secrets.CONNECT_LICENSE }} command: - uv run pytest -s --junit-xml=./integration/reports/${{ matrix.CONNECT_VERSION }}.xml + make -C ./integration test-${{ matrix.CONNECT_VERSION }} - uses: actions/upload-artifact@v5 if: always() with: diff --git a/integration/Makefile b/integration/Makefile new file mode 100644 index 00000000..749c0460 --- /dev/null +++ b/integration/Makefile @@ -0,0 +1,100 @@ +include ../vars.mk + +# pytest settings +PYTEST_ARGS ?= "-s" + +.DEFAULT_GOAL := latest + +define GET_PORT +$(UV) run -- python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()' +endef + +.PHONY: $(CONNECT_VERSIONS) \ + all \ + clean \ + latest \ + print-versions \ + help \ + test-% + +# Versions +CONNECT_VERSIONS := \ + 2025.10.0 \ + 2025.09.1 \ + 2025.09.0 \ + 2025.07.0 \ + 2025.06.0 \ + 2025.05.0 \ + 2025.04.0 \ + 2025.03.0 \ + 2025.02.0 \ + 2025.01.0 \ + 2024.12.0 \ + 2024.11.0 \ + 2024.09.0 \ + 2024.08.0 \ + 2024.06.0 \ + 2024.05.0 \ + 2024.04.1 \ + 2024.04.0 \ + 2024.03.0 \ + 2024.02.0 \ + 2024.01.0 \ + 2023.12.0 \ + 2023.10.0 \ + 2023.09.0 \ + 2023.07.0 \ + 2023.06.0 \ + 2023.05.0 \ + 2023.01.1 \ + 2023.01.0 \ + 2022.12.0 \ + 2022.11.0 + +clean: + rm -rf logs reports + find . -type d -empty -delete + +# Run pytest for a specific version (assumes Connect is already running). +test-%: + @mkdir -p logs reports + $(UV) run pytest $(PYTEST_ARGS) \ + --junitxml=reports/junit-$*.xml | tee logs/$*.log + +# Spin up Connect for a specific version and run tests. +$(CONNECT_VERSIONS): %: + PORT=$$($(GET_PORT)); \ + uv run --with https://github.com/posit-dev/with-connect.git \ + with-connect --version $* --port $$PORT \ + -- $(MAKE) test-$* + +# Run test suite against all Connect versions. +all: $(CONNECT_VERSIONS:%=%) + +# Run test suite against latest Connect version. +latest: + $(MAKE) $(firstword $(CONNECT_VERSIONS)) + +# Show available versions +print-versions: + @printf "%s\n" $(strip $(CONNECT_VERSIONS)) + +# Show help message. +help: + @echo "Makefile Targets:" + @echo " latest (default) Run test suite for latest Connect version." + @echo " all Run test suite for all Connect versions." + @echo " Run test suite for the specified Connect version. (e.g., make 2025.10.0)" + @echo " clean Clean up logs and reports directories." + @echo " print-versions Show the available Connect versions." + @echo " help Show this help message." + @echo + @echo "Common Usage:" + @echo " make Run test suite for latest Connect version (default)." + @echo " make latest Run test suite for latest Connect version." + @echo " make 2025.10.0 Run test suite for specific Connect version." + @echo " make all Run test suite for all Connect versions." + @echo " make -j 4 all Run test suite in parallel for all Connect versions." + @echo + @echo "Environment Variables:" + @echo " PYTEST_ARGS Arguments to pass to pytest. Default: \"-s\"" From 76e8452c80c29e9772609415f023af4f4dcde493 Mon Sep 17 00:00:00 2001 From: tdstein Date: Mon, 10 Nov 2025 11:12:20 -0500 Subject: [PATCH 5/6] remove old docker stuff --- Dockerfile | 15 --------------- Makefile | 8 +------- 2 files changed, 1 insertion(+), 22 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 84c92634..00000000 --- a/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3 - -RUN apt-get update && apt-get install -y make - -WORKDIR /sdk - -COPY Makefile pyproject.toml vars.mk uv.lock ./ - -# Run before `COPY src src` to cache dependencies for faster iterative builds -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip make docker-deps - -COPY .git .git -COPY src src - -RUN --mount=type=cache,mode=0755,target=/root/.cache/pip make dev diff --git a/Makefile b/Makefile index 1121951a..57ea8f6b 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ include vars.mk .DEFAULT_GOAL := all -.PHONY: build clean cov default dev docker-deps docs ensure-uv fmt fix install it lint test uninstall version help +.PHONY: build clean cov default dev docs ensure-uv fmt fix install it lint test uninstall version help $(UV_LOCK): dev $(UV) lock @@ -35,12 +35,6 @@ cov-xml: dev dev: ensure-uv $(UV) pip install --upgrade -e . -docker-deps: ensure-uv - # Sync given the `uv.lock` file - # --frozen : assert that the lock file exists - # --no-install-project : do not install the project itself, but install its dependencies - $(UV) sync --frozen --no-install-project - docs: ensure-uv $(MAKE) -C ./docs From 4c83672cc3107edd11e4055dacdbf641ecf6a18f Mon Sep 17 00:00:00 2001 From: Taylor Steinberg Date: Mon, 10 Nov 2025 12:20:47 -0500 Subject: [PATCH 6/6] Update integration/Makefile --- integration/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/integration/Makefile b/integration/Makefile index 749c0460..eab83e00 100644 --- a/integration/Makefile +++ b/integration/Makefile @@ -21,7 +21,6 @@ endef CONNECT_VERSIONS := \ 2025.10.0 \ 2025.09.1 \ - 2025.09.0 \ 2025.07.0 \ 2025.06.0 \ 2025.05.0 \