From c705bacae987a5880921bd68a1efe43b60619f13 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 09:47:35 -0400 Subject: [PATCH 01/37] new docker image --- .github/workflows/deploy_tests.yml | 1 - integration-testing/docker-compose.yml | 4 +++- integration-testing/docker/rstudio-connect.gcfg | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 27b893f1..ca3078c1 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -82,7 +82,6 @@ jobs: remote: "yes" run: | cd integration-testing - docker compose pull connect docker compose up -d connect just ../test/connect-rsconnect-python/test/rsconnect-python/test-rsconnect-python-repo diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 1e418a0e..980b55d9 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -21,7 +21,9 @@ services: connect: hostname: connect - image: rstudio/rstudio-connect:bionic + build: + context: ./docker + dockerfile: connect.Dockerfile restart: always ports: - 3939:3939 diff --git a/integration-testing/docker/rstudio-connect.gcfg b/integration-testing/docker/rstudio-connect.gcfg index 564d6b80..0920bb4c 100644 --- a/integration-testing/docker/rstudio-connect.gcfg +++ b/integration-testing/docker/rstudio-connect.gcfg @@ -20,7 +20,7 @@ Executable = /opt/python/3.9.5/bin/python [Quarto] Enabled = true -Executable = /usr/local/bin/quarto +Executable = /opt/quarto/bin/ [Authentication] BasicAuth = true From f2d67fd80c134fb1a0cc73a4e261d5804ae450ee Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 09:51:55 -0400 Subject: [PATCH 02/37] forgot the dockerfile --- integration-testing/docker/connect.Dockerfile | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 integration-testing/docker/connect.Dockerfile diff --git a/integration-testing/docker/connect.Dockerfile b/integration-testing/docker/connect.Dockerfile new file mode 100644 index 00000000..a875a6df --- /dev/null +++ b/integration-testing/docker/connect.Dockerfile @@ -0,0 +1,6 @@ +FROM rstudio/rstudio-connect:jammy + +RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ + mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ + source ~/.profile From c19adf0d7c548d18eb64d4adc50c897b5c8e00d2 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 09:57:41 -0400 Subject: [PATCH 03/37] pass quarto version --- integration-testing/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 980b55d9..532ff8f8 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -33,6 +33,7 @@ services: environment: RSTUDIO_CONNECT_HASTE: "enabled" RSC_LICENSE: ${CONNECT_LICENSE} + QUARTO_VERSION: ${QUARTO_VERSION} cypress: image: cypress/included:12.7.0 From ce581e3e78d52c64992df820e7588849165daaab Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 10:22:32 -0400 Subject: [PATCH 04/37] use args --- integration-testing/docker-compose.yml | 4 +++- integration-testing/docker/connect.Dockerfile | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 532ff8f8..a5ced9bc 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -24,6 +24,8 @@ services: build: context: ./docker dockerfile: connect.Dockerfile + args: + QUARTO_VERSION: ${QUARTO_VERSION} restart: always ports: - 3939:3939 @@ -34,7 +36,7 @@ services: RSTUDIO_CONNECT_HASTE: "enabled" RSC_LICENSE: ${CONNECT_LICENSE} QUARTO_VERSION: ${QUARTO_VERSION} - + cypress: image: cypress/included:12.7.0 depends_on: diff --git a/integration-testing/docker/connect.Dockerfile b/integration-testing/docker/connect.Dockerfile index a875a6df..2702e436 100644 --- a/integration-testing/docker/connect.Dockerfile +++ b/integration-testing/docker/connect.Dockerfile @@ -1,5 +1,7 @@ FROM rstudio/rstudio-connect:jammy - +ARG QUARTO_VERSION +ENV QUARTO_VERSION=${QUARTO_VERSION} +RUN echo "QUARTO_VERSION is ${QUARTO_VERSION}" RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ From 20002306043734ba3141bfbc7eccc24c379bd694 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 10:27:39 -0400 Subject: [PATCH 05/37] dir already exists --- integration-testing/docker/connect.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-testing/docker/connect.Dockerfile b/integration-testing/docker/connect.Dockerfile index 2702e436..fbc1ef11 100644 --- a/integration-testing/docker/connect.Dockerfile +++ b/integration-testing/docker/connect.Dockerfile @@ -3,6 +3,6 @@ ARG QUARTO_VERSION ENV QUARTO_VERSION=${QUARTO_VERSION} RUN echo "QUARTO_VERSION is ${QUARTO_VERSION}" RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ - mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ source ~/.profile From 6106eb32d0744956d7fe1c2a40d1dec0037f35e9 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 10:38:53 -0400 Subject: [PATCH 06/37] fix quarto path --- integration-testing/docker/rstudio-connect.gcfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-testing/docker/rstudio-connect.gcfg b/integration-testing/docker/rstudio-connect.gcfg index 0920bb4c..fa5f2f74 100644 --- a/integration-testing/docker/rstudio-connect.gcfg +++ b/integration-testing/docker/rstudio-connect.gcfg @@ -20,7 +20,7 @@ Executable = /opt/python/3.9.5/bin/python [Quarto] Enabled = true -Executable = /opt/quarto/bin/ +Executable = /opt/quarto/bin/quarto [Authentication] BasicAuth = true From 8c47045483b78539b7496575e324e26babc2b41c Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 10:56:54 -0400 Subject: [PATCH 07/37] update python --- integration-testing/docker/rstudio-connect.gcfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-testing/docker/rstudio-connect.gcfg b/integration-testing/docker/rstudio-connect.gcfg index fa5f2f74..54857433 100644 --- a/integration-testing/docker/rstudio-connect.gcfg +++ b/integration-testing/docker/rstudio-connect.gcfg @@ -15,8 +15,8 @@ NoWarning = true [Python] Enabled = true -Executable = /opt/python/3.8.10/bin/python -Executable = /opt/python/3.9.5/bin/python +Executable = /opt/python/3.8.17/bin/python +Executable = /opt/python/3.9.17/bin/python [Quarto] Enabled = true From ebdc1e3f92ea06b433c04476603b7bd0835fcf46 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 11:08:44 -0400 Subject: [PATCH 08/37] cleanup --- integration-testing/docker/connect.Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-testing/docker/connect.Dockerfile b/integration-testing/docker/connect.Dockerfile index fbc1ef11..7c71954f 100644 --- a/integration-testing/docker/connect.Dockerfile +++ b/integration-testing/docker/connect.Dockerfile @@ -1,6 +1,5 @@ FROM rstudio/rstudio-connect:jammy ARG QUARTO_VERSION -ENV QUARTO_VERSION=${QUARTO_VERSION} RUN echo "QUARTO_VERSION is ${QUARTO_VERSION}" RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ From 886e16439900eb8656519ce3421c545949e0d94b Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 11:15:10 -0400 Subject: [PATCH 09/37] install quarto on client --- integration-testing/docker/client.Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration-testing/docker/client.Dockerfile b/integration-testing/docker/client.Dockerfile index 2986a095..ce8b3947 100644 --- a/integration-testing/docker/client.Dockerfile +++ b/integration-testing/docker/client.Dockerfile @@ -18,6 +18,11 @@ RUN pip install rsconnect-jupyter --pre && \ pip install pipenv && \ jupyter-nbextension install --sys-prefix --py rsconnect_jupyter +RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ + mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ + source ~/.profile + CMD cd ../ && \ rm -rf ~/.jupyter/ && \ pip install . && \ From cc0d451b61ec5de5e7441972ad31f610b199f118 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 11:39:44 -0400 Subject: [PATCH 10/37] set path to get quarto --- .github/workflows/deploy_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index ca3078c1..add887b6 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -76,7 +76,7 @@ jobs: RSC_AUTOMATION_PAT: "${{ secrets.CONNECT_PAT }}" ADMIN_API_KEY: "${{ secrets.ADMIN_API_KEY }}" QUARTO_VERSION: "1.4.546" - + PATH: "$PATH:/connect/ext/quarto/bin/" # This allows us to start Connect separately in our own docker container CONNECT_SERVER: "http://localhost:3939" remote: "yes" From 181629e42e4d362b0367ad8b9ca991d73e021549 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 11:59:01 -0400 Subject: [PATCH 11/37] use client --- .github/workflows/deploy_tests.yml | 4 ++-- integration-testing/docker-compose.yml | 8 +++++++- integration-testing/docker/client.Dockerfile | 6 ++++-- integration-testing/docker/connect.Dockerfile | 1 - 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index add887b6..25f7e27a 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -76,14 +76,14 @@ jobs: RSC_AUTOMATION_PAT: "${{ secrets.CONNECT_PAT }}" ADMIN_API_KEY: "${{ secrets.ADMIN_API_KEY }}" QUARTO_VERSION: "1.4.546" - PATH: "$PATH:/connect/ext/quarto/bin/" # This allows us to start Connect separately in our own docker container CONNECT_SERVER: "http://localhost:3939" remote: "yes" run: | cd integration-testing docker compose up -d connect - just ../test/connect-rsconnect-python/test/rsconnect-python/test-rsconnect-python-repo + docker compose up -d client + docker-compose run --rm client just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index a5ced9bc..1b5c656c 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -14,11 +14,17 @@ services: build: context: ./docker dockerfile: client.Dockerfile + args: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} volumes: - ../:/rsconnect-python working_dir: /rsconnect-python/integration-testing entrypoint: '' - + environment: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} + connect: hostname: connect build: diff --git a/integration-testing/docker/client.Dockerfile b/integration-testing/docker/client.Dockerfile index ce8b3947..c628dcde 100644 --- a/integration-testing/docker/client.Dockerfile +++ b/integration-testing/docker/client.Dockerfile @@ -1,9 +1,11 @@ -FROM python:3.9 +ARG PY_VERSION=${PY_VERSION} +FROM python:${PY_VERSION} COPY ./requirements.txt . EXPOSE 9999 VOLUME ../../:/rsconnect-python/ WORKDIR /rsconnect-python/integration-testing +ARG QUARTO_VERSION RUN apt-get update && \ apt-get -y install sudo @@ -19,7 +21,7 @@ RUN pip install rsconnect-jupyter --pre && \ jupyter-nbextension install --sys-prefix --py rsconnect_jupyter RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ - mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ source ~/.profile diff --git a/integration-testing/docker/connect.Dockerfile b/integration-testing/docker/connect.Dockerfile index 7c71954f..c74ecfe6 100644 --- a/integration-testing/docker/connect.Dockerfile +++ b/integration-testing/docker/connect.Dockerfile @@ -1,6 +1,5 @@ FROM rstudio/rstudio-connect:jammy ARG QUARTO_VERSION -RUN echo "QUARTO_VERSION is ${QUARTO_VERSION}" RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ From a9384e1e63983d02ed4fd5e4d37eef938e853299 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 12:17:56 -0400 Subject: [PATCH 12/37] fix dockerfile --- integration-testing/docker/client.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-testing/docker/client.Dockerfile b/integration-testing/docker/client.Dockerfile index c628dcde..b907ee6a 100644 --- a/integration-testing/docker/client.Dockerfile +++ b/integration-testing/docker/client.Dockerfile @@ -21,9 +21,9 @@ RUN pip install rsconnect-jupyter --pre && \ jupyter-nbextension install --sys-prefix --py rsconnect_jupyter RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ - tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ - source ~/.profile + . ~/.profile CMD cd ../ && \ rm -rf ~/.jupyter/ && \ From c12a318a605f898f8b4e40e1f737e45c7db66f66 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 12:26:09 -0400 Subject: [PATCH 13/37] add conda --- integration-testing/docker/client.Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integration-testing/docker/client.Dockerfile b/integration-testing/docker/client.Dockerfile index b907ee6a..d6c44ada 100644 --- a/integration-testing/docker/client.Dockerfile +++ b/integration-testing/docker/client.Dockerfile @@ -16,6 +16,11 @@ RUN mkdir -p /libs-client && \ ENV PATH=$PATH:/libs-client +RUN curl -fsSL -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ + && chmod 755 miniconda.sh \ + && ./miniconda.sh -b -p /opt/miniconda \ + && rm -rf miniconda.sh + RUN pip install rsconnect-jupyter --pre && \ pip install pipenv && \ jupyter-nbextension install --sys-prefix --py rsconnect_jupyter From ef30ee385e0ab55a4da7acb36e67c4c48d43b586 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 12:38:45 -0400 Subject: [PATCH 14/37] add deps --- integration-testing/docker/client.Dockerfile | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/integration-testing/docker/client.Dockerfile b/integration-testing/docker/client.Dockerfile index d6c44ada..bf1adadb 100644 --- a/integration-testing/docker/client.Dockerfile +++ b/integration-testing/docker/client.Dockerfile @@ -7,8 +7,15 @@ VOLUME ../../:/rsconnect-python/ WORKDIR /rsconnect-python/integration-testing ARG QUARTO_VERSION -RUN apt-get update && \ - apt-get -y install sudo +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + git \ + sudo \ + vim \ + jq \ + wget RUN mkdir -p /libs-client && \ curl -fsSL https://github.com/casey/just/releases/download/1.1.2/just-1.1.2-x86_64-unknown-linux-musl.tar.gz \ @@ -16,6 +23,13 @@ RUN mkdir -p /libs-client && \ ENV PATH=$PATH:/libs-client +RUN git clone --depth=1 https://github.com/bats-core/bats-core.git /libs/bats-core \ + && cd /libs/bats-core \ + && ./install.sh /libs/bats-core/installation \ + && git clone --depth=1 https://github.com/bats-core/bats-support.git /libs/bats-support \ + && git clone --depth=1 https://github.com/bats-core/bats-file.git /libs/bats-file \ + && git clone --depth=1 https://github.com/bats-core/bats-assert.git /libs/bats-assert + RUN curl -fsSL -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ && chmod 755 miniconda.sh \ && ./miniconda.sh -b -p /opt/miniconda \ From 34e11e7e6d4750e9611301053e3186fb8e96cb45 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 13:25:18 -0400 Subject: [PATCH 15/37] fix network mode for client --- integration-testing/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 1b5c656c..5fa54d5b 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -20,6 +20,7 @@ services: volumes: - ../:/rsconnect-python working_dir: /rsconnect-python/integration-testing + network_mode: host entrypoint: '' environment: QUARTO_VERSION: ${QUARTO_VERSION} From 6f49ebb59ab5390edd0ed3633c762c62b35f4481 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 13:47:35 -0400 Subject: [PATCH 16/37] use localhost --- integration-testing/docker-compose.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 5fa54d5b..e5fd49b1 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -2,10 +2,8 @@ version: '3' services: client: - image: client - hostname: client healthcheck: - test: ["CMD", "curl", "-f", "http://client:9999/tree"] + test: ["CMD", "curl", "-f", "http://localhost:9999/tree"] interval: 40s timeout: 3s retries: 30 From 0261890445b5b4347d84054aaa9e90257e307705 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 14:01:06 -0400 Subject: [PATCH 17/37] use a new service --- .github/workflows/deploy_tests.yml | 4 ++-- integration-testing/docker-compose.yml | 24 ++++++++++++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 25f7e27a..7c2a0dc9 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -82,8 +82,8 @@ jobs: run: | cd integration-testing docker compose up -d connect - docker compose up -d client - docker-compose run --rm client just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev + docker compose up -d client-deploy + docker-compose run --rm client-deploy just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index e5fd49b1..3e1aecb2 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -1,9 +1,11 @@ version: '3' services: - client: + client-jupyter: + image: client + hostname: client healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9999/tree"] + test: ["CMD", "curl", "-f", "http://client:9999/tree"] interval: 40s timeout: 3s retries: 30 @@ -23,6 +25,24 @@ services: environment: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} + + client-deploy: + ports: + - 9999:9999 + build: + context: ./docker + dockerfile: client.Dockerfile + args: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} + volumes: + - ../:/rsconnect-python + working_dir: /rsconnect-python/integration-testing + network_mode: host + entrypoint: '' + environment: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} connect: hostname: connect From 8027579493596bee9b7751d21629834f9fdcd108 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 14:07:19 -0400 Subject: [PATCH 18/37] new dockerfile --- .github/workflows/deploy_tests.yml | 4 ++-- integration-testing/docker-compose.yml | 7 +++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 7c2a0dc9..7e7185f2 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -82,8 +82,8 @@ jobs: run: | cd integration-testing docker compose up -d connect - docker compose up -d client-deploy - docker-compose run --rm client-deploy just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev + docker compose up -d client-cli + docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 3e1aecb2..05291f66 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -20,18 +20,17 @@ services: volumes: - ../:/rsconnect-python working_dir: /rsconnect-python/integration-testing - network_mode: host entrypoint: '' environment: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} - client-deploy: + client-cli: ports: - 9999:9999 build: context: ./docker - dockerfile: client.Dockerfile + dockerfile: cli.Dockerfile args: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} @@ -65,7 +64,7 @@ services: cypress: image: cypress/included:12.7.0 depends_on: - client: + client-jupyter: condition: service_healthy build: context: ./docker From 6ef721c9b77c17e6e065599be3445364a5f5dfa2 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 14:12:09 -0400 Subject: [PATCH 19/37] add new dockerfile --- integration-testing/docker/cli.Dockerfile | 45 +++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 integration-testing/docker/cli.Dockerfile diff --git a/integration-testing/docker/cli.Dockerfile b/integration-testing/docker/cli.Dockerfile new file mode 100644 index 00000000..7913e6b0 --- /dev/null +++ b/integration-testing/docker/cli.Dockerfile @@ -0,0 +1,45 @@ +ARG PY_VERSION=${PY_VERSION} +FROM python:${PY_VERSION} +COPY ./requirements.txt . +EXPOSE 9999 +VOLUME ../../:/rsconnect-python/ + +WORKDIR /rsconnect-python/integration-testing +ARG QUARTO_VERSION + +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + git \ + sudo \ + vim \ + jq \ + wget + +RUN mkdir -p /libs-client && \ + curl -fsSL https://github.com/casey/just/releases/download/1.1.2/just-1.1.2-x86_64-unknown-linux-musl.tar.gz \ + | tar -C /libs-client -xz just + +ENV PATH=$PATH:/libs-client + +RUN git clone --depth=1 https://github.com/bats-core/bats-core.git /libs/bats-core \ + && cd /libs/bats-core \ + && ./install.sh /libs/bats-core/installation \ + && git clone --depth=1 https://github.com/bats-core/bats-support.git /libs/bats-support \ + && git clone --depth=1 https://github.com/bats-core/bats-file.git /libs/bats-file \ + && git clone --depth=1 https://github.com/bats-core/bats-assert.git /libs/bats-assert + +RUN curl -fsSL -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ + && chmod 755 miniconda.sh \ + && ./miniconda.sh -b -p /opt/miniconda \ + && rm -rf miniconda.sh + +RUN pip install rsconnect-jupyter --pre && \ + pip install pipenv && \ + jupyter-nbextension install --sys-prefix --py rsconnect_jupyter + +RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ + mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ + . ~/.profile \ No newline at end of file From d3a0f886c51902e971699291e2b4af3b042af9a7 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 15:46:46 -0400 Subject: [PATCH 20/37] use gh actions docker --- .github/workflows/deploy_tests.yml | 19 +++++--- integration-testing/docker-compose.yml | 37 ++------------- integration-testing/docker/cli.Dockerfile | 45 ------------------- integration-testing/docker/client.Dockerfile | 32 ++----------- integration-testing/docker/connect.Dockerfile | 1 + 5 files changed, 21 insertions(+), 113 deletions(-) delete mode 100644 integration-testing/docker/cli.Dockerfile diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 7e7185f2..83aa30fb 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -13,7 +13,7 @@ jobs: fetch-depth: 0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - + # install python - uses: actions/setup-python@v4 with: python-version: 3.8.x @@ -21,11 +21,20 @@ jobs: - run: pip freeze - run: make dist id: create_dist + # download build artifacts - uses: actions/upload-artifact@v3 with: name: distributions path: dist/ - + # install quarto + - uses: quarto-dev/quarto-actions/setup@v2 + with: + version: 1.4.549 + # install bats + - uses: mig4/setup-bats@v1 + with: + bats-version: 1.2.1 + test-connect: needs: distributions runs-on: ubuntu-latest @@ -80,10 +89,8 @@ jobs: CONNECT_SERVER: "http://localhost:3939" remote: "yes" run: | - cd integration-testing - docker compose up -d connect - docker compose up -d client-cli - docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev + cd integration-testing/ + just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 05291f66..1e418a0e 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: - client-jupyter: + client: image: client hostname: client healthcheck: @@ -14,42 +14,14 @@ services: build: context: ./docker dockerfile: client.Dockerfile - args: - QUARTO_VERSION: ${QUARTO_VERSION} - PY_VERSION: ${PY_VERSION} volumes: - ../:/rsconnect-python working_dir: /rsconnect-python/integration-testing entrypoint: '' - environment: - QUARTO_VERSION: ${QUARTO_VERSION} - PY_VERSION: ${PY_VERSION} - client-cli: - ports: - - 9999:9999 - build: - context: ./docker - dockerfile: cli.Dockerfile - args: - QUARTO_VERSION: ${QUARTO_VERSION} - PY_VERSION: ${PY_VERSION} - volumes: - - ../:/rsconnect-python - working_dir: /rsconnect-python/integration-testing - network_mode: host - entrypoint: '' - environment: - QUARTO_VERSION: ${QUARTO_VERSION} - PY_VERSION: ${PY_VERSION} - connect: hostname: connect - build: - context: ./docker - dockerfile: connect.Dockerfile - args: - QUARTO_VERSION: ${QUARTO_VERSION} + image: rstudio/rstudio-connect:bionic restart: always ports: - 3939:3939 @@ -59,12 +31,11 @@ services: environment: RSTUDIO_CONNECT_HASTE: "enabled" RSC_LICENSE: ${CONNECT_LICENSE} - QUARTO_VERSION: ${QUARTO_VERSION} - + cypress: image: cypress/included:12.7.0 depends_on: - client-jupyter: + client: condition: service_healthy build: context: ./docker diff --git a/integration-testing/docker/cli.Dockerfile b/integration-testing/docker/cli.Dockerfile deleted file mode 100644 index 7913e6b0..00000000 --- a/integration-testing/docker/cli.Dockerfile +++ /dev/null @@ -1,45 +0,0 @@ -ARG PY_VERSION=${PY_VERSION} -FROM python:${PY_VERSION} -COPY ./requirements.txt . -EXPOSE 9999 -VOLUME ../../:/rsconnect-python/ - -WORKDIR /rsconnect-python/integration-testing -ARG QUARTO_VERSION - -RUN apt-get update -y && apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - ca-certificates \ - git \ - sudo \ - vim \ - jq \ - wget - -RUN mkdir -p /libs-client && \ - curl -fsSL https://github.com/casey/just/releases/download/1.1.2/just-1.1.2-x86_64-unknown-linux-musl.tar.gz \ - | tar -C /libs-client -xz just - -ENV PATH=$PATH:/libs-client - -RUN git clone --depth=1 https://github.com/bats-core/bats-core.git /libs/bats-core \ - && cd /libs/bats-core \ - && ./install.sh /libs/bats-core/installation \ - && git clone --depth=1 https://github.com/bats-core/bats-support.git /libs/bats-support \ - && git clone --depth=1 https://github.com/bats-core/bats-file.git /libs/bats-file \ - && git clone --depth=1 https://github.com/bats-core/bats-assert.git /libs/bats-assert - -RUN curl -fsSL -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ - && chmod 755 miniconda.sh \ - && ./miniconda.sh -b -p /opt/miniconda \ - && rm -rf miniconda.sh - -RUN pip install rsconnect-jupyter --pre && \ - pip install pipenv && \ - jupyter-nbextension install --sys-prefix --py rsconnect_jupyter - -RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ - mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ - ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ - . ~/.profile \ No newline at end of file diff --git a/integration-testing/docker/client.Dockerfile b/integration-testing/docker/client.Dockerfile index bf1adadb..2986a095 100644 --- a/integration-testing/docker/client.Dockerfile +++ b/integration-testing/docker/client.Dockerfile @@ -1,21 +1,12 @@ -ARG PY_VERSION=${PY_VERSION} -FROM python:${PY_VERSION} +FROM python:3.9 COPY ./requirements.txt . EXPOSE 9999 VOLUME ../../:/rsconnect-python/ WORKDIR /rsconnect-python/integration-testing -ARG QUARTO_VERSION -RUN apt-get update -y && apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - ca-certificates \ - git \ - sudo \ - vim \ - jq \ - wget +RUN apt-get update && \ + apt-get -y install sudo RUN mkdir -p /libs-client && \ curl -fsSL https://github.com/casey/just/releases/download/1.1.2/just-1.1.2-x86_64-unknown-linux-musl.tar.gz \ @@ -23,27 +14,10 @@ RUN mkdir -p /libs-client && \ ENV PATH=$PATH:/libs-client -RUN git clone --depth=1 https://github.com/bats-core/bats-core.git /libs/bats-core \ - && cd /libs/bats-core \ - && ./install.sh /libs/bats-core/installation \ - && git clone --depth=1 https://github.com/bats-core/bats-support.git /libs/bats-support \ - && git clone --depth=1 https://github.com/bats-core/bats-file.git /libs/bats-file \ - && git clone --depth=1 https://github.com/bats-core/bats-assert.git /libs/bats-assert - -RUN curl -fsSL -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ - && chmod 755 miniconda.sh \ - && ./miniconda.sh -b -p /opt/miniconda \ - && rm -rf miniconda.sh - RUN pip install rsconnect-jupyter --pre && \ pip install pipenv && \ jupyter-nbextension install --sys-prefix --py rsconnect_jupyter -RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ - mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ - ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ - . ~/.profile - CMD cd ../ && \ rm -rf ~/.jupyter/ && \ pip install . && \ diff --git a/integration-testing/docker/connect.Dockerfile b/integration-testing/docker/connect.Dockerfile index c74ecfe6..2d68f746 100644 --- a/integration-testing/docker/connect.Dockerfile +++ b/integration-testing/docker/connect.Dockerfile @@ -1,5 +1,6 @@ FROM rstudio/rstudio-connect:jammy ARG QUARTO_VERSION + RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ From 49df3bd2983f89af77eac375593f404f5fc81c21 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 16:15:32 -0400 Subject: [PATCH 21/37] fix connect --- .github/workflows/deploy_tests.yml | 20 +++++++----------- integration-testing/docker-compose.yml | 28 +++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 83aa30fb..87135333 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -13,7 +13,7 @@ jobs: fetch-depth: 0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # install python + - uses: actions/setup-python@v4 with: python-version: 3.8.x @@ -21,20 +21,11 @@ jobs: - run: pip freeze - run: make dist id: create_dist - # download build artifacts - uses: actions/upload-artifact@v3 with: name: distributions path: dist/ - # install quarto - - uses: quarto-dev/quarto-actions/setup@v2 - with: - version: 1.4.549 - # install bats - - uses: mig4/setup-bats@v1 - with: - bats-version: 1.2.1 - + test-connect: needs: distributions runs-on: ubuntu-latest @@ -89,8 +80,11 @@ jobs: CONNECT_SERVER: "http://localhost:3939" remote: "yes" run: | - cd integration-testing/ - just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev + cd integration-testing + docker compose build + docker compose up -d connect + docker compose up -d client-cli + docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 1e418a0e..4a0d312b 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -19,9 +19,33 @@ services: working_dir: /rsconnect-python/integration-testing entrypoint: '' + client-cli: + ports: + - 9999:9999 + build: + context: ./docker + dockerfile: cli.Dockerfile + args: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} + volumes: + - ../:/rsconnect-python + working_dir: /rsconnect-python/integration-testing + network_mode: host + entrypoint: '' + environment: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} + connect: hostname: connect - image: rstudio/rstudio-connect:bionic + image: rstudio/rstudio-connect:jammy + build: + context: ./docker + dockerfile: connect.Dockerfile + args: + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} restart: always ports: - 3939:3939 @@ -31,6 +55,8 @@ services: environment: RSTUDIO_CONNECT_HASTE: "enabled" RSC_LICENSE: ${CONNECT_LICENSE} + QUARTO_VERSION: ${QUARTO_VERSION} + PY_VERSION: ${PY_VERSION} cypress: image: cypress/included:12.7.0 From 3ac0756b4b63912ca1a9f7437f32dcf7a2834cbf Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 16:25:28 -0400 Subject: [PATCH 22/37] build separately --- .github/workflows/deploy_tests.yml | 3 +- integration-testing/docker/cli.Dockerfile | 45 +++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 integration-testing/docker/cli.Dockerfile diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 87135333..c6a556ab 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -81,7 +81,8 @@ jobs: remote: "yes" run: | cd integration-testing - docker compose build + docker compose build connect + docker compose build client-cli docker compose up -d connect docker compose up -d client-cli docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev diff --git a/integration-testing/docker/cli.Dockerfile b/integration-testing/docker/cli.Dockerfile new file mode 100644 index 00000000..7913e6b0 --- /dev/null +++ b/integration-testing/docker/cli.Dockerfile @@ -0,0 +1,45 @@ +ARG PY_VERSION=${PY_VERSION} +FROM python:${PY_VERSION} +COPY ./requirements.txt . +EXPOSE 9999 +VOLUME ../../:/rsconnect-python/ + +WORKDIR /rsconnect-python/integration-testing +ARG QUARTO_VERSION + +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + ca-certificates \ + git \ + sudo \ + vim \ + jq \ + wget + +RUN mkdir -p /libs-client && \ + curl -fsSL https://github.com/casey/just/releases/download/1.1.2/just-1.1.2-x86_64-unknown-linux-musl.tar.gz \ + | tar -C /libs-client -xz just + +ENV PATH=$PATH:/libs-client + +RUN git clone --depth=1 https://github.com/bats-core/bats-core.git /libs/bats-core \ + && cd /libs/bats-core \ + && ./install.sh /libs/bats-core/installation \ + && git clone --depth=1 https://github.com/bats-core/bats-support.git /libs/bats-support \ + && git clone --depth=1 https://github.com/bats-core/bats-file.git /libs/bats-file \ + && git clone --depth=1 https://github.com/bats-core/bats-assert.git /libs/bats-assert + +RUN curl -fsSL -o miniconda.sh https://repo.anaconda.com/miniconda/Miniconda3-py38_4.10.3-Linux-x86_64.sh \ + && chmod 755 miniconda.sh \ + && ./miniconda.sh -b -p /opt/miniconda \ + && rm -rf miniconda.sh + +RUN pip install rsconnect-jupyter --pre && \ + pip install pipenv && \ + jupyter-nbextension install --sys-prefix --py rsconnect_jupyter + +RUN curl -fsSLO https://github.com/quarto-dev/quarto-cli/releases/download/v${QUARTO_VERSION}/quarto-${QUARTO_VERSION}-linux-amd64.tar.gz && \ + mkdir /opt/quarto && tar xf quarto-${QUARTO_VERSION}-linux-amd64.tar.gz -C /opt/quarto --strip-components 1 && \ + ( echo ""; echo 'export PATH=$PATH:/opt/quarto/bin' ; echo "" ) >> ~/.profile && \ + . ~/.profile \ No newline at end of file From 1b2ba0e2699a0839fa573b2fc3be4f11e8b9f754 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 16:37:19 -0400 Subject: [PATCH 23/37] add tmate --- .github/workflows/deploy_tests.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index c6a556ab..73b1fe9b 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -66,7 +66,11 @@ jobs: with: name: distributions path: dist/ - + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true - name: Run rsconnect-python Tests env: CONNECT_LICENSE: "${{ secrets.RSC_LICENSE }}" From 0e09617cec48cceb3483c3e8d49ef650e769c3a4 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 16:47:59 -0400 Subject: [PATCH 24/37] move tmate --- .github/workflows/deploy_tests.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index 73b1fe9b..d3739466 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -66,11 +66,7 @@ jobs: with: name: distributions path: dist/ - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true + - name: Run rsconnect-python Tests env: CONNECT_LICENSE: "${{ secrets.RSC_LICENSE }}" @@ -90,8 +86,14 @@ jobs: docker compose up -d connect docker compose up -d client-cli docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev - - # Videos are captured whether the suite fails or passes + + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + with: + limit-access-to-actor: true + + # Videos are captured whether the suite fails or passes - name: Save videos uses: actions/upload-artifact@v3 if: failure() From c9515b9d2afc760b76b70702f65e7546fb711039 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Tue, 26 Mar 2024 20:57:06 -0400 Subject: [PATCH 25/37] add api key --- integration-testing/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 4a0d312b..9d719465 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -20,8 +20,6 @@ services: entrypoint: '' client-cli: - ports: - - 9999:9999 build: context: ./docker dockerfile: cli.Dockerfile @@ -36,6 +34,7 @@ services: environment: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} + ADMIN_API_KEY: ${ADMIN_API_KEY} connect: hostname: connect From 53c2c3fa744ec4e2a64bd0aa44a6a44d50e970eb Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 08:10:20 -0400 Subject: [PATCH 26/37] use api_key --- integration-testing/docker-compose.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index 9d719465..f05071dd 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -26,6 +26,7 @@ services: args: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} + ADMIN_API_KEY: ${ADMIN_API_KEY} volumes: - ../:/rsconnect-python working_dir: /rsconnect-python/integration-testing @@ -34,7 +35,7 @@ services: environment: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} - ADMIN_API_KEY: ${ADMIN_API_KEY} + API_KEY: ${ADMIN_API_KEY} connect: hostname: connect From fde81746fb988586502b5433467e71f2281ad495 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 08:10:41 -0400 Subject: [PATCH 27/37] use api_key --- integration-testing/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index f05071dd..e2a2559e 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -26,7 +26,6 @@ services: args: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} - ADMIN_API_KEY: ${ADMIN_API_KEY} volumes: - ../:/rsconnect-python working_dir: /rsconnect-python/integration-testing From cb29ab49a7ef2fe6c5fd8a28388cf018026d1337 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 08:48:21 -0400 Subject: [PATCH 28/37] fix connect on main run --- .github/workflows/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4c43d69..18b74263 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -207,6 +207,7 @@ jobs: env: CONNECT_LICENSE: ${{ secrets.RSC_LICENSE }} ADMIN_API_KEY: ${{ secrets.ADMIN_API_KEY }} + QUARTO_VERSION: "1.4.546" steps: - uses: actions/checkout@v4 with: @@ -218,7 +219,8 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build Containers run: | - just integration-testing/build + docker compose build connect + docker compose build client - name: Start Connect + rsconnect-jupyter run: | just integration-testing/up From 28273b7699a3d932e8fc6f16dbbfb6a5a4d75b96 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 08:51:17 -0400 Subject: [PATCH 29/37] build in the right dir --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 18b74263..869a456a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -219,6 +219,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build Containers run: | + cd integration-testing docker compose build connect docker compose build client - name: Start Connect + rsconnect-jupyter From 401a894faf474371de79164152b6cc3e2f546bae Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 09:12:29 -0400 Subject: [PATCH 30/37] build and run connect --- .github/workflows/main.yml | 13 ++++++------- integration-testing/justfile | 4 ++-- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 869a456a..360d514c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -217,15 +217,14 @@ jobs: - uses: extractions/setup-just@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build Containers + - name: Build and Run Containers run: | cd integration-testing docker compose build connect docker compose build client - - name: Start Connect + rsconnect-jupyter - run: | - just integration-testing/up - + docker compose up -d connect + docker compose up -d client + - name: Run Cypress Tests run: | export ADMIN_API_KEY="${{ secrets.ADMIN_API_KEY }}" @@ -306,9 +305,9 @@ jobs: remote: "yes" run: | cd integration-testing - docker compose pull connect + docker compose build connect docker compose up -d connect - just ../test/connect-rsconnect-python/test/rsconnect-python/test-rsconnect-python-repo + docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/justfile b/integration-testing/justfile index c4512232..9e40e123 100644 --- a/integration-testing/justfile +++ b/integration-testing/justfile @@ -1,5 +1,6 @@ export RSC_LICENSE := env_var_or_default("RSC_LICENSE", "$CONNECT_LICENSE") export ADMIN_API_KEY := env_var_or_default('ADMIN_API_KEY', "${ADMIN_API_KEY}") +export QUARTO_VERSION := env_var_or_default('QUARTO_VERSION', '1.4.546') all: @@ -10,11 +11,10 @@ build: docker compose build up: - docker compose pull connect && \ docker compose up -d client && docker compose up -d connect up-cypress: - docker compose up --exit-code-from cypress + docker compose up cypress --exit-code-from cypress # use this target if you want to run Cypress # without shutting down all containers after each run From f4639d6606a029492809be749eea6be456672548 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 09:33:11 -0400 Subject: [PATCH 31/37] use a separate connect instance to avoid build in CI --- .github/workflows/deploy_tests.yml | 4 +- .github/workflows/main.yml | 1 - integration-testing/docker-compose.yml | 24 ++++++++++-- .../docker/rstudio-connect-cli.gcfg | 37 +++++++++++++++++++ .../docker/rstudio-connect.gcfg | 8 ++-- integration-testing/justfile | 2 +- 6 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 integration-testing/docker/rstudio-connect-cli.gcfg diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index d3739466..df6d7b3c 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -81,9 +81,9 @@ jobs: remote: "yes" run: | cd integration-testing - docker compose build connect + docker compose build connect-cli docker compose build client-cli - docker compose up -d connect + docker compose up -d connect-cli docker compose up -d client-cli docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 360d514c..5e351ada 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -227,7 +227,6 @@ jobs: - name: Run Cypress Tests run: | - export ADMIN_API_KEY="${{ secrets.ADMIN_API_KEY }}" just integration-testing/up-cypress # Videos are captured whether the suite fails or passes diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index e2a2559e..fe6e6e35 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -36,8 +36,10 @@ services: PY_VERSION: ${PY_VERSION} API_KEY: ${ADMIN_API_KEY} - connect: - hostname: connect + # customized connect built with updated quarto version + # used for nightly deploy_tests.yml that include quarto projects + connect-cli: + hostname: connect-cli image: rstudio/rstudio-connect:jammy build: context: ./docker @@ -49,7 +51,7 @@ services: ports: - 3939:3939 volumes: - - $PWD/docker/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg + - $PWD/docker/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect-cli.gcfg privileged: true environment: RSTUDIO_CONNECT_HASTE: "enabled" @@ -57,6 +59,22 @@ services: QUARTO_VERSION: ${QUARTO_VERSION} PY_VERSION: ${PY_VERSION} + # connect from public docker hub + # used jupyter-notebook and deploy tests in CI, main.yml + connect: + hostname: connect + image: rstudio/rstudio-connect:bionic + restart: always + ports: + - 3939:3939 + volumes: + - $PWD/docker/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg + privileged: true + environment: + RSTUDIO_CONNECT_HASTE: "enabled" + RSC_LICENSE: ${CONNECT_LICENSE} + + cypress: image: cypress/included:12.7.0 depends_on: diff --git a/integration-testing/docker/rstudio-connect-cli.gcfg b/integration-testing/docker/rstudio-connect-cli.gcfg new file mode 100644 index 00000000..54857433 --- /dev/null +++ b/integration-testing/docker/rstudio-connect-cli.gcfg @@ -0,0 +1,37 @@ +; Posit Connect test environment configuration + +[Server] +SenderEmail = testuser@testrsc.com +EmailProvider = print +DataDir = /connect-rsconnect-data +EnableSitemap = true +AllowConfirmedUsers = true +Address = http://localhost:3939 +JumpStartEnabled = false + +[HTTP] +Listen = :3939 +NoWarning = true + +[Python] +Enabled = true +Executable = /opt/python/3.8.17/bin/python +Executable = /opt/python/3.9.17/bin/python + +[Quarto] +Enabled = true +Executable = /opt/quarto/bin/quarto + +[Authentication] +BasicAuth = true +InsecureDefaultUserAPIKey = true +APIKeyBcryptCost = 4 + +[Authorization] +DefaultUserRole = "publisher" + +[Database] +SeedUsers = true + +[Mount] +BaseDir = /connect-rsconnect-mount diff --git a/integration-testing/docker/rstudio-connect.gcfg b/integration-testing/docker/rstudio-connect.gcfg index 54857433..b2e03834 100644 --- a/integration-testing/docker/rstudio-connect.gcfg +++ b/integration-testing/docker/rstudio-connect.gcfg @@ -15,12 +15,12 @@ NoWarning = true [Python] Enabled = true -Executable = /opt/python/3.8.17/bin/python -Executable = /opt/python/3.9.17/bin/python +Executable = /opt/python/3.8.10/bin/python +Executable = /opt/python/3.9.5/bin/python [Quarto] Enabled = true -Executable = /opt/quarto/bin/quarto +Executable = /usr/local/bin/quarto [Authentication] BasicAuth = true @@ -34,4 +34,4 @@ DefaultUserRole = "publisher" SeedUsers = true [Mount] -BaseDir = /connect-rsconnect-mount +BaseDir = /connect-rsconnect-mount \ No newline at end of file diff --git a/integration-testing/justfile b/integration-testing/justfile index 9e40e123..e145a327 100644 --- a/integration-testing/justfile +++ b/integration-testing/justfile @@ -14,7 +14,7 @@ up: docker compose up -d client && docker compose up -d connect up-cypress: - docker compose up cypress --exit-code-from cypress + docker compose up --exit-code-from cypress # use this target if you want to run Cypress # without shutting down all containers after each run From a1e53b4b58ee6462cbb99231104b512bae76d243 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 09:36:31 -0400 Subject: [PATCH 32/37] revert changes --- .github/workflows/main.yml | 16 +++++++--------- integration-testing/justfile | 1 + 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5e351ada..06e618c0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -207,7 +207,6 @@ jobs: env: CONNECT_LICENSE: ${{ secrets.RSC_LICENSE }} ADMIN_API_KEY: ${{ secrets.ADMIN_API_KEY }} - QUARTO_VERSION: "1.4.546" steps: - uses: actions/checkout@v4 with: @@ -217,13 +216,12 @@ jobs: - uses: extractions/setup-just@v1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Build and Run Containers + - name: Build Containers run: | - cd integration-testing - docker compose build connect - docker compose build client - docker compose up -d connect - docker compose up -d client + just integration-testing/build + - name: Start Connect + rsconnect-jupyter + run: | + just integration-testing/up - name: Run Cypress Tests run: | @@ -304,9 +302,9 @@ jobs: remote: "yes" run: | cd integration-testing - docker compose build connect + docker compose pull connect docker compose up -d connect - docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev + just ../test/connect-rsconnect-python/test/rsconnect-python/test-rsconnect-python-repo # Videos are captured whether the suite fails or passes - name: Save videos diff --git a/integration-testing/justfile b/integration-testing/justfile index e145a327..882f2ea6 100644 --- a/integration-testing/justfile +++ b/integration-testing/justfile @@ -11,6 +11,7 @@ build: docker compose build up: + docker compose pull connect && \ docker compose up -d client && docker compose up -d connect up-cypress: From af6e19de234d36c41fe7b77413ce73dd13978595 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 09:38:19 -0400 Subject: [PATCH 33/37] revert changes --- integration-testing/justfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-testing/justfile b/integration-testing/justfile index 882f2ea6..5e654997 100644 --- a/integration-testing/justfile +++ b/integration-testing/justfile @@ -15,7 +15,7 @@ up: docker compose up -d client && docker compose up -d connect up-cypress: - docker compose up --exit-code-from cypress + docker compose up cypress --exit-code-from cypress # use this target if you want to run Cypress # without shutting down all containers after each run From a19f99af23cbbea4ad4a960c6c90ca85c84d8c95 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 09:43:59 -0400 Subject: [PATCH 34/37] separte build commands --- .github/workflows/main.yml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 06e618c0..fc9e94a8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -201,7 +201,23 @@ jobs: run: | pytest tests/test_main_system_caches.py pytest -m 'vetiver' + - name: Run rsconnect-python Tests + env: + CONNECT_LICENSE: "${{ secrets.RSC_LICENSE }}" + PY_VERSION: ${{ matrix.PY_VERSION }} + TEST_SUBSET: "CI" + RSC_AUTOMATION_PAT: "${{ secrets.CONNECT_PAT }}" + ADMIN_API_KEY: "${{ secrets.ADMIN_API_KEY }}" + QUARTO_VERSION: "1.4.546" + # This allows us to start Connect separately in our own docker container + CONNECT_SERVER: "http://localhost:3939" + remote: "yes" + run: | + cd integration-testing + docker compose pull connect + docker compose up -d connect + just ../test/connect-rsconnect-python/test/rsconnect-python/test-rsconnect-python-repo test-jupyter: runs-on: ubuntu-latest env: @@ -218,7 +234,9 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build Containers run: | - just integration-testing/build + cd integration-testing + docker compose build client + docker compose build cypress - name: Start Connect + rsconnect-jupyter run: | just integration-testing/up From 62b8678869834be3be6cfe63113546d3680f3a52 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 10:53:14 -0400 Subject: [PATCH 35/37] fix config --- .github/workflows/deploy_tests.yml | 8 +------- integration-testing/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy_tests.yml b/.github/workflows/deploy_tests.yml index df6d7b3c..01c0cb13 100644 --- a/.github/workflows/deploy_tests.yml +++ b/.github/workflows/deploy_tests.yml @@ -87,13 +87,7 @@ jobs: docker compose up -d client-cli docker-compose run --rm client-cli just ../test/connect-rsconnect-python/test/rsconnect-python/_start-dev - - name: Setup tmate session - if: ${{ failure() }} - uses: mxschmitt/action-tmate@v3 - with: - limit-access-to-actor: true - - # Videos are captured whether the suite fails or passes + # Videos are captured whether the suite fails or passes - name: Save videos uses: actions/upload-artifact@v3 if: failure() diff --git a/integration-testing/docker-compose.yml b/integration-testing/docker-compose.yml index fe6e6e35..b60f9265 100644 --- a/integration-testing/docker-compose.yml +++ b/integration-testing/docker-compose.yml @@ -51,7 +51,7 @@ services: ports: - 3939:3939 volumes: - - $PWD/docker/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect-cli.gcfg + - $PWD/docker/rstudio-connect-cli.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg privileged: true environment: RSTUDIO_CONNECT_HASTE: "enabled" @@ -63,7 +63,7 @@ services: # used jupyter-notebook and deploy tests in CI, main.yml connect: hostname: connect - image: rstudio/rstudio-connect:bionic + image: rstudio/rstudio-connect:jammy restart: always ports: - 3939:3939 From b443e80aba3c9a0c80beeaa48fac4e4c40711f40 Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 11:09:43 -0400 Subject: [PATCH 36/37] update conf --- integration-testing/docker/rstudio-connect.gcfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration-testing/docker/rstudio-connect.gcfg b/integration-testing/docker/rstudio-connect.gcfg index b2e03834..e1550d09 100644 --- a/integration-testing/docker/rstudio-connect.gcfg +++ b/integration-testing/docker/rstudio-connect.gcfg @@ -15,8 +15,8 @@ NoWarning = true [Python] Enabled = true -Executable = /opt/python/3.8.10/bin/python -Executable = /opt/python/3.9.5/bin/python +Executable = /opt/python/3.8.17/bin/python +Executable = /opt/python/3.9.17/bin/python [Quarto] Enabled = true From a71a5d0911c099b66ce38cd799b7d877c2100d0c Mon Sep 17 00:00:00 2001 From: Kevin Gartland Date: Wed, 27 Mar 2024 11:35:56 -0400 Subject: [PATCH 37/37] fix main now --- .github/workflows/main.yml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fc9e94a8..859daf86 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -201,23 +201,7 @@ jobs: run: | pytest tests/test_main_system_caches.py pytest -m 'vetiver' - - name: Run rsconnect-python Tests - env: - CONNECT_LICENSE: "${{ secrets.RSC_LICENSE }}" - PY_VERSION: ${{ matrix.PY_VERSION }} - TEST_SUBSET: "CI" - RSC_AUTOMATION_PAT: "${{ secrets.CONNECT_PAT }}" - ADMIN_API_KEY: "${{ secrets.ADMIN_API_KEY }}" - QUARTO_VERSION: "1.4.546" - - # This allows us to start Connect separately in our own docker container - CONNECT_SERVER: "http://localhost:3939" - remote: "yes" - run: | - cd integration-testing - docker compose pull connect - docker compose up -d connect - just ../test/connect-rsconnect-python/test/rsconnect-python/test-rsconnect-python-repo + test-jupyter: runs-on: ubuntu-latest env: