diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 4a752628b..3d2cf9692 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -13,10 +13,10 @@ Run these as you would any other R test suite with `devtools::test()`. A second suite runs integration tests against a live Connect server running locally in Docker. This has some additional requirements. -- You need a valid Connect license key or file. Put the contents of the license key, or the path to the license file, in the `RSC_LICENSE` environment variable. +- You need a valid Connect license file (`.lic` file). Place it in the root of the repository as `connect-license.lic`. - You need Docker. - If you're running on an ARM (non-Intel) Mac, `export DOCKER_DEFAULT_PLATFORM=linux/amd64` -- Run `connectapi:::build_test_env()` to set up the Connect processes in docker +- Run `connectapi:::build_test_env(connect_license_path = "connect-license.lic")` to set up the Connect processes in docker - By default, this will run against a contemporary version of Connect. To test against an older version, set the environment variable `CONNECT_VERSION` to something else and then run `build_test_env()`. - Set `CONNECTAPI_INTEGRATED=true` in the environment to enable running the integration tests (they're skipped by default). - Run them with `source("tests/test-integrated.R")` diff --git a/.github/local/test-connect-ci.yml b/.github/local/test-connect-ci.yml index 359e67891..197f54ede 100644 --- a/.github/local/test-connect-ci.yml +++ b/.github/local/test-connect-ci.yml @@ -7,13 +7,13 @@ services: ports: - 3939 privileged: true - environment: - - RSC_LICENSE + volumes: + - "${RSC_LICENSE_FILE}:/var/lib/rstudio-connect/license.lic" connect2: hostname: connect2 image: rstudio/rstudio-connect:${CONNECT_VERSION} ports: - 3939 privileged: true - environment: - - RSC_LICENSE + volumes: + - "${RSC_LICENSE_FILE}:/var/lib/rstudio-connect/license.lic" diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index cfbd9390c..42c2e1c4b 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -28,7 +28,6 @@ jobs: env: CONNECT_VERSION: ${{ matrix.version }} GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} CONNECT_LICENSE_FILE: ${{ secrets.CONNECT_LICENSE_FILE }} CONNECTAPI_INTEGRATED: true @@ -54,11 +53,11 @@ jobs: shell: Rscript {0} - name: Set up license file - run: echo "$CONNECT_LICENSE_FILE" > $RSC_LICENSE + run: echo "$CONNECT_LICENSE_FILE" > /tmp/connect.lic - name: Setup test environment run: | - connectapi:::build_test_env() + connectapi:::build_test_env(connect_license_path = "/tmp/connect.lic") shell: Rscript {0} - uses: r-lib/actions/check-r-package@v2 diff --git a/.github/workflows/pkgdown.yaml b/.github/workflows/pkgdown.yaml index 669c7b99a..a7b7bb50e 100644 --- a/.github/workflows/pkgdown.yaml +++ b/.github/workflows/pkgdown.yaml @@ -19,7 +19,6 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} CONNECT_VERSION: 2024.03.0 - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} CONNECT_LICENSE_FILE: ${{ secrets.CONNECT_LICENSE_FILE }} steps: @@ -37,11 +36,11 @@ jobs: needs: website - name: Set up license file - run: echo "$CONNECT_LICENSE_FILE" > $RSC_LICENSE + run: echo "$CONNECT_LICENSE_FILE" > /tmp/connect.lic - name: Start Connect run: | - Rscript -e 'connectapi:::build_test_env()' + Rscript -e 'connectapi:::build_test_env(connect_license_path = "/tmp/connect.lic")' - name: Build site run: pkgdown::build_site_github_pages(new_process = FALSE, install = FALSE) diff --git a/.github/workflows/pr-commands.yaml b/.github/workflows/pr-commands.yaml index 2d3f44aa9..1cbe16268 100644 --- a/.github/workflows/pr-commands.yaml +++ b/.github/workflows/pr-commands.yaml @@ -14,7 +14,6 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} CONNECT_VERSION: 2024.03.0 - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} CONNECT_LICENSE_FILE: ${{ secrets.CONNECT_LICENSE_FILE }} CONNECTAPI_INTEGRATED: true steps: @@ -32,11 +31,11 @@ jobs: needs: coverage - name: Set up license file - run: echo "$CONNECT_LICENSE_FILE" > $RSC_LICENSE + run: echo "$CONNECT_LICENSE_FILE" > /tmp/connect.lic - name: Setup test environment run: | - connectapi:::build_test_env() + connectapi:::build_test_env(connect_license_path = "/tmp/connect.lic") shell: Rscript {0} - uses: r-lib/actions/check-r-package@v2 diff --git a/.github/workflows/test-coverage.yaml b/.github/workflows/test-coverage.yaml index 54bda9426..42fc3ce72 100644 --- a/.github/workflows/test-coverage.yaml +++ b/.github/workflows/test-coverage.yaml @@ -14,7 +14,6 @@ jobs: env: GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} CONNECT_VERSION: 2024.03.0 - RSC_LICENSE: ${{ secrets.RSC_LICENSE }} CONNECT_LICENSE_FILE: ${{ secrets.CONNECT_LICENSE_FILE }} CONNECTAPI_INTEGRATED: true @@ -34,11 +33,11 @@ jobs: needs: coverage - name: Set up license file - run: echo "$CONNECT_LICENSE_FILE" > $RSC_LICENSE + run: echo "$CONNECT_LICENSE_FILE" > /tmp/connect.lic - name: Setup test environment run: | - connectapi:::build_test_env() + connectapi:::build_test_env(connect_license_path = "/tmp/connect.lic") shell: Rscript {0} - name: Test coverage diff --git a/Makefile b/Makefile index 6df74bda0..3a859db0e 100644 --- a/Makefile +++ b/Makefile @@ -48,27 +48,17 @@ mail-down: connect-up: NETWORK=${NETWORK} \ - RSC_LICENSE=$(RSC_LICENSE) \ - CONNECT_VERSION=$(CONNECT_VERSION) \ - docker compose -f inst/ci/test-connect.yml -f .github/local/make-network.yml up -d - -connect-down: - NETWORK=${NETWORK} \ - docker compose -f inst/ci/test-connect.yml -f .github/local/make-network.yml down - -connect-file-up: - NETWORK=${NETWORK} \ - RSC_LICENSE=$(RSC_LICENSE) \ + RSC_LICENSE_FILE=$(RSC_LICENSE_FILE) \ CONNECT_VERSION=$(CONNECT_VERSION) \ docker compose -f inst/ci/test-connect-lic.yml -f .github/local/make-network.yml up -d -connect-file-down: +connect-down: NETWORK=${NETWORK} \ docker compose -f inst/ci/test-connect-lic.yml -f .github/local/make-network.yml down test-env-up: NETWORK=${NETWORK} \ - RSC_LICENSE=$(RSC_LICENSE) \ + RSC_LICENSE_FILE=$(RSC_LICENSE_FILE) \ CONNECT_VERSION=$(CONNECT_VERSION) \ docker compose -f .github/local/test-connect-ci.yml -f .github/local/make-network.yml up -d diff --git a/R/utils-ci.R b/R/utils-ci.R index 1b0974ed0..42eabff6b 100644 --- a/R/utils-ci.R +++ b/R/utils-ci.R @@ -1,33 +1,5 @@ # nocov start -# TODO: A nicer way to execute these system commands... -# - debug output... better error handling... etc. - -determine_license_env <- function(license) { - if (fs::file_exists(license) && fs::path_ext(license) == "lic") { - # Docker needs this to be an absolute path - license <- fs::path_abs(license) - cat_line("determine_license: looks like a license file") - return(list( - type = "file", - cmd_params = c( - "-v", - paste0(license, ":/var/lib/rstudio-connect/license.lic") - ), - env_params = c(RSC_LICENSE_FILE = license) - )) - } else { - cat_line("determine_license: looks like a license key") - return(list( - type = "key", - cmd_params = c(), - env_params = c( - RSC_LICENSE = license - ) - )) - } -} - version_to_docker_tag <- function(version) { # Prior to 2022.09.0, the plain version number was the tag # After, it's "-" @@ -49,27 +21,25 @@ version_to_docker_tag <- function(version) { } compose_start <- function( - connect_license = Sys.getenv("RSC_LICENSE"), + connect_license_path, connect_version, clean = TRUE ) { warn_dire("compose_start") scoped_dire_silence() - stopifnot(nchar(connect_license) > 0) + stopifnot(fs::file_exists(connect_license_path)) + connect_license_path <- fs::path_abs(connect_license_path) - license_details <- determine_license_env(connect_license) - compose_file <- switch( - license_details$type, - "file" = "ci/test-connect-lic.yml", - "ci/test-connect.yml" + compose_file_path <- system.file( + "ci/test-connect-lic.yml", + package = "connectapi" ) - compose_file_path <- system.file(compose_file, package = "connectapi") env_vars <- c( CONNECT_VERSION = version_to_docker_tag(connect_version), PATH = Sys.getenv("PATH"), - license_details$env_params + RSC_LICENSE_FILE = connect_license_path ) # system2 needs a character vector of name=value env_vars <- paste(names(env_vars), env_vars, sep = "=") @@ -105,9 +75,13 @@ compose_find_hosts <- function(prefix) { containers <- grep(prefix, docker_ps_output, value = TRUE) ports <- sub(".*0\\.0\\.0\\.0:([0-9]+)->3939.*", "\\1", containers) + container_names <- sub(".*\\s+([^ ]+)$", "\\1", containers) cat_line(glue::glue("docker: got ports {ports[1]} and {ports[2]}")) - paste0("http://localhost:", ports) + list( + hosts = paste0("http://localhost:", ports), + container_names = container_names + ) } @@ -143,7 +117,7 @@ update_renviron_creds <- function( } build_test_env <- function( - connect_license = Sys.getenv("RSC_LICENSE"), + connect_license_path, clean = TRUE, username = "admin", password = "admin0", @@ -153,16 +127,16 @@ build_test_env <- function( scoped_dire_silence() compose_start( - connect_license = connect_license, + connect_license_path = connect_license_path, clean = clean, connect_version = connect_version ) # It was ci_connect before but it's ci-connect on my machine now; # this is a regex so it will match either - hosts <- compose_find_hosts(prefix = "ci.connect") + container_info <- compose_find_hosts(prefix = "ci.connect") - wait_for_connect_ready <- function(host, timeout = 120) { + wait_for_connect_ready <- function(host, container_name, timeout = 120) { client <- HackyConnect$new(server = host, api_key = NULL) start_time <- Sys.time() last_msg <- start_time @@ -175,7 +149,8 @@ build_test_env <- function( { res <- client$GET(url = client$server_url("__ping__"), parser = NULL) httr::status_code(res) == 200 - } + }, + silent = TRUE ) if (isTRUE(ok)) { return(invisible(TRUE)) @@ -186,21 +161,58 @@ build_test_env <- function( } Sys.sleep(1) } + + # Before failing, capture diagnostics + cat_line("Connect did not become ready in time. Capturing diagnostics...") + cat_line(glue::glue("=== Docker logs for {container_name} ===")) + try( + { + logs <- system2( + "docker", + c("logs", "--tail", "100", container_name), + stdout = TRUE, + stderr = TRUE + ) + cat(logs, sep = "\n") + }, + silent = TRUE + ) + + cat_line(glue::glue("=== Docker inspect for {container_name} ===")) + try( + { + inspect <- system2( + "docker", + c("inspect", container_name), + stdout = TRUE, + stderr = TRUE + ) + cat(inspect, sep = "\n") + }, + silent = TRUE + ) + stop("Connect did not become ready in time: ", ping_url) } - wait_for_connect_ready(hosts[1]) - wait_for_connect_ready(hosts[2]) + wait_for_connect_ready( + container_info$hosts[1], + container_name = container_info$container_names[1] + ) + wait_for_connect_ready( + container_info$hosts[2], + container_name = container_info$container_names[2] + ) cat_line("connect: creating first admin...") a1 <- create_first_admin( - hosts[1], + container_info$hosts[1], "admin", "admin0", "admin@example.com" ) a2 <- create_first_admin( - hosts[2], + container_info$hosts[2], "admin", "admin0", "admin@example.com" diff --git a/inst/ci/test-connect.yml b/inst/ci/test-connect.yml deleted file mode 100644 index cddecddea..000000000 --- a/inst/ci/test-connect.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: '2.3' -services: - - connect: - hostname: connect - image: ghcr.io/rstudio/rstudio-connect:${CONNECT_VERSION} - scale: 2 - ports: - - 3939 - privileged: true - environment: - - RSC_LICENSE - - RSC_HASTE_ENABLED=true - platform: linux/amd64 - # if you need to customize Connect config until we have env vars - # volumes: - # - ../../tmp.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg