Merge branch 'master' into 177-cacert-option

tdstein · web-flow · commit ec4b5bff166c · 2023-01-19T10:32:18.000-05:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -146,3 +146,29 @@ jobs:
             repo: 'rsconnect-jupyter',
             event_type: 'rsconnect_python_latest'
           })
+  test-rsconnect:
+    name: "Test vetiver"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+          python -m pip install -r vetiver-testing/vetiver-requirements.txt
+      - name: run RStudio Connect
+        run: |
+          docker-compose up --build -d
+          pip freeze > requirements.txt
+          make dev
+        env:
+          RSC_LICENSE: ${{ secrets.RSC_LICENSE }}
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+      # NOTE: edited to run checks for python package
+      - name: Run tests
+        run: |
+          pytest -m 'vetiver'
diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml
@@ -7,24 +7,23 @@ on:
 env:
   SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
   SNYK_ORG: rstudio-connect
-  SNYK_PROJECT: rsconnect-python
 
 jobs:
-  python-dependencies:
+  snyk-monitor:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@master
-      - name: Run Snyk on dependencies
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Run Snyk (setup.py)
         uses: snyk/actions/python@master
         with:
           command: monitor
-          args: --file=setup.py --print-deps --project-name=${{ env.SNYK_PROJECT }} --org=${{ env.SNYK_ORG }}
-  python-code:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@master
-      - name: Run Snyk static analysis
+          args: --file=setup.py --package-manager=pip --project-name=setup.py --org=${{ env.SNYK_ORG }}
+
+      - name: Run Snyk (requirements.txt)
         uses: snyk/actions/python@master
         with:
-          command: code test
-          args: --project-name=${{ env.SNYK_PROJECT }} --org=${{ env.SNYK_ORG }} rsconnect/
+          command: monitor
+          args: --file=requirements.txt --package-manager=pip --project-name=requirements.txt --org=${{ env.SNYK_ORG }}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Removed
+
+- `rsconnect-python` no longer considers the `RETICULATE_PYTHON` environment variable.
+  In environments where `RETICULATE_PYTHON` is set outside a project context (e.g. by a Posit Workbench administrator),
+  attempting to deploy content or write manifests in projects using virtual environments required explicitly setting `--python /path/to/virtualenv/python`.
+  Removing `RETICULATE_PYTHON` detection should simplify the use of the CLI in this case.
+
 ## [1.13.0] - 2022-12-02
 
 ### Added
diff --git a/Makefile b/Makefile
@@ -20,6 +20,8 @@ endif
 
 TEST_ENV =
 
+RSC_API_KEYS=vetiver-testing/rsconnect_api_keys.json
+
 ifneq ($(CONNECT_SERVER),)
   TEST_ENV += CONNECT_SERVER=$(CONNECT_SERVER)
 endif
@@ -165,3 +167,21 @@ promote-docs-in-s3:
 		--cache-control max-age=300 \
 		docs/site/ \
 		s3://docs.rstudio.com/rsconnect-python/
+
+
+dev: vetiver-testing/rsconnect_api_keys.json
+
+dev-start:
+	docker-compose up -d
+	docker-compose exec -T rsconnect bash < vetiver-testing/setup-rsconnect/add-users.sh
+	# curl fails with error 52 without a short sleep....
+	sleep 5
+	curl -s --retry 10 --retry-connrefused http://localhost:3939
+
+dev-stop:
+	docker-compose down
+	rm -f $(RSC_API_KEYS)
+
+$(RSC_API_KEYS): dev-start
+	python vetiver-testing/setup-rsconnect/dump_api_keys.py $@
+	
diff --git a/README.md b/README.md
@@ -220,8 +220,7 @@ specified package list.
 
 If there is no `requirements.txt` file or the `--force-generate` option is specified,
 the package dependencies will be determined from the current Python environment, or
-from an alternative Python executable specified via the `--python` option or via the
-`RETICULATE_PYTHON` environment variable:
+from an alternative Python executable specified via the `--python` option:
 
 ```bash
 rsconnect deploy notebook --python /path/to/python my-notebook.ipynb
@@ -255,8 +254,7 @@ in a later deployment. Use the `write-manifest` command to do this.
 The `write-manifest` command will also create a `requirements.txt` file, if it does
 not already exist or the `--force-generate` option is specified. It will contain the
 package dependencies from the current Python environment, or from an alternative
-Python executable specified in the `--python` option or via the `RETICULATE_PYTHON`
-environment variable.
+Python executable specified in the `--python` option.
 
 Here is an example of the `write-manifest` command:
 
@@ -349,8 +347,7 @@ the specified package list.
 
 If there is no `requirements.txt` file or the `--force-generate` option is specified,
 the package dependencies will be determined from the current Python environment, or
-from an alternative Python executable specified via the `--python` option or via the
-`RETICULATE_PYTHON` environment variable:
+from an alternative Python executable specified via the `--python` option:
 
 ```bash
 rsconnect deploy api --python /path/to/python my-api/
@@ -371,8 +368,7 @@ manifest in a later deployment. Use the `write-manifest` command to do this.
 The `write-manifest` command will also create a `requirements.txt` file, if it does
 not already exist or the `--force-generate` option is specified. It will contain
 the package dependencies from the current Python environment, or from an alternative
-Python executable specified in the `--python` option or via the `RETICULATE_PYTHON`
-environment variable.
+Python executable specified in the `--python` option.
 
 Here is an example of the `write-manifest` command:
 
diff --git a/conftest.py b/conftest.py
@@ -1,7 +1,22 @@
 import sys
+import pytest
 
 from os.path import abspath, dirname
 
 
 HERE = dirname(abspath(__file__))
 sys.path.insert(0, HERE)
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--vetiver", action="store_true", default=False, help="run vetiver tests"
+    )
+
+def pytest_configure(config):
+    config.addinivalue_line("markers", "vetiver: test for vetiver interaction")
+
+def pytest_collection_modifyitems(config, items):
+    if config.getoption("--vetiver"):
+        return
+    skip_vetiver = pytest.mark.skip(reason="need --vetiver option to run")
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -0,0 +1,17 @@
+version: '3.2'
+
+services:
+
+  rsconnect:
+    image: rstudio/rstudio-connect:latest
+    restart: always
+    ports:
+      - 3939:3939
+    volumes:
+      - $PWD/vetiver-testing/setup-rsconnect/users.txt:/etc/users.txt
+      - $PWD/vetiver-testing/setup-rsconnect/rstudio-connect.gcfg:/etc/rstudio-connect/rstudio-connect.gcfg
+    # by default, mysql rounds to 4 decimals, but tests require more precision
+    privileged: true
+    environment:
+      RSTUDIO_CONNECT_HASTE: "enabled"
+      RSC_LICENSE: ${RSC_LICENSE}
diff --git a/pyproject.toml b/pyproject.toml
@@ -10,3 +10,8 @@ omit = ["tests/*"]
 
 [tool.setuptools_scm]
 write_to = "rsconnect/version.py"
+
+[tool.pytest.ini_options]
+markers = [
+    "vetiver: tests for vetiver",
+]
diff --git a/rsconnect/actions.py b/rsconnect/actions.py
@@ -113,7 +113,6 @@ def which_python(python, env=os.environ):
 
     In priority order:
     * --python specified on the command line
-    * RETICULATE_PYTHON defined in the environment
     * the python binary running this script
     """
     warn("This method has been moved and will be deprecated.", DeprecationWarning, stacklevel=2)
@@ -122,9 +121,6 @@ def which_python(python, env=os.environ):
             raise RSConnectException('The file, "%s", does not exist or is not executable.' % python)
         return python
 
-    if "RETICULATE_PYTHON" in env:
-        return os.path.expanduser(env["RETICULATE_PYTHON"])
-
     return sys.executable
 
 
diff --git a/rsconnect/bundle.py b/rsconnect/bundle.py
@@ -1080,17 +1080,13 @@ def which_python(python, env=os.environ):
 
     In priority order:
     * --python specified on the command line
-    * RETICULATE_PYTHON defined in the environment
     * the python binary running this script
     """
     if python:
         if not (exists(python) and os.access(python, os.X_OK)):
             raise RSConnectException('The file, "%s", does not exist or is not executable.' % python)
         return python
 
-    if "RETICULATE_PYTHON" in env:
-        return os.path.expanduser(env["RETICULATE_PYTHON"])
-
     return sys.executable
 
 
diff --git a/tests/test_bundle.py b/tests/test_bundle.py
@@ -652,7 +652,6 @@ def test_which_python(self):
 
         self.assertEqual(which_python(sys.executable), sys.executable)
         self.assertEqual(which_python(None), sys.executable)
-        self.assertEqual(which_python(None, {"RETICULATE_PYTHON": "fake-python"}), "fake-python")
 
     def test_default_title(self):
         self.assertEqual(_default_title("testing.txt"), "testing")
diff --git a/tests/test_vetiver_pins.py b/tests/test_vetiver_pins.py
@@ -0,0 +1,95 @@
+import pytest
+
+vetiver = pytest.importorskip("vetiver", reason="vetiver library not installed")
+
+import json # noqa
+import pins # noqa
+import pandas as pd # noqa
+import numpy as np # noqa
+
+from pins.boards import BoardRsConnect  # noqa
+from pins.rsconnect.api import RsConnectApi  # noqa
+from pins.rsconnect.fs import RsConnectFs  # noqa
+from rsconnect.api import RSConnectServer, RSConnectClient # noqa
+
+RSC_SERVER_URL = "http://localhost:3939"
+RSC_KEYS_FNAME = "vetiver-testing/rsconnect_api_keys.json"
+
+pytestmark = pytest.mark.vetiver  # noqa
+
+
+def get_key(name):
+    with open(RSC_KEYS_FNAME) as f:
+        api_key = json.load(f)[name]
+        return api_key
+
+
+def rsc_from_key(name):
+    with open(RSC_KEYS_FNAME) as f:
+        api_key = json.load(f)[name]
+        return RsConnectApi(RSC_SERVER_URL, api_key)
+
+
+def rsc_fs_from_key(name):
+
+    rsc = rsc_from_key(name)
+
+    return RsConnectFs(rsc)
+
+
+def rsc_delete_user_content(rsc):
+    guid = rsc.get_user()["guid"]
+    content = rsc.get_content(owner_guid=guid)
+    for entry in content:
+        rsc.delete_content_item(entry["guid"])
+
+
+@pytest.fixture(scope="function")
+def rsc_short():
+    # tears down content after each test
+    fs_susan = rsc_fs_from_key("susan")
+
+    # delete any content that might already exist
+    rsc_delete_user_content(fs_susan.api)
+
+    yield BoardRsConnect("", fs_susan, allow_pickle_read=True)  # fs_susan.ls to list content
+
+    rsc_delete_user_content(fs_susan.api)
+
+
+def test_deploy(rsc_short):
+    np.random.seed(500)
+
+    # Load data, model
+    X_df, y = vetiver.mock.get_mock_data()
+    model = vetiver.mock.get_mock_model().fit(X_df, y)
+
+    v = vetiver.VetiverModel(model=model, ptype_data=X_df, model_name="susan/model")
+
+    board = pins.board_rsconnect(server_url=RSC_SERVER_URL, api_key=get_key("susan"), allow_pickle_read=True)
+
+    vetiver.vetiver_pin_write(board=board, model=v)
+    connect_server = RSConnectServer(url=RSC_SERVER_URL, api_key=get_key("susan"))
+
+    vetiver.deploy_rsconnect(
+        connect_server=connect_server,
+        board=board,
+        pin_name="susan/model",
+        title="testapivetiver",
+        extra_files=["requirements.txt"],
+    )
+
+    # get url of where content lives
+    client = RSConnectClient(connect_server)
+    dicts = client.content_search()
+    rsc_api = list(filter(lambda x: x["title"] == "testapivetiver", dicts))
+    content_url = rsc_api[0].get("content_url")
+
+    h = {"Authorization": 'Key {}'.format(get_key("susan"))}
+
+    endpoint = vetiver.vetiver_endpoint(content_url + "/predict")
+    response = vetiver.predict(endpoint, X_df, headers=h)
+
+    assert isinstance(response, pd.DataFrame), response
+    assert response.iloc[0, 0] == 44.47
+    assert len(response) == 100
diff --git a/vetiver-testing/setup-rsconnect/add-users.sh b/vetiver-testing/setup-rsconnect/add-users.sh
@@ -0,0 +1 @@
+awk ' { system("useradd -m -s /bin/bash "$1); system("echo \""$1":"$2"\" | chpasswd"); system("id "$1) } ' /etc/users.txt
diff --git a/vetiver-testing/setup-rsconnect/dump_api_keys.py b/vetiver-testing/setup-rsconnect/dump_api_keys.py
@@ -0,0 +1,21 @@
+import json
+import sys
+
+from pins.rsconnect.api import _HackyConnect
+
+OUT_FILE = sys.argv[1]
+
+
+def get_api_key(user, password, email):
+    rsc = _HackyConnect("http://localhost:3939")
+
+    return rsc.create_first_admin(user, password, email).api_key
+
+
+api_keys = {
+    "admin": get_api_key("admin", "admin0", "admin@example.com"),
+    "susan": get_api_key("susan", "susan", "susan@example.com"),
+    "derek": get_api_key("derek", "derek", "derek@example.com"),
+}
+
+json.dump(api_keys, open(OUT_FILE, "w"))
diff --git a/vetiver-testing/setup-rsconnect/rstudio-connect.gcfg b/vetiver-testing/setup-rsconnect/rstudio-connect.gcfg
@@ -0,0 +1,23 @@
+[Server]
+DataDir = /data
+Address = http://localhost:3939
+
+[HTTP]
+Listen = :3939
+
+[Authentication]
+Provider = pam
+
+[Authorization]
+DefaultUserRole = publisher
+
+[Python]
+Enabled = true
+Executable = /opt/python/3.8.10/bin/python
+Executable = /opt/python/3.9.5/bin/python
+
+[RPackageRepository "CRAN"]
+URL = https://packagemanager.rstudio.com/cran/__linux__/bionic/latest
+
+[RPackageRepository "RSPM"]
+URL = https://packagemanager.rstudio.com/cran/__linux__/bionic/latest
diff --git a/vetiver-testing/setup-rsconnect/users.txt b/vetiver-testing/setup-rsconnect/users.txt
@@ -0,0 +1,4 @@
+admin admin0
+test  test
+susan susan
+derek derek
diff --git a/vetiver-testing/vetiver-requirements.txt b/vetiver-testing/vetiver-requirements.txt
