diff --git a/CHANGELOG.md b/CHANGELOG.md index dbeb480f..aa7930ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the name of the environment variables to the help output for those options that use environment variables as a default value. - Added support for deploying Shiny Express applications. +- Added a `--retry` flag to the `rsconnect content build run` command to re-run + builds for all content in the NEEDS_BUILD, ABORTED, ERROR, or RUNNING state. ### Changed - Improved the error and warning outputs when options conflict by providing the source @@ -19,6 +21,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated verbose mode to output the source of all options being used when processing the CLI command. +### Fixed +- Interrupting a long-running `rsconnect content build run` command with `^C` + will now update the local state file before attempting graceful cleanup. This + should help prevent users from getting stuck a "build already running" state. + See [#467](https://github.com/rstudio/rsconnect-python/issues/467) for details. + ## [1.21.0] - 2023-10-26 ### Fixed diff --git a/Makefile b/Makefile index 280ae9f9..68135bc7 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ ifneq ($(GITHUB_RUN_ID),) endif TEST_ENV = +TEST_ENV += CONNECT_CONTENT_BUILD_DIR="rsconnect-build-test" ifneq ($(CONNECT_SERVER),) TEST_ENV += CONNECT_SERVER=$(CONNECT_SERVER) @@ -26,9 +27,6 @@ endif ifneq ($(CONNECT_API_KEY),) TEST_ENV += CONNECT_API_KEY=$(CONNECT_API_KEY) endif -ifneq ($(CONNECT_CONTENT_BUILD_DIR),) - TEST_ENV += CONNECT_CONTENT_BUILD_DIR=$(CONNECT_CONTENT_BUILD_DIR) -endif # NOTE: See the `dist` target for why this exists. SOURCE_DATE_EPOCH := $(shell date +%s) @@ -49,16 +47,6 @@ shell-%: test-%: PYTHON_VERSION=$* $(RUNNER) '$(TEST_ENV) $(TEST_COMMAND)' -mock-test-%: clean-stores - @$(MAKE) -C mock_connect image up - @sleep 1 - trap "$(MAKE) -C mock_connect down" EXIT; \ - CONNECT_CONTENT_BUILD_DIR="rsconnect-build-test" \ - CONNECT_SERVER="http://$(HOSTNAME):3939" \ - CONNECT_API_KEY="0123456789abcdef0123456789abcdef" \ - $(MAKE) test-$* - @$(MAKE) -C mock_connect down - fmt-%: $(RUNNER) 'black .' diff --git a/README.md b/README.md index 24ce84e8..78774eec 100644 --- a/README.md +++ b/README.md @@ -823,6 +823,9 @@ Once the content items have been added, you may initiate a build using the `rsconnect content build run` subcommand. This command will attempt to build all "tracked" content that has the status `NEEDS_BUILD`. +> To re-run failed builds, use `rsconnect content build run --retry`. This will build +all tracked content in any of the following states: `[NEEDS_BUILD, ABORTED, ERROR, RUNNING]`. + ```bash rsconnect content build run # [INFO] 2021-12-14T13:02:45-0500 Initializing ContentBuildStore for https://connect.example.org:3939 diff --git a/mock_connect/.gitignore b/mock_connect/.gitignore deleted file mode 100644 index 81d3c210..00000000 --- a/mock_connect/.gitignore +++ /dev/null @@ -1 +0,0 @@ -mock_connect.log diff --git a/mock_connect/Dockerfile b/mock_connect/Dockerfile deleted file mode 100644 index da015748..00000000 --- a/mock_connect/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM python:3.7-alpine -MAINTAINER Posit Connect - -# Add the Python packags we need. -RUN pip install flask==2.1.3 diff --git a/mock_connect/Makefile b/mock_connect/Makefile deleted file mode 100644 index ac605b2a..00000000 --- a/mock_connect/Makefile +++ /dev/null @@ -1,49 +0,0 @@ -# -# Makefile for our mock Connect server. -# -# The targets here provide all the support for running a mock version of Connect. -# -MOCK_IMAGE?="rstudio/connect:mock" -MOCK_HOST?="mock-connect" -PRE_FETCH_FILE?="data.json" -LOG_FILE?="mock_connect.log" - -# Build the image in which the mock server will run. -image: - docker build -t ${MOCK_IMAGE} . - -# Bring the mock Connect server up. Note that it is run as a daemon. -up: - @echo "Starting ${MOCK_HOST} ..." - @docker run --rm -d -t --init \ - --name ${MOCK_HOST} \ - --volume $(CURDIR):/rsconnect \ - --volume $(CURDIR)/../tests/testdata:/testdata \ - --env=FLASK_APP=/rsconnect/mock_connect/main.py \ - --env=PRE_FETCH_FILE=${PRE_FETCH_FILE} \ - --publish 3939:3939 \ - --workdir /rsconnect \ - ${MOCK_IMAGE} \ - flask run --host=0.0.0.0 --port=3939 1>/dev/null - @docker logs -f ${MOCK_HOST} > ${LOG_FILE} & - -# Bring the mock Connect server down. -down: - $(eval CONNECT_EXISTS=$(shell docker container inspect ${MOCK_HOST} > /dev/null 2>&1 && echo 0 || echo 1)) - @if [ "${CONNECT_EXISTS}" = "0" ] ; then \ - echo "Stopping ${MOCK_HOST} ..."; \ - docker stop ${MOCK_HOST} 1>/dev/null; \ - fi; - -# Clean up after ourselves. -clean: - @rm -f ${LOG_FILE} - -# Start the mock Connect docker image but just dump into a shell. -sh: - @docker run --name mock_connect --rm -it \ - -v $(CURDIR):/rsconnect -w /rsconnect \ - ${MOCK_IMAGE} sh - -# Nothing is real... -.PHONY: image up down clean sh diff --git a/mock_connect/README.md b/mock_connect/README.md deleted file mode 100644 index 5b25296a..00000000 --- a/mock_connect/README.md +++ /dev/null @@ -1,61 +0,0 @@ -## Mock Posit Connect - -This directory holds a mocked version of Posit Connect to support testing. This -works by providing a Docker container equipped with Python (and necessary support -libraries). The container looks to the outside world like an installation of Posit -Connect. - -### Restrictions - -- The mock Posit Connect does not support HTTPS connections; only HTTP. -- Not all Connect endpoints are implemented; only those needed to verify `rsconnect-python` - functionality. - -## Data Handling - -The mock server does everything in memory, rather than use an external database. If you -want to see what's currently there, visit the root index page of the mock server -(`http://localhost:3939/` by default) and you'll see a dump of them. This is useful in -seeing the results of updating APIs. - -The server comes with only one predefined object defined, the "admin" user. It does allow -you to preload data if you wish. This is intended to provide support for specific testing -scenarios. To preload data, create a JSON file containing the data you want loaded. See -the provided [data file](data.json) for the basic structure. Look at the -[data code file](mock_connect/data.py) file for supported attributes for each object. - -If the JSON data for an object does not contain an `id` attribute, a unique one will be -automatically assigned. As new objects are created, each ID is guaranteed to be unique. -For objects that have a `guid` attribute, these also are automatically filled in if not -provided. All other attribute values (even creation dates) are the responsibility of -the data file. - -The definition of a user in the preload data file may contain an extra attribute called -`api_key` to specify an API key for the user. - -Once you have the JSON file with the data you want to preload, specify its name as the value -of the `PRE_FETCH_FILE` environment variable when invoking `make up` to start the server. - -## The `Makefile` - -The `Makefile` contains several useful targets. - -- `image` -- This target builds the Docker image in which the mock Connect server will be run. -- `up` -- This target starts the Docker container, exposing the mock server on port `3939`. - The container is started as a daemon. The output of the server is directed to a file - called `mock_connect.log` (use the `LOG_FILE` environment variable to change this). -- `down` -- This target stops the Docker container running the mock server. -- `clean` -- This target cleans up work files, like the log file. -- `sh` -- This target brings up the Docker container but dumps you into a `sh` shell rather - than starting the mock server. - -> **Note:** -> - There is no `all` target. -> - `make` without a target will run the `image` target. -> - There are no dependencies between targets. - -## Future - -In the future, the pre-fetch data file will also allow for tailoring server responses based -on request input. This makes it significantly easier to simulate a variety of errors so -that non-happy paths may also be fully tested. diff --git a/mock_connect/data.json b/mock_connect/data.json deleted file mode 100644 index c2bacf8d..00000000 --- a/mock_connect/data.json +++ /dev/null @@ -1,162 +0,0 @@ -{ - "apps": [ - { - "id": "121", - "guid": "015143da-b75f-407c-81b1-99c4a724341e", - "name": "plumber-async", - "title": "plumber-async", - "owner_username": "bob", - "owner_first_name": "Bobby", - "owner_last_name": "McBobface", - "owner_email": "bob@posit.co", - "owner_locked": false - } - ], - "users": [ - { - "api_key": "0123456789abcdef0123456789abcdef", - "guid": "29a74070-2c13-4ef9-a898-cfc6bcf0f275", - "username": "admin", - "first_name": "Super", - "last_name": "User", - "email": "admin@example.com", - "user_role": "administrator", - "password": "", - "confirmed": true, - "locked": false, - "created_time": "2018-08-29T19:25:23.68280816Z", - "active_time": "2018-08-30T23:49:18.421238194Z", - "updated_time": "2018-08-29T19:25:23.68280816Z", - "privileges": [ - "add_users", - "add_vanities", - "change_app_permissions", - "change_apps", - "change_groups", - "change_usernames", - "change_users", - "change_variant_schedule", - "create_groups", - "edit_run_as", - "edit_runtime", - "lock_users", - "publish_apps", - "remove_apps", - "remove_groups", - "remove_users", - "remove_vanities", - "view_app_settings", - "view_apps" - ] - } - ], - "bundles": [ - { - "id": 142, - "app_id": "18", - "_tar_file": "/testdata/bundle.tar.gz" - } - ], - "content": [ - { - "guid": "015143da-b75f-407c-81b1-99c4a724341e", - "name": "plumber-async", - "title": "plumber-async", - "description": "", - "access_type": "acl", - "connection_timeout": 3600, - "read_timeout": 3600, - "init_timeout": 60, - "idle_timeout": 120, - "max_processes": 3, - "min_processes": 0, - "max_conns_per_process": 50, - "load_factor": 0.2, - "created_time": "2021-11-01T20:43:32Z", - "last_deployed_time": "2021-11-30T16:56:21Z", - "bundle_id": "176", - "app_mode": "api", - "content_category": "api", - "parameterized": false, - "cluster_name": "Local", - "image_name": null, - "r_version": "3.6.3", - "py_version": null, - "quarto_version": null, - "run_as": null, - "run_as_current_user": false, - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "content_url": "http://localhost:3939/content/015143da-b75f-407c-81b1-99c4a724341e/", - "dashboard_url": "http://localhost:3939/connect/#/apps/015143da-b75f-407c-81b1-99c4a724341e", - "app_role": "owner", - "id": "121" - }, - { - "guid": "4ffc819c-065c-420c-88eb-332db1133317", - "name": "logs-api-python", - "title": "logs-api-python", - "description": "", - "access_type": "acl", - "connection_timeout": null, - "read_timeout": null, - "init_timeout": null, - "idle_timeout": null, - "max_processes": null, - "min_processes": null, - "max_conns_per_process": null, - "load_factor": null, - "created_time": "2021-07-19T19:17:32Z", - "last_deployed_time": "2021-11-30T16:56:21Z", - "bundle_id": "142", - "app_mode": "python-api", - "content_category": "", - "parameterized": false, - "cluster_name": "Local", - "image_name": null, - "r_version": null, - "py_version": "3.8.2", - "quarto_version": null, - "run_as": null, - "run_as_current_user": false, - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "content_url": "http://localhost:3939/content/4ffc819c-065c-420c-88eb-332db1133317/", - "dashboard_url": "http://localhost:3939/connect/#/apps/4ffc819c-065c-420c-88eb-332db1133317", - "app_role": "owner", - "id": "18" - }, - { - "guid": "bcc74209-3a81-4b9c-acd5-d24a597c256c", - "name": "static", - "title": "static", - "description": "", - "access_type": "acl", - "connection_timeout": null, - "read_timeout": null, - "init_timeout": null, - "idle_timeout": null, - "max_processes": null, - "min_processes": null, - "max_conns_per_process": null, - "load_factor": null, - "created_time": "2021-11-30T14:33:57Z", - "last_deployed_time": "2021-11-30T15:51:07Z", - "bundle_id": "195", - "app_mode": "rmd-static", - "content_category": "", - "parameterized": false, - "cluster_name": "Local", - "image_name": null, - "r_version": "3.6.3", - "py_version": null, - "quarto_version": null, - "run_as": null, - "run_as_current_user": false, - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "content_url": "http://localhost:3939/content/bcc74209-3a81-4b9c-acd5-d24a597c256c/", - "dashboard_url": "http://localhost:3939/connect/#/apps/bcc74209-3a81-4b9c-acd5-d24a597c256c", - "app_role": "owner", - "id": "135" - } - ], - "tasks": [] -} diff --git a/mock_connect/mock_connect/__init__.py b/mock_connect/mock_connect/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/mock_connect/mock_connect/data.py b/mock_connect/mock_connect/data.py deleted file mode 100644 index 1b012805..00000000 --- a/mock_connect/mock_connect/data.py +++ /dev/null @@ -1,402 +0,0 @@ -""" -This provides our "database" handling. It defines our object models along -with appropriate storage/retrieval actions. -""" -import datetime -import io -import json -import sys -import tarfile -import uuid -from enum import Enum -from json import JSONDecodeError -from os import environ -from os.path import isfile -from typing import Union - - -def timestamp(): - return datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z" - - -def default_url(): - return "http://127.0.0.1:3939" - - -class DBObject(object): - json_excludes = [] # remove fields from the json response - show = ["id"] # show these fields in the generated HTML - generated_id = {} - instances = {} - - @classmethod - def _get_all_ids(cls): - name = cls.__name__ - if name in cls.instances: - return cls.instances[name].keys() - return [] - - @classmethod - def _next_id(cls): - name = cls.__name__ - if name not in cls.generated_id: - cls.generated_id[name] = range(1, sys.maxsize**10).__iter__() - new_id = next(cls.generated_id[name]) - existing_ids = cls._get_all_ids() - while new_id in existing_ids: - existing_ids = cls._get_all_ids() - return new_id - - @classmethod - def get_object(cls, db_id: Union[int, str]): - name = cls.__name__ - if name in cls.instances: - if db_id in cls.instances[name]: - return cls.instances[name][db_id] - elif isinstance(db_id, str): # if guid was provided, find by guid instead of id - return next(filter(lambda x: x.guid == db_id, cls.instances[name].values()), None) - return None - - @classmethod - def get_all_objects(cls): - name = cls.__name__ - if name in cls.instances: - return cls.instances[name].values() - return [] - - @classmethod - def _save(cls, instance): - name = cls.__name__ - if name not in cls.instances: - cls.instances[name] = {} - cls.instances[name][instance.id] = instance - - @classmethod - def get_table_headers(cls): - return "%s" % "".join(cls.show) - - def __init__(self, id: str = None, needs_guid: bool = False, **kwargs): - self.id = int(id) if id else self._next_id() - if needs_guid: - self.guid = kwargs.get("guid", str(uuid.uuid4())) - self._save(self) - - def update_from(self, data: dict): - self.__dict__.update(data) - - def to_dict(self) -> dict: - return self.__dict__.copy() - - def get_table_row(self): - items = [str(self.__getattribute__(item)) for item in self.show] - return "%s" % "".join(items) - - -class AppMode(Enum): - STATIC = 4 - JUPYTER_STATIC = 7 - - @staticmethod - def value_of(name: str): - name = name.upper().replace("-", "_") - return AppMode[name].value if name in AppMode else None - - -class Application(DBObject): - json_excludes = ["_base_url"] - show = ["id", "guid", "name", "title", "url"] - - @classmethod - def get_app_by_name(cls, name: str): - for app in cls.get_all_objects(): - if name == app.name: - return app - return None - - def __init__(self, **kwargs): - super(Application, self).__init__(needs_guid=True, **kwargs) - self._base_url = kwargs.get("_base_url", default_url()) - self.name = kwargs.get("name") - self.title = kwargs.get("title") - self.url = "%scontent/%s" % (self._base_url, self.id) - self.owner_username = kwargs.get("owner_username") - self.owner_first_name = kwargs.get("owner_first_name") - self.owner_last_name = kwargs.get("owner_last_name") - self.owner_email = kwargs.get("owner_email") - self.owner_locked = kwargs.get("owner_locked") - self.bundle_id = None - self.needs_config = True - self.access_type = None - self.description = "" - self.app_mode = None - self.created_time = timestamp() - self.last_deployed_time = None - - def bundle_deployed(self, bundle, new_app_mode): - self.bundle_id = bundle.id - self.app_mode = new_app_mode - self.last_deployed_time = timestamp() - - def get_bundle(self): - if self.bundle_id is not None: - return Bundle.get_object(self.bundle_id) - return None - - -class User(DBObject): - show = ["id", "guid", "username", "first_name", "last_name"] - - @classmethod - def get_user_by_api_key(cls, key: str): - if key in api_keys: - return User.get_object(api_keys[key]) - return None - - def __init__(self, **kwargs): - super(User, self).__init__(needs_guid=True, **kwargs) - self.username = kwargs.get("username") - self.first_name = kwargs.get("first_name") - self.last_name = kwargs.get("last_name") - self.email = kwargs.get("email") - self.user_role = kwargs.get("user_role") - self.password = kwargs.get("password") - self.confirmed = kwargs.get("confirmed") - self.locked = kwargs.get("locked") - self.created_time = kwargs.get("created_time") - self.updated_time = kwargs.get("updated_time") - self.active_time = kwargs.get("active_time") - self.privileges = kwargs.get("privileges") - - -class Bundle(DBObject): - json_excludes = ["_tar_data"] - show = ["id", "app_id"] - - def __init__(self, **kwargs): - super(Bundle, self).__init__(**kwargs) - self.app_id = kwargs.get("app_id") - self.created_time = timestamp() - self.updated_time = self.created_time - self._tar_data = kwargs.get("_tar_data") - self._tar_file = kwargs.get("_tar_file") - - def read_bundle_data(self): - if self._tar_data: - return io.BytesIO(self._tar_data) - elif self._tar_file: - with open(self._tar_file, "rb") as fh: - self._tar_data = fh.read() - return io.BytesIO(self._tar_data) - - def read_bundle_file(self, file_name): - raw_bytes = io.BytesIO(self._tar_data) - with tarfile.open("r:gz", fileobj=raw_bytes) as tar: - return tar.extractfile(file_name).read() - - def get_manifest(self): - manifest_data = self.read_bundle_file("manifest.json").decode("utf-8") - return json.loads(manifest_data) - - def get_rendered_content(self): - manifest = self.get_manifest() - meta = manifest["metadata"] - # noinspection SpellCheckingInspection - file_name = meta.get("primary_html") or meta.get("entrypoint") - return self.read_bundle_file(file_name).decode("utf-8") - - -class Task(DBObject): - def __init__(self, **kwargs): - super(Task, self).__init__(**kwargs) - self.user_id = kwargs.get("user_id", 0) - self.finished = kwargs.get("finished", True) - self.code = kwargs.get("code", 0) - self.error = kwargs.get("error", "") - self.last_status = kwargs.get("last_status", 0) - self.status = kwargs.get("status", ["Building static content", "Deploying static content"]) - - -class Content(DBObject): - json_excludes = ["_base_url"] - show = ["guid", "name", "app_mode", "r_version", "py_version", "quarto_version"] - - def __init__(self, **kwargs): - super(Content, self).__init__(needs_guid=True, **kwargs) - self._base_url = kwargs.get("_base_url", default_url()) - self.name = kwargs.get("name") - self.title = self.name + "+ title" - self.description = self.name + "+ description" - self.bundle_id = kwargs.get("bundle_id") - self.app_mode = kwargs.get("app_mode") - self.content_category = kwargs.get("content_category") - self.content_url = "%scontent/%s/" % (self._base_url, self.guid) - self.dashboard_url = "%sconnect/#/apps/%s" % (self._base_url, self.guid) - self.created_time = timestamp() - self.last_deployed_time = timestamp() - self.r_version = kwargs.get("r_version", "4.1.1") - self.py_version = kwargs.get("py_version", "3.9.9") - self.quarto_version = kwargs.get("quarto_version", "0.2.318") - self.owner_guid = kwargs.get("owner_guid") - self.access_type = kwargs.get("access_type", "acl") - self.connection_timeout = kwargs.get("connection_timeout") - self.read_timeout = kwargs.get("read_timeout") - self.init_timeout = kwargs.get("init_timeout") - self.idle_timeout = kwargs.get("idle_timeout") - self.max_processes = kwargs.get("max_processes") - self.min_processes = kwargs.get("min_processes") - self.max_conns_per_process = kwargs.get("max_conns_per_process") - self.load_factor = kwargs.get("load_factor") - self.parameterized = kwargs.get("parameterized", False) - self.cluster_name = kwargs.get("cluster_name") - self.image_name = kwargs.get("image_name") - self.run_as = kwargs.get("run_as") - self.run_as_current_user = kwargs.get("run_as_current_user", False) - self.app_role = kwargs.get("app_role", "owner") - - -def _apply_pre_fetch_data(data: dict): - for key, value in data.items(): - if key in tag_map: - if isinstance(value, dict): - value = [value] - cls = tag_map[key] - for item in value: - new_object = cls(**item) - if cls == User and "api_key" in item: - api_keys[item["api_key"]] = new_object.id - else: - print("WARNING: Unknown pre-fetch data type: %s" % key) - - -def _pre_fetch_data(): - file_name = environ.get("PRE_FETCH_FILE") - if file_name is not None and len(file_name) > 0: - if isfile(file_name): - try: - with open(file_name) as fd: - _apply_pre_fetch_data(json.load(fd)) - except JSONDecodeError: - print("WARNING: Unable to read pre-fetch file %s as JSON." % file_name) - else: - print("WARNING: Pre-fetch file %s does not exist." % file_name) - - -def _format_section(sections, name, rows): - table = "\n%s
\n" % "\n".join(rows) - sections.append("

%ss

\n%s" % (name, table)) - - -def get_data_dump(): - ts = ' style="border-style: solid; border-width: 1px"' - sections = [] - for cls in (Application, Content, Bundle, Task, User): - rows = [item.get_table_row() for item in cls.get_all_objects()] - rows.insert(0, cls.get_table_headers()) - _format_section(sections, cls.__name__, rows) - rows = ["API KeyUser ID"] - for key, value in api_keys.items(): - rows.append("%s%s" % (key, value)) - _format_section(sections, "API Key", rows) - text = "\n".join(sections) - text = text.replace("", "" % ts) - text = text.replace("", "" % ts) - return text - - -default_server_settings = { - "version": "", - "build": "", - "about": "", - "authentication": { - "handles_credentials": True, - "handles_login": True, - "external_user_data": False, - "external_user_search": False, - "groups_enabled": True, - "external_group_data": False, - "unique_usernames": True, - "name_editable_by": "adminandself", - "email_editable_by": "adminandself", - "username_editable_by": "adminandself", - "role_editable_by": "adminandself", - "challenge_response_enabled": False, - "name": "RStudio Connect", - "notice": "", - }, - "license": { - "ts": 1581001093714, - "status": "activated", - "expiration": 1835136000000, - "days-left": 2942, - "edition": "", - "cores": "0", - "connections": "0", - "has-key": True, - "has-trial": False, - "type": "local", - "shiny-users": "5", - "users": "15", - "user-activity-days": "365", - "allow-apis": "1", - }, - "google_analytics_tracking_id": "", - "viewer_kiosk": False, - "mail_all": False, - "mail_configured": True, - "recent_visibility": "viewer", - "public_warning": "", - "logged_in_warning": "", - "logout_url": "__logout__", - "metrics_rrd_enabled": True, - "metrics_instrumentation": True, - "customized_landing": False, - "self_registration": True, - "prohibited_usernames": [ - "connect", - "apps", - "users", - "groups", - "setpassword", - "user-completion", - "confirm", - "recent", - "reports", - "plots", - "unpublished", - "settings", - "metrics", - "tokens", - "help", - "login", - "welcome", - "register", - "resetpassword", - "content", - ], - "username_validator": "default", - "viewers_can_only_see_themselves": False, - "http_warning": False, - "queue_ui": True, - "v1_dev_api": False, - "license_expiration_ui_warning": True, - "runtimes": ["R", "Python"], - "dashboard_build_guid_based_routes": False, - "dashboard_fail_id_based_routes": False, - "expanded_view_ui": True, - "default_content_list_view": "compact", - "maximum_app_image_size": 10000000, - "server_settings_toggler": True, - "vue_logs_panel": True, - "vue_web_sudo_login": False, - "vue_content_list": False, - "git_enabled": True, - "git_available": True, - "documentation_dashboard": False, -} - -tag_map = {"apps": Application, "users": User, "bundles": Bundle, "tasks": Task, "content": Content} - -api_keys = {} - -# Handle any pre-fetching we should do. -_pre_fetch_data() diff --git a/mock_connect/mock_connect/http_helpers.py b/mock_connect/mock_connect/http_helpers.py deleted file mode 100644 index 868d3fc0..00000000 --- a/mock_connect/mock_connect/http_helpers.py +++ /dev/null @@ -1,93 +0,0 @@ -""" -This provides some low-level things to make our HTTP life easier. -""" -import re -from functools import wraps - -from typing import Dict, List - -# noinspection PyPackageRequirements -from flask import abort, after_this_request, g, jsonify, request - -from .data import DBObject, User - -digits = re.compile(r"^\d+$") - - -def error(code, reason): - """ - This sets up flask to return an error. - - :param code: the HTTP status code to return with the error. - :param reason: the text of the error message to return. - """ - - def set_code(response): - response.status_code = code - return response - - after_this_request(set_code) - - return {"error": reason, "code": code} - - -def safe_delete(key, data: dict): - try: - del data[key] - except KeyError: - pass - - -def _make_json_ready(obj): - if isinstance(obj, DBObject): - data = obj.to_dict() - for key in obj.json_excludes: - safe_delete(key, data) - obj = data - elif isinstance(obj, Dict): - for key, value in obj.items(): - obj[key] = _make_json_ready(value) - elif isinstance(obj, List): - obj = [_make_json_ready(item) for item in obj] - return obj - - -def endpoint( - authenticated: bool = False, - auth_optional: bool = False, - cls=None, - writes_json: bool = False, -): - def decorator(function): - @wraps(function) - def wrapper(object_id=None, *args, **kwargs): - if authenticated: - auth = request.headers.get("Authorization") - user = None - if auth is not None and auth.startswith("Key "): - user = User.get_user_by_api_key(auth[4:]) - - if user is None and not auth_optional: - abort(401) - - g.user = user - - if cls is None: - result = _make_json_ready(function(*args, **kwargs)) - else: - if digits.match(object_id): - object_id = int(object_id) - item = cls.get_object(object_id) - if item is None: - result = error(404, "%s with ID %s not found." % (cls.__name__, object_id)) - else: - result = _make_json_ready(function(item, *args, **kwargs)) - - if writes_json: - result = jsonify(result) - - return result - - return wrapper - - return decorator diff --git a/mock_connect/mock_connect/main.py b/mock_connect/mock_connect/main.py deleted file mode 100644 index c6778bdf..00000000 --- a/mock_connect/mock_connect/main.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -This is the main file, run via `flask run`, for the mock Connect server. -""" -import sys -from os.path import basename - -# noinspection PyPackageRequirements -from flask import Flask, Blueprint, g, request, url_for, send_file - -from .data import ( - Application, - AppMode, - Bundle, - Content, - Task, - get_data_dump, - default_server_settings, -) -from .http_helpers import endpoint, error - -app = Flask(__name__) -api = Blueprint("api", __name__) - - -@app.route("/") -def index(): - return ( - """ -Posit Connect -- Mocked -

Posit Connect -- Mocked

-

Welcome to the mocked Posit Connect! -

-%s - -""" - % get_data_dump() - ) - - -@api.route("me") -@endpoint(authenticated=True, writes_json=True) -def me(): - return g.user - - -@api.route("applications", methods=["GET", "POST"]) -@endpoint(authenticated=True, writes_json=True) -def applications(): - if request.method == "POST": - connect_app = request.get_json(force=True) - name = connect_app.get("name") - if name and Application.get_app_by_name(name) is not None: - return error(409, "An object with that name already exists.") - title = connect_app["title"] if "title" in connect_app else "" - return Application( - name=name, - title=title, - owner_username=g.user.username, - owner_first_name=g.user.first_name, - owner_last_name=g.user.last_name, - owner_email=g.user.email, - owner_locked=g.user.locked, - _base_url=url_for("index", _external=True), - ) - else: - count = int(request.args.get("count", 10000)) - search = request.args.get("search") - - def match(app_to_match): - return search is None or app_to_match.title.startswith(search) - - matches = list(filter(match, Application.get_all_objects()))[:count] - return { - "count": len(matches), - "total": len(matches), - "applications": matches, - } - - -# noinspection PyUnresolvedReferences -@api.route("applications/", methods=["GET", "POST"]) -@endpoint(authenticated=True, cls=Application, writes_json=True) -def get_application(connect_app): - if request.method == "POST": - connect_app.update_from(request.get_json(force=True)) - - return connect_app - - -# noinspection PyUnresolvedReferences -@api.route("applications//config") -@endpoint(authenticated=True, cls=Application, writes_json=True) -def config(connect_app): - return {"config_url": connect_app.url} - - -# noinspection PyUnresolvedReferences -@api.route("applications//upload", methods=["POST"]) -@endpoint(authenticated=True, cls=Application, writes_json=True) -def upload(connect_app): - return Bundle(app_id=connect_app.id, _tar_data=request.data) - - -# noinspection PyUnresolvedReferences -@api.route("applications//deploy", methods=["POST"]) -@endpoint(authenticated=True, cls=Application, writes_json=True) -def deploy(connect_app): - bundle_id = request.get_json(force=True).get("bundle") - if bundle_id is None: - return error(400, "bundle_id is required") # message and status code probably wrong - bundle = Bundle.get_object(bundle_id) - if bundle is None: - return error(404, "bundle %s not found" % bundle_id) # message and status code probably wrong - - manifest = bundle.get_manifest() - old_app_mode = connect_app.app_mode - # noinspection SpellCheckingInspection - new_app_mode = AppMode.value_of(manifest["metadata"]["appmode"]) - - if old_app_mode is not None and old_app_mode != new_app_mode: - return error(400, "Cannot change app mode once deployed") # message and status code probably wrong - - connect_app.bundle_deployed(bundle, new_app_mode) - - return Task() - - -# noinspection PyUnresolvedReferences -@api.route("tasks/") -@endpoint(authenticated=True, cls=Task, writes_json=True) -def get_task(task): - return task - - -@api.route("server_settings") -@endpoint(authenticated=True, auth_optional=True, writes_json=True) -def server_settings(): - settings = default_server_settings.copy() - - # If the endpoint was hit with a valid user, fill in some extra stuff. - if g.user is not None: - settings["version"] = "1.8.1-9999" - settings["build"] = '"9709a0fd93"' - settings["about"] = "RStudio Connect v1.8.1-9999" - - return settings - - -@api.route("v1/server_settings/python") -@endpoint(authenticated=True, writes_json=True) -def python_settings(): - v = sys.version_info - v = "%d.%d.%d" % (v[0], v[1], v[2]) - - return { - "installations": [{"version": v}], - "api_enabled": True, - } - - -# noinspection PyUnresolvedReferences -@app.route("/content/apps/") -@endpoint(cls=Application) -def get_content(connect_app): - bundle = connect_app.get_bundle() - if bundle is None: - return error(400, "The content has not been deployed.") # message and status code probably wrong - return bundle.get_rendered_content() - - -# noinspection PyUnresolvedReferences -@api.route("v1/content/") -@endpoint(authenticated=True, cls=Content, writes_json=True) -def v1_get_content(content): - return content - - -# noinspection PyUnresolvedReferences -@api.route("v1/content") -@endpoint(authenticated=True, writes_json=True) -def v1_content(): - return list(Content.get_all_objects()) - - -# This endpoint is kind of a cheat, we dont actually do any validation -# that the requested bundle belongs to this piece of content -# noinspection PyUnresolvedReferences -@api.route("v1/content//bundles//download") -@endpoint(authenticated=True, cls=Bundle) -def v1_content_bundle_download(bundle: Bundle, content_id): - print(content_id) - return send_file( - bundle.read_bundle_data(), - mimetype="application/tar+gzip", - as_attachment=True, - download_name=basename(bundle._tar_file) if bundle._tar_file else None, - ) - - -@api.route("v1/content//build", methods=["POST"]) -@endpoint(authenticated=True, writes_json=True) -def v1_content_build(): - bundle_id = request.get_json(force=True).get("bundle_id") - if bundle_id is None: - return error(400, "bundle_id is required") # message and status code probably wrong - - task = Task() - return {"task_id": task.id} - - -app.register_blueprint(api, url_prefix="/__api__") diff --git a/rsconnect/actions_content.py b/rsconnect/actions_content.py index 42f67307..da6b2bc9 100644 --- a/rsconnect/actions_content.py +++ b/rsconnect/actions_content.py @@ -93,7 +93,17 @@ def build_history(connect_server, guid): return _content_build_store.get_build_history(guid) -def build_start(connect_server, parallelism, aborted=False, error=False, all=False, poll_wait=2, debug=False): +def build_start( + connect_server, + parallelism, + aborted=False, + error=False, + running=False, + retry=False, + all=False, + poll_wait=2, + debug=False, +): init_content_build_store(connect_server) if _content_build_store.get_build_running(): raise RSConnectException("There is already a build running on this server: %s" % connect_server.url) @@ -105,6 +115,12 @@ def build_start(connect_server, parallelism, aborted=False, error=False, all=Fal all_content = list(map(lambda x: ContentGuidWithBundle(x["guid"], x["bundle_id"]), all_content)) build_add_content(connect_server, all_content) else: + # --retry is shorthand for --aborted --error --running + if retry: + aborted = True + error = True + running = True + aborted_content = [] if aborted: logger.info("Adding ABORTED content to build...") @@ -115,8 +131,14 @@ def build_start(connect_server, parallelism, aborted=False, error=False, all=Fal logger.info("Adding ERROR content to build...") error_content = _content_build_store.get_content_items(status=BuildStatus.ERROR) error_content = list(map(lambda x: ContentGuidWithBundle(x["guid"], x["bundle_id"]), error_content)) - if len(aborted_content + error_content) > 0: - build_add_content(connect_server, aborted_content + error_content) + running_content = [] + if running: + logger.info("Adding RUNNING content to build...") + running_content = _content_build_store.get_content_items(status=BuildStatus.RUNNING) + running_content = list(map(lambda x: ContentGuidWithBundle(x["guid"], x["bundle_id"]), running_content)) + + if len(aborted_content + error_content + running_content) > 0: + build_add_content(connect_server, aborted_content + error_content + running_content) content_items = _content_build_store.get_content_items(status=BuildStatus.NEEDS_BUILD) if len(content_items) == 0: @@ -172,13 +194,28 @@ def build_start(connect_server, parallelism, aborted=False, error=False, all=Fal exit(1) except KeyboardInterrupt: ContentBuildStore._BUILD_ABORTED = True + logger.info("Content build interrupted...") + logger.info( + "Content that was in the RUNNING state may still be building on the " + + "Connect server. Server builds will not be interrupted." + ) + logger.info( + "To find content items that _may_ still be running on the server, " + + "use: rsconnect content build ls --status RUNNING" + ) + logger.info( + "To retry the content build, including items that were interrupted " + + "or failed, use: rsconnect content build run --retry" + ) finally: + # make sure that we always mark the build as complete but note + # there's no guarantee that the content_executor or build_monitor + # were allowed to shut down gracefully, they may have been interrupted. + _content_build_store.set_build_running(False) if content_executor: content_executor.shutdown(wait=False) if build_monitor: build_monitor.shutdown() - # make sure that we always mark the build as complete once we finish our cleanup - _content_build_store.set_build_running(False) def _monitor_build(connect_server, content_items): diff --git a/rsconnect/main.py b/rsconnect/main.py index 83e1bdf2..ade11a1b 100644 --- a/rsconnect/main.py +++ b/rsconnect/main.py @@ -2384,8 +2384,14 @@ def get_build_logs( default=1, help="Defines the number of builds that can run concurrently. Defaults to 1.", ) -@click.option("--aborted", is_flag=True, help="Build content that is in the ABORTED state.") -@click.option("--error", is_flag=True, help="Build content that is in the ERROR state.") +@click.option("--aborted", is_flag=True, hidden=True, help="Build content that is in the ABORTED state.") +@click.option("--error", is_flag=True, hidden=True, help="Build content that is in the ERROR state.") +@click.option("--running", is_flag=True, hidden=True, help="Build content that is in the RUNNING state.") +@click.option( + "--retry", + is_flag=True, + help="Build all content that is in the NEEDS_BUILD, ABORTED, ERROR, or RUNNING state.", +) @click.option("--all", is_flag=True, help="Build all content, even if it is already marked as COMPLETE.") @click.option( "--poll-wait", @@ -2416,6 +2422,8 @@ def start_content_build( parallelism: int, aborted: bool, error: bool, + running: bool, + retry: bool, all: bool, poll_wait: float, format: str, @@ -2427,7 +2435,7 @@ def start_content_build( logger.set_log_output_format(format) with cli_feedback("", stderr=True): ce = RSConnectExecutor(ctx, name, server, api_key, insecure, cacert, logger=None).validate_server() - build_start(ce.remote_server, parallelism, aborted, error, all, poll_wait, debug) + build_start(ce.remote_server, parallelism, aborted, error, running, retry, all, poll_wait, debug) @cli.group(no_args_is_help=True, help="Interact with Posit Connect's system API.") diff --git a/tests/rsconnect-build-test/connect_remote_6443.json b/tests/rsconnect-build-test/connect_remote_6443.json deleted file mode 100644 index 32f7e05c..00000000 --- a/tests/rsconnect-build-test/connect_remote_6443.json +++ /dev/null @@ -1,83 +0,0 @@ -{ - "rsconnect_build_running": false, - "rsconnect_content": { - "c96db3f3-87a1-4df5-9f58-eb109c397718": { - "guid": "c96db3f3-87a1-4df5-9f58-eb109c397718", - "bundle_id": "177", - "title": "orphan-proc-shiny-test", - "name": "orphan-proc-shiny-test", - "app_mode": "shiny", - "content_url": "https://connect.remote:6443/content/c96db3f3-87a1-4df5-9f58-eb109c397718/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/c96db3f3-87a1-4df5-9f58-eb109c397718", - "created_time": "2021-11-04T18:07:12Z", - "last_deployed_time": "2021-11-10T19:10:56Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "NEEDS_BUILD" - }, - "fe673896-f92a-40cc-be4c-e4872bb90a37": { - "guid": "fe673896-f92a-40cc-be4c-e4872bb90a37", - "bundle_id": "185", - "title": "interactive-rmd", - "name": "interactive-rmd", - "app_mode": "rmd-shiny", - "content_url": "https://connect.remote:6443/content/fe673896-f92a-40cc-be4c-e4872bb90a37/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/fe673896-f92a-40cc-be4c-e4872bb90a37", - "created_time": "2021-11-15T15:37:53Z", - "last_deployed_time": "2021-11-15T15:37:57Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "ERROR" - }, - "a0b6b5a2-5fbe-4293-8310-4f80054bc24f": { - "guid": "a0b6b5a2-5fbe-4293-8310-4f80054bc24f", - "bundle_id": "184", - "title": "stock-report-jupyter", - "name": "stock-report-jupyter", - "app_mode": "jupyter-static", - "content_url": "https://connect.remote:6443/content/a0b6b5a2-5fbe-4293-8310-4f80054bc24f/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/a0b6b5a2-5fbe-4293-8310-4f80054bc24f", - "created_time": "2021-11-15T15:27:18Z", - "last_deployed_time": "2021-11-15T15:35:27Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "RUNNING" - }, - "23315cc9-ed2a-40ad-9e99-e5e49066531a": { - "guid": "23315cc9-ed2a-40ad-9e99-e5e49066531a", - "bundle_id": "180", - "title": "static-rmd", - "name": "static-rmd2", - "app_mode": "rmd-static", - "content_url": "https://connect.remote:6443/content/23315cc9-ed2a-40ad-9e99-e5e49066531a/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/23315cc9-ed2a-40ad-9e99-e5e49066531a", - "created_time": "2021-11-15T15:20:58Z", - "last_deployed_time": "2021-11-15T15:25:31Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "COMPLETE", - "rsconnect_last_build_time": "2021-12-13T18:10:38Z", - "rsconnect_last_build_log": "/logs/localhost_3939/23315cc9-ed2a-40ad-9e99-e5e49066531a/ZUf44zVWHjODv1Rq.log", - "rsconnect_build_task_result": { - "id": "ZUf44zVWHjODv1Rq", - "user_id": 1, - "result": { - "type": "", - "data": null - }, - "finished": true, - "code": 0, - "error": "" - } - }, - "015143da-b75f-407c-81b1-99c4a724341e": { - "guid": "015143da-b75f-407c-81b1-99c4a724341e", - "bundle_id": "176", - "title": "plumber-async", - "name": "plumber-async", - "app_mode": "api", - "content_url": "https://connect.remote:6443/content/015143da-b75f-407c-81b1-99c4a724341e/", - "dashboard_url": "https://connect.remote:6443/connect/#/apps/015143da-b75f-407c-81b1-99c4a724341e", - "created_time": "2021-11-01T20:43:32Z", - "last_deployed_time": "2021-11-03T17:48:59Z", - "owner_guid": "edf26318-0027-4d9d-bbbb-54703ebb1855", - "rsconnect_build_status": "ERROR" - } - } -} \ No newline at end of file diff --git a/tests/test_main_content.py b/tests/test_main_content.py index 7306b7e9..8f380c5f 100644 --- a/tests/test_main_content.py +++ b/tests/test_main_content.py @@ -3,35 +3,100 @@ import shutil import tarfile import unittest + +import httpretty from click.testing import CliRunner from rsconnect.main import cli from rsconnect import VERSION +from rsconnect.api import RSConnectServer from rsconnect.models import BuildStatus -from rsconnect.metadata import _normalize_server_url +from rsconnect.metadata import ContentBuildStore, _normalize_server_url -from .utils import apply_common_args, require_api_key, require_connect +from .utils import apply_common_args -# run these tests in the order they are defined -# because we are integration testing the state file -unittest.TestLoader.sortTestMethodsUsing = None +# These tests need to run in order because they share the same tempdir +# For some reason setup and teardown aren't enough to fully reset the state +# between tests. Overriding the env var CONNECT_CONTENT_BUILD_DIR to be a tempdir +# would be preferable but this is fine for now. +TEMP_DIR="rsconnect-build-test" -_bundle_download_dest = "download.tar.gz" -_content_guids = [ - "015143da-b75f-407c-81b1-99c4a724341e", - "4ffc819c-065c-420c-88eb-332db1133317", - "bcc74209-3a81-4b9c-acd5-d24a597c256c", -] -_test_build_dir = "rsconnect-build-test" +def register_uris(connect_server: str): + def register_content_endpoints(i: int, guid: str): + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/v1/content/{guid}", + body=open(f"tests/testdata/connect-responses/describe-content-{i}.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + ) + httpretty.register_uri( + httpretty.POST, + f"{connect_server}/__api__/v1/content/{guid}/build", + body='{"task_id": "1234"}', + adding_headers={"Content-Type": "application/json"}, + ) + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/applications/{guid}/config", + body='{' + + f'"config_url": "{connect_server}/connect/#/apps/{guid}",' + + f'"logs_url": "{connect_server}/connect/#/apps/{guid}"' + + '}', + adding_headers={"Content-Type": "application/json"}, + ) + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/server_settings", + body=open("tests/testdata/connect-responses/server_settings.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + ) + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/me", + body=open("tests/testdata/connect-responses/me.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + ) + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/v1/content", + body=open("tests/testdata/connect-responses/list-content.json", "r").read(), + adding_headers={"Content-Type": "application/json"}, + ) + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/v1/content/7d59c5c7-c4a7-4950-acc3-3943b7192bc4/bundles/92/download", + body=open("tests/testdata/bundle.tar.gz", "rb").read(), + adding_headers={"Content-Type": "application/tar+gzip"}, + ) + httpretty.register_uri( + httpretty.GET, + f"{connect_server}/__api__/tasks/1234", + body="""{ + "id": "1234", + "user_id": 0, + "status": ["status1", "status2", "status3"], + "result": {"type": "", "data": ""}, + "finished": true, + "code": 0, + "error": "", + "last_status": 0 + }""", + adding_headers={"Content-Type": "application/json"}, + ) + register_content_endpoints(1, "7d59c5c7-c4a7-4950-acc3-3943b7192bc4") + register_content_endpoints(2, "ab497e4b-b706-4ae7-be49-228979a95eb4") + register_content_endpoints(3, "cdfed1f7-0e09-40eb-996d-0ef77ea2d797") class TestContentSubcommand(unittest.TestCase): @classmethod def tearDownClass(cls): - if os.path.exists(_bundle_download_dest): - os.remove(_bundle_download_dest) - if os.path.exists(_test_build_dir): - shutil.rmtree(_test_build_dir, ignore_errors=True) + if os.path.exists(TEMP_DIR): + shutil.rmtree(TEMP_DIR, ignore_errors=True) + + def setUp(self): + self.connect_server = "http://localhost:3939" + self.api_key = "testapikey123" def test_version(self): runner = CliRunner() @@ -39,97 +104,153 @@ def test_version(self): self.assertEqual(result.exit_code, 0, result.output) self.assertIn(VERSION, result.output) + @httpretty.activate(verbose=True, allow_net_connect=False) def test_content_search(self): - connect_server = require_connect() - api_key = require_api_key() + register_uris(self.connect_server) runner = CliRunner() args = ["content", "search"] - apply_common_args(args, server=connect_server, key=api_key) + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) response = json.loads(result.output) self.assertIsNotNone(response, result.output) - self.assertEqual(len(response), 3, result.output) + self.assertEqual(len(response), 4, result.output) + @httpretty.activate(verbose=True, allow_net_connect=False) def test_content_describe(self): - connect_server = require_connect() - api_key = require_api_key() + register_uris(self.connect_server) runner = CliRunner() - args = ["content", "describe", "-g", _content_guids[0], "-g", _content_guids[1]] - apply_common_args(args, server=connect_server, key=api_key) + args = ["content", "describe", + "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "-g", "ab497e4b-b706-4ae7-be49-228979a95eb4"] + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) response = json.loads(result.output) self.assertIn("id", response[0]) self.assertIn("id", response[1]) - self.assertEqual(response[0]["guid"], _content_guids[0]) - self.assertEqual(response[1]["guid"], _content_guids[1]) + self.assertEqual(response[0]["guid"], "7d59c5c7-c4a7-4950-acc3-3943b7192bc4") + self.assertEqual(response[1]["guid"], "ab497e4b-b706-4ae7-be49-228979a95eb4") + @httpretty.activate(verbose=True, allow_net_connect=False) def test_content_download_bundle(self): - connect_server = require_connect() - api_key = require_api_key() + register_uris(self.connect_server) runner = CliRunner() - args = ["content", "download-bundle", "-g", _content_guids[1], "-o", _bundle_download_dest] - apply_common_args(args, server=connect_server, key=api_key) + args = ["content", "download-bundle", + "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "-o", f"{TEMP_DIR}/bundle.tar.gz"] + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) - with tarfile.open(_bundle_download_dest, mode="r:gz") as tgz: - self.assertIsNotNone(tgz.extractfile("manifest.json").read()) + with tarfile.open(f"{TEMP_DIR}/bundle.tar.gz", mode="r:gz") as tgz: + manifest = json.loads(tgz.extractfile("manifest.json").read()) + self.assertIn("metadata", manifest) + @httpretty.activate(verbose=True, allow_net_connect=False) def test_build(self): - connect_server = require_connect() - api_key = require_api_key() + register_uris(self.connect_server) runner = CliRunner() # add a content item - args = ["content", "build", "add", "-g", _content_guids[0]] - apply_common_args(args, server=connect_server, key=api_key) + args = ["content", "build", "add", "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4"] + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) self.assertTrue( - os.path.exists("%s/%s.json" % (_test_build_dir, _normalize_server_url(os.environ.get("CONNECT_SERVER")))) + os.path.exists("%s/%s.json" % (TEMP_DIR, _normalize_server_url(self.connect_server))) ) # list the "tracked" content - args = ["content", "build", "ls", "-g", _content_guids[0]] - apply_common_args(args, server=connect_server, key=api_key) + args = ["content", "build", "ls", "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4"] + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) listing = json.loads(result.output) self.assertTrue(len(listing) == 1) - self.assertEqual(listing[0]["guid"], _content_guids[0]) - self.assertEqual(listing[0]["bundle_id"], "176") + self.assertEqual(listing[0]["guid"], "7d59c5c7-c4a7-4950-acc3-3943b7192bc4") + self.assertEqual(listing[0]["bundle_id"], "92") self.assertEqual(listing[0]["rsconnect_build_status"], BuildStatus.NEEDS_BUILD) # run the build args = ["content", "build", "run", "--debug"] - apply_common_args(args, server=connect_server, key=api_key) + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) # check that the build succeeded - args = ["content", "build", "ls", "-g", _content_guids[0]] - apply_common_args(args, server=connect_server, key=api_key) + args = ["content", "build", "ls", "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4"] + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) listing = json.loads(result.output) self.assertTrue(len(listing) == 1) self.assertEqual(listing[0]["rsconnect_build_status"], BuildStatus.COMPLETE) + @httpretty.activate(verbose=True, allow_net_connect=False) + def test_build_retry(self): + register_uris(self.connect_server) + runner = CliRunner() + + # add 3 content items + args = ["content", "build", "add", + "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "-g", "ab497e4b-b706-4ae7-be49-228979a95eb4", + "-g", "cdfed1f7-0e09-40eb-996d-0ef77ea2d797"] + apply_common_args(args, server=self.connect_server, key=self.api_key) + result = runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, result.output) + self.assertTrue( + os.path.exists("%s/%s.json" % (TEMP_DIR, _normalize_server_url(self.connect_server))) + ) + + # change the content build status so it looks like it was interrupted/failed + store = ContentBuildStore(RSConnectServer(self.connect_server, self.api_key)) + store.set_content_item_build_status("7d59c5c7-c4a7-4950-acc3-3943b7192bc4", BuildStatus.RUNNING) + store.set_content_item_build_status("ab497e4b-b706-4ae7-be49-228979a95eb4", BuildStatus.ABORTED) + store.set_content_item_build_status("cdfed1f7-0e09-40eb-996d-0ef77ea2d797", BuildStatus.ERROR) + + # run the build + args = ["content", "build", "run", "--retry"] + apply_common_args(args, server=self.connect_server, key=self.api_key) + result = runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, result.output) + + # check that the build succeeded + args = ["content", "build", "ls", + "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "-g", "ab497e4b-b706-4ae7-be49-228979a95eb4", + "-g", "cdfed1f7-0e09-40eb-996d-0ef77ea2d797"] + apply_common_args(args, server=self.connect_server, key=self.api_key) + result = runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, result.output) + listing = json.loads(result.output) + self.assertTrue(len(listing) == 3) + self.assertEqual(listing[0]["rsconnect_build_status"], BuildStatus.COMPLETE) + self.assertEqual(listing[1]["rsconnect_build_status"], BuildStatus.COMPLETE) + self.assertEqual(listing[2]["rsconnect_build_status"], BuildStatus.COMPLETE) + + @httpretty.activate(verbose=True, allow_net_connect=False) def test_build_rm(self): - connect_server = require_connect() - api_key = require_api_key() + register_uris(self.connect_server) runner = CliRunner() - # remove a content item - args = ["content", "build", "rm", "-g", _content_guids[0]] - apply_common_args(args, server=connect_server, key=api_key) + # remove all the content items + args = ["content", "build", "rm", "-g", "7d59c5c7-c4a7-4950-acc3-3943b7192bc4"] + apply_common_args(args, server=self.connect_server, key=self.api_key) + result = runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, result.output) + args = ["content", "build", "rm", "-g", "ab497e4b-b706-4ae7-be49-228979a95eb4"] + apply_common_args(args, server=self.connect_server, key=self.api_key) + result = runner.invoke(cli, args) + self.assertEqual(result.exit_code, 0, result.output) + args = ["content", "build", "rm", "-g", "cdfed1f7-0e09-40eb-996d-0ef77ea2d797"] + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) - # check that it was removed + # check that they were removed args = ["content", "build", "ls"] - apply_common_args(args, server=connect_server, key=api_key) + apply_common_args(args, server=self.connect_server, key=self.api_key) result = runner.invoke(cli, args) self.assertEqual(result.exit_code, 0, result.output) listing = json.loads(result.output) diff --git a/tests/testdata/connect-responses/describe-content-1.json b/tests/testdata/connect-responses/describe-content-1.json new file mode 100644 index 00000000..b9709600 --- /dev/null +++ b/tests/testdata/connect-responses/describe-content-1.json @@ -0,0 +1,45 @@ +{ + "guid": "7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "name": "geyser-app-git-1697741538712", + "title": "geyser-app-git", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": null, + "created_time": "2023-10-19T18:52:18Z", + "last_deployed_time": "2023-10-24T22:46:37Z", + "bundle_id": "92", + "app_mode": "shiny", + "content_category": "", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": null, + "quarto_version": null, + "r_environment_management": true, + "default_r_environment_management": null, + "py_environment_management": null, + "default_py_environment_management": null, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/7d59c5c7-c4a7-4950-acc3-3943b7192bc4/", + "dashboard_url": "http://localhost:3939/connect/#/apps/7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "app_role": "owner", + "id": "25" +} diff --git a/tests/testdata/connect-responses/describe-content-2.json b/tests/testdata/connect-responses/describe-content-2.json new file mode 100644 index 00000000..a34ad30a --- /dev/null +++ b/tests/testdata/connect-responses/describe-content-2.json @@ -0,0 +1,45 @@ +{ + "guid": "ab497e4b-b706-4ae7-be49-228979a95eb4", + "name": "shinyrmd-collab-test", + "title": "shinyrmd-collab-test", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": "rstudio/connect:ubuntu22", + "created_time": "2023-09-27T15:17:19Z", + "last_deployed_time": "2023-09-27T15:19:25Z", + "bundle_id": "22", + "app_mode": "shiny", + "content_category": "", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": null, + "quarto_version": null, + "r_environment_management": false, + "default_r_environment_management": false, + "py_environment_management": null, + "default_py_environment_management": false, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/ab497e4b-b706-4ae7-be49-228979a95eb4/", + "dashboard_url": "http://localhost:3939/connect/#/apps/ab497e4b-b706-4ae7-be49-228979a95eb4", + "app_role": "owner", + "id": "15" +} diff --git a/tests/testdata/connect-responses/describe-content-3.json b/tests/testdata/connect-responses/describe-content-3.json new file mode 100644 index 00000000..fc16d37f --- /dev/null +++ b/tests/testdata/connect-responses/describe-content-3.json @@ -0,0 +1,45 @@ +{ + "guid": "cdfed1f7-0e09-40eb-996d-0ef77ea2d797", + "name": "plumbr-testing", + "title": "plumbr-testing", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": null, + "created_time": "2023-09-11T18:32:48Z", + "last_deployed_time": "2023-09-11T18:32:50Z", + "bundle_id": "17", + "app_mode": "api", + "content_category": "api", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": null, + "quarto_version": null, + "r_environment_management": true, + "default_r_environment_management": null, + "py_environment_management": null, + "default_py_environment_management": null, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/cdfed1f7-0e09-40eb-996d-0ef77ea2d797/", + "dashboard_url": "http://localhost:3939/connect/#/apps/cdfed1f7-0e09-40eb-996d-0ef77ea2d797", + "app_role": "owner", + "id": "12" +} diff --git a/tests/testdata/connect-responses/list-content.json b/tests/testdata/connect-responses/list-content.json new file mode 100644 index 00000000..3acdddb6 --- /dev/null +++ b/tests/testdata/connect-responses/list-content.json @@ -0,0 +1,182 @@ +[ + { + "guid": "7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "name": "geyser-app-git-1697741538712", + "title": "geyser-app-git", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": null, + "created_time": "2023-10-19T18:52:18Z", + "last_deployed_time": "2023-10-24T22:46:37Z", + "bundle_id": "92", + "app_mode": "shiny", + "content_category": "", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": null, + "quarto_version": null, + "r_environment_management": true, + "default_r_environment_management": null, + "py_environment_management": null, + "default_py_environment_management": null, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/7d59c5c7-c4a7-4950-acc3-3943b7192bc4/", + "dashboard_url": "http://localhost:3939/connect/#/apps/7d59c5c7-c4a7-4950-acc3-3943b7192bc4", + "app_role": "owner", + "id": "25" + }, + { + "guid": "ab497e4b-b706-4ae7-be49-228979a95eb4", + "name": "shinyrmd-collab-test", + "title": "shinyrmd-collab-test", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": "rstudio/connect:ubuntu22", + "created_time": "2023-09-27T15:17:19Z", + "last_deployed_time": "2023-09-27T15:19:25Z", + "bundle_id": "22", + "app_mode": "shiny", + "content_category": "", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": null, + "quarto_version": null, + "r_environment_management": false, + "default_r_environment_management": false, + "py_environment_management": null, + "default_py_environment_management": false, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/ab497e4b-b706-4ae7-be49-228979a95eb4/", + "dashboard_url": "http://localhost:3939/connect/#/apps/ab497e4b-b706-4ae7-be49-228979a95eb4", + "app_role": "owner", + "id": "15" + }, + { + "guid": "cdfed1f7-0e09-40eb-996d-0ef77ea2d797", + "name": "plumbr-testing", + "title": "plumbr-testing", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": null, + "created_time": "2023-09-11T18:32:48Z", + "last_deployed_time": "2023-09-11T18:32:50Z", + "bundle_id": "17", + "app_mode": "api", + "content_category": "api", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": null, + "quarto_version": null, + "r_environment_management": true, + "default_r_environment_management": null, + "py_environment_management": null, + "default_py_environment_management": null, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/cdfed1f7-0e09-40eb-996d-0ef77ea2d797/", + "dashboard_url": "http://localhost:3939/connect/#/apps/cdfed1f7-0e09-40eb-996d-0ef77ea2d797", + "app_role": "owner", + "id": "12" + }, + { + "guid": "11f88569-6d83-435d-b211-49e5ebd67ec2", + "name": "reticulate", + "title": "reticulate", + "description": "", + "access_type": "acl", + "connection_timeout": null, + "read_timeout": null, + "init_timeout": null, + "idle_timeout": null, + "max_processes": null, + "min_processes": null, + "max_conns_per_process": null, + "load_factor": null, + "memory_request": null, + "memory_limit": null, + "cpu_request": null, + "cpu_limit": null, + "amd_gpu_limit": null, + "nvidia_gpu_limit": null, + "service_account_name": null, + "default_image_name": null, + "created_time": "2023-08-16T20:23:24Z", + "last_deployed_time": "2023-08-16T21:15:21Z", + "bundle_id": "16", + "app_mode": "rmd-static", + "content_category": "", + "parameterized": false, + "cluster_name": "Local", + "image_name": null, + "r_version": "4.3.0", + "py_version": "3.11.3", + "quarto_version": null, + "r_environment_management": false, + "default_r_environment_management": true, + "py_environment_management": false, + "default_py_environment_management": true, + "run_as": null, + "run_as_current_user": false, + "owner_guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "content_url": "http://localhost:3939/content/11f88569-6d83-435d-b211-49e5ebd67ec2/", + "dashboard_url": "http://localhost:3939/connect/#/apps/11f88569-6d83-435d-b211-49e5ebd67ec2", + "app_role": "owner", + "id": "11" + } +] diff --git a/tests/testdata/connect-responses/me.json b/tests/testdata/connect-responses/me.json new file mode 100644 index 00000000..7abc2e69 --- /dev/null +++ b/tests/testdata/connect-responses/me.json @@ -0,0 +1,42 @@ +{ + "email": "rsc@example.com", + "username": "admin", + "first_name": "admin1", + "last_name": "", + "password": "", + "user_role": "administrator", + "created_time": "2023-07-24T18:49:28.584906476Z", + "updated_time": "2023-10-17T19:24:03.386371574Z", + "active_time": "2023-12-01T20:27:06.22605874Z", + "confirmed": true, + "locked": false, + "guid": "820f092f-d564-4ab5-819a-0f4d2f03d11e", + "preferences": { + "hide_jump_start": true, + "jump_start_language": "python", + "hide_auth_origin_help": false + }, + "privileges": [ + "add_users", + "add_vanities", + "build_bundles", + "change_app_permissions", + "change_apps", + "change_groups", + "change_usernames", + "change_users", + "change_variant_schedule", + "create_groups", + "edit_run_as", + "edit_runtime", + "lock_users", + "manage_repository", + "publish_apps", + "remove_apps", + "remove_groups", + "remove_users", + "remove_vanities", + "view_app_settings", + "view_apps" + ] +} diff --git a/tests/testdata/connect-responses/server_settings.json b/tests/testdata/connect-responses/server_settings.json new file mode 100644 index 00000000..ca3c3003 --- /dev/null +++ b/tests/testdata/connect-responses/server_settings.json @@ -0,0 +1,115 @@ +{ + "hostname": "connect", + "version": "2023.11.0-dev+103-g11242ee129", + "build": "v2023.10.0-103-g11242ee129", + "about": "Posit Connect v2023.11.0-dev+103-g11242ee129", + "authentication": { + "handles_credentials": true, + "handles_login": true, + "challenge_response_enabled": false, + "external_user_data": false, + "external_user_search": false, + "external_user_id": false, + "groups_enabled": true, + "external_group_search": false, + "external_group_members": false, + "external_group_id": false, + "external_group_owner": false, + "unique_usernames": true, + "name_editable_by": "adminandself", + "email_editable_by": "adminandself", + "username_editable_by": "adminandself", + "role_editable_by": "adminandself", + "name": "Posit Connect", + "notice": "", + "warning_delay": 10 + }, + "license": { + "ts": 1701462400891, + "status": "activated", + "expiration": 1711929600000, + "days-left": 122, + "has-key": true, + "has-trial": false, + "tier": "enterprise", + "sku-year": "2023", + "edition": "", + "cores": "0", + "connections": "0", + "type": "local", + "users": "20", + "user-activity-days": "365", + "shiny-users": "20", + "allow-apis": "1", + "custom-branding": "1", + "current-user-execution": "1", + "enable-launcher": "1" + }, + "license_expiration_ui_warning": true, + "deprecated_settings": false, + "deprecated_settings_ui_warning": true, + "google_analytics_tracking_id": "", + "viewer_kiosk": false, + "mail_all": false, + "mail_configured": true, + "public_warning": "", + "logged_in_warning": "", + "logout_url": "__logout__", + "metrics_rrd_enabled": true, + "metrics_instrumentation": true, + "customized_landing": false, + "self_registration": true, + "prohibited_usernames": [ + "connect", + "apps", + "users", + "groups", + "setpassword", + "user-completion", + "confirm", + "recent", + "reports", + "plots", + "unpublished", + "settings", + "metrics", + "tokens", + "help", + "login", + "welcome", + "register", + "resetpassword", + "content" + ], + "username_validator": "default", + "viewers_can_only_see_themselves": false, + "http_warning": false, + "queue_ui": true, + "runtimes": [ + "R", + "Python", + "Quarto" + ], + "default_content_list_view": "compact", + "maximum_app_image_size": 10000000, + "server_settings_toggler": true, + "git_enabled": true, + "git_available": true, + "dashboard_path": "/connect", + "system_display_name": "Posit Connect", + "hide_viewer_documentation": false, + "jump_start_enabled": true, + "permission_request": true, + "tableau_integration_enabled": true, + "self_test_enabled": true, + "execution_type": "native", + "enable_runtime_constraints": true, + "enable_image_management": false, + "enable_runtime_cache_management": true, + "default_image_selection_enabled": true, + "default_environment_management_selection": true, + "default_r_environment_management": true, + "default_py_environment_management": true, + "new_parameterization_enabled": false, + "use_window_location": false +}