From 6c3e3081b7421e50d64bdd47c3f249aae6094eb4 Mon Sep 17 00:00:00 2001 From: Karan Gathani Date: Thu, 14 Nov 2024 09:19:06 -0800 Subject: [PATCH 1/4] Add nightly tests --- .github/workflows/nightly-tests.yml | 38 +++++++++++++++++++++++++++++ tests/run_smoke_test.py | 30 +++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 .github/workflows/nightly-tests.yml create mode 100644 tests/run_smoke_test.py diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml new file mode 100644 index 0000000..78401a5 --- /dev/null +++ b/.github/workflows/nightly-tests.yml @@ -0,0 +1,38 @@ +name: Nightly Tests for Shiny Assistant + +on: + schedule: + - cron: "0 3 * * *" + push: + branches: + - "testing**" + workflow_dispatch: + +jobs: + process-issues: + runs-on: ubuntu-latest + permissions: write-all + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: "18" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + playwright install chromium --with-deps + + - name: Process issue description + id: process-description + run: python tests/run_smoke_test.py diff --git a/tests/run_smoke_test.py b/tests/run_smoke_test.py new file mode 100644 index 0000000..fb52b35 --- /dev/null +++ b/tests/run_smoke_test.py @@ -0,0 +1,30 @@ +import time + +from playwright.sync_api import Playwright, sync_playwright + + +def run(playwright: Playwright): + chromium = playwright.chromium + browser = chromium.launch() + page = browser.new_page() + page.goto("https://gallery.shinyapps.io/assistant") + page.get_by_label("Python").check() + page.get_by_text("Who can see my activity?").click() + page.get_by_text("Who can see my activity?").click() + page.get_by_role("textbox", name="Enter a message...").click() + page.get_by_role("textbox", name="Enter a message...").fill( + """ +Help me write a shiny app with orange cards +""" + ) + page.get_by_label("Send message").click() + time.sleep(12) # required since we are streaming responses + message_contents = page.query_selector_all(".message-content") + last_message_content = message_contents[-1].text_content() + if "orange" not in last_message_content: + raise AssertionError("Expected orange in last message content") + browser.close() + + +with sync_playwright() as playwright: + run(playwright) From a75aa8b3b134db42cb1b692335785c0bf101cd15 Mon Sep 17 00:00:00 2001 From: Karan Gathani Date: Thu, 14 Nov 2024 09:27:09 -0800 Subject: [PATCH 2/4] install from tests/requirements.txt --- .github/workflows/nightly-tests.yml | 2 +- tests/requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 tests/requirements.txt diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 78401a5..74ce7d3 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -30,7 +30,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements.txt + pip install -r tests/requirements.txt playwright install chromium --with-deps - name: Process issue description diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 0000000..508a5f4 --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1 @@ +playwright From 99c2a5df4ebe9de2ebaaee31fb612e341e194e0e Mon Sep 17 00:00:00 2001 From: Karan Gathani Date: Thu, 14 Nov 2024 09:37:38 -0800 Subject: [PATCH 3/4] update name of step --- .github/workflows/nightly-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 74ce7d3..7eafb64 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -33,6 +33,6 @@ jobs: pip install -r tests/requirements.txt playwright install chromium --with-deps - - name: Process issue description - id: process-description + - name: Run playwright test + id: playwright-test run: python tests/run_smoke_test.py From 5764a3c14226da565e70124f79c6e7ba6c3c05be Mon Sep 17 00:00:00 2001 From: Karan Gathani Date: Thu, 14 Nov 2024 12:35:47 -0800 Subject: [PATCH 4/4] Add tests to run for every PR --- .github/workflows/ci-tests.yml | 42 +++++++++++++++++++++++++++++ .github/workflows/nightly-tests.yml | 2 +- test_local_app.py | 12 +++++++++ tests/requirements.txt | 2 ++ tests/run_smoke_test.py | 22 +++++---------- tests/utils.py | 18 +++++++++++++ 6 files changed, 81 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/ci-tests.yml create mode 100644 test_local_app.py create mode 100644 tests/utils.py diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml new file mode 100644 index 0000000..cf3af7c --- /dev/null +++ b/.github/workflows/ci-tests.yml @@ -0,0 +1,42 @@ +name: CI Tests for Shiny Assistant + + +on: + pull_request: + branches: + - main + push: + branches: + - "testing**" + + +jobs: + smoke-test: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: "18" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r tests/requirements.txt + playwright install chromium --with-deps + + - name: Run playwright test + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + id: playwright-test + run: pytest test_local_app.py diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml index 7eafb64..e8e6af2 100644 --- a/.github/workflows/nightly-tests.yml +++ b/.github/workflows/nightly-tests.yml @@ -9,7 +9,7 @@ on: workflow_dispatch: jobs: - process-issues: + nightly-tests: runs-on: ubuntu-latest permissions: write-all diff --git a/test_local_app.py b/test_local_app.py new file mode 100644 index 0000000..fbb811c --- /dev/null +++ b/test_local_app.py @@ -0,0 +1,12 @@ +from playwright.sync_api import Page +from shiny.run import ShinyAppProc + +from tests.utils import send_message_and_verify + + +def test_shiny_assistant(page: Page, local_app: ShinyAppProc) -> None: + page.goto(local_app.url) + + send_message_and_verify( + page, "Help me write a shiny app with orange cards", "orange" + ) diff --git a/tests/requirements.txt b/tests/requirements.txt index 508a5f4..3e7b297 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1 +1,3 @@ playwright +pytest-playwright + diff --git a/tests/run_smoke_test.py b/tests/run_smoke_test.py index fb52b35..c50641b 100644 --- a/tests/run_smoke_test.py +++ b/tests/run_smoke_test.py @@ -1,6 +1,5 @@ -import time - from playwright.sync_api import Playwright, sync_playwright +from utils import send_message_and_verify def run(playwright: Playwright): @@ -8,21 +7,12 @@ def run(playwright: Playwright): browser = chromium.launch() page = browser.new_page() page.goto("https://gallery.shinyapps.io/assistant") - page.get_by_label("Python").check() - page.get_by_text("Who can see my activity?").click() - page.get_by_text("Who can see my activity?").click() - page.get_by_role("textbox", name="Enter a message...").click() - page.get_by_role("textbox", name="Enter a message...").fill( - """ -Help me write a shiny app with orange cards -""" + + send_message_and_verify( + page, + "Help me write a shiny app with orange cards", + "orange" ) - page.get_by_label("Send message").click() - time.sleep(12) # required since we are streaming responses - message_contents = page.query_selector_all(".message-content") - last_message_content = message_contents[-1].text_content() - if "orange" not in last_message_content: - raise AssertionError("Expected orange in last message content") browser.close() diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..c11ebc0 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,18 @@ +import time +from playwright.sync_api import Page + + +def send_message_and_verify(page: Page, message: str, expected_text: str) -> None: + page.get_by_label("Python").check() + page.get_by_text("Who can see my activity?").click() + page.get_by_text("Who can see my activity?").click() + page.get_by_role("textbox", name="Enter a message...").click() + page.get_by_role("textbox", name="Enter a message...").fill(message) + page.get_by_label("Send message").click() + time.sleep(12) # required since we are streaming responses + message_contents = page.query_selector_all(".message-content") + last_message_content = message_contents[-1].text_content() + if expected_text not in last_message_content: + raise AssertionError( + f"Expected '{expected_text}' in last message content but got: {last_message_content}" + )