From 20911e61f5bcd708b3d31f8bf5c13277e30d7e31 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 12:08:35 -0500 Subject: [PATCH 1/7] Always turn on video and tracing when using `make playwright-debug` --- tests/playwright/playwright-pytest.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/playwright/playwright-pytest.ini b/tests/playwright/playwright-pytest.ini index 8eb2dbd0d..7374a2a36 100644 --- a/tests/playwright/playwright-pytest.ini +++ b/tests/playwright/playwright-pytest.ini @@ -7,8 +7,10 @@ asyncio_mode=strict # --numprocesses auto: number of testing workers. auto is number of (virtual) cores # --video=retain-on-failure: playwright saves recording of any failed test # --tracing=retain-on-failure: playwright saves trace of any failed test +# --video=on: Always save playwright recording for every test +# --tracing=on: Always save playwright trace for every test # -vv: Extra extra verbose output # # --headed: Headed browser testing # # -r P: Show extra test summary info: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. (w)arnings... # --maxfail=1: Stop after 1 failure has occurred -addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=retain-on-failure --video=retain-on-failure +addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=on --video=on From 44b915167a3d5a3e3b423c1013dbbd78bc2688d7 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 12:32:19 -0500 Subject: [PATCH 2/7] Test deploys apps, but only attempt deploy job given valid conditions --- .github/workflows/pytest.yaml | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 5770ec3bc..ef3215dca 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -118,9 +118,35 @@ jobs: path: test-results/ retention-days: 5 - playwright-deploys: + playwright-deploys-local: if: github.event_name != 'release' + runs-on: ${{ matrix.os }} + strategy: + matrix: + # Matches deploy server python version + python-version: ["3.10"] + os: [ubuntu-latest] + fail-fast: false + + steps: + - uses: actions/checkout@v3 + - name: Setup py-shiny + uses: ./.github/py-shiny/setup + with: + python-version: ${{ matrix.python-version }} + + - name: Test deploys example apps locally + timeout-minutes: 5 + env: + DEPLOY_APPS: "false" + run: | + make playwright-deploys SUB_FILE=". -vv" + + playwright-deploys: + needs: [playwright-deploys-local] + if: github.event_name != 'release' && (github.event_name == 'push' || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'deploy'))) # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR) + # Only one is allowed to run at a time because it is deploying to the same server location. concurrency: playwright-deploys runs-on: ${{ matrix.os }} strategy: @@ -137,8 +163,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + # Make sure the apps pass locally - name: Test deploys example apps locally - timeout-minutes: 5 + timeout-minutes: 5 # ~10s locally env: DEPLOY_APPS: "false" run: | From 38bbe4317c17117bdc1ac2c45bb24f2c97fe194b Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 12:32:41 -0500 Subject: [PATCH 3/7] Mark test as flaky --- .../shiny/bugs/0696-resolve-id/test_0696_resolve_id.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/playwright/shiny/bugs/0696-resolve-id/test_0696_resolve_id.py b/tests/playwright/shiny/bugs/0696-resolve-id/test_0696_resolve_id.py index aed2ba65d..be00085ae 100644 --- a/tests/playwright/shiny/bugs/0696-resolve-id/test_0696_resolve_id.py +++ b/tests/playwright/shiny/bugs/0696-resolve-id/test_0696_resolve_id.py @@ -32,6 +32,7 @@ OutputTextVerbatim, OutputUi, ) +from examples.example_apps import reruns, reruns_delay from mod_state import expect_default_mod_state, expect_mod_state from playwright.sync_api import Page @@ -107,6 +108,7 @@ def expect_default_outputs(page: Page, module_id: str): # Sidebars do not seem to work on webkit. Skipping test on webkit +@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay) @pytest.mark.skip_browser("webkit") def test_module_support(page: Page, local_app: ShinyAppProc) -> None: page.set_viewport_size({"width": 3000, "height": 6000}) From 3111039eee4ac1e03eae2091bef2d7cec14d943b Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 13:04:31 -0500 Subject: [PATCH 4/7] Slow down debug mode --- tests/playwright/playwright-pytest.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/playwright/playwright-pytest.ini b/tests/playwright/playwright-pytest.ini index 7374a2a36..b64fa3f64 100644 --- a/tests/playwright/playwright-pytest.ini +++ b/tests/playwright/playwright-pytest.ini @@ -13,4 +13,4 @@ asyncio_mode=strict # # --headed: Headed browser testing # # -r P: Show extra test summary info: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. (w)arnings... # --maxfail=1: Stop after 1 failure has occurred -addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=on --video=on +addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=on --video=on --slowmo=250 From 07e8188dcea783e9bde629bc92db06fec91b9e0b Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 13:06:13 -0500 Subject: [PATCH 5/7] Use a single browser tab throughout testing session to almost entirely eliminate startup/teardown time --- tests/playwright/conftest.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/playwright/conftest.py b/tests/playwright/conftest.py index 82882e0b5..c088af76d 100644 --- a/tests/playwright/conftest.py +++ b/tests/playwright/conftest.py @@ -39,6 +39,16 @@ "retry_with_timeout", ) +from playwright.sync_api import BrowserContext, Page + + +# Make a single page fixture that can be used by all tests +# By using a single page, the browser is only launched once and all tests run in the same tab / page. +@pytest.fixture(scope="session") +def page(browser: BrowserContext) -> Page: + return browser.new_page() + + here = PurePath(__file__).parent here_root = here.parent.parent From 62360bccbedddf447e7f22b642b2be229bd05c67 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 13:37:52 -0500 Subject: [PATCH 6/7] Go to `about:blank` between tests --- tests/playwright/conftest.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tests/playwright/conftest.py b/tests/playwright/conftest.py index c088af76d..6707dd25e 100644 --- a/tests/playwright/conftest.py +++ b/tests/playwright/conftest.py @@ -43,12 +43,21 @@ # Make a single page fixture that can be used by all tests -# By using a single page, the browser is only launched once and all tests run in the same tab / page. @pytest.fixture(scope="session") -def page(browser: BrowserContext) -> Page: +# By using a single page, the browser is only launched once and all tests run in the same tab / page. +def session_page(browser: BrowserContext) -> Page: return browser.new_page() +@pytest.fixture(scope="function") +# By going to `about:blank`, we _reset_ the page to a known state before each test. +# It is not perfect, but it is faster than making a new page for each test. +# This must be done before each test +def page(session_page: Page) -> Page: + session_page.goto("about:blank") + return session_page + + here = PurePath(__file__).parent here_root = here.parent.parent From 1a08fd9337b1cefbba21c7bc8f3d9c64f8754761 Mon Sep 17 00:00:00 2001 From: Barret Schloerke Date: Wed, 6 Mar 2024 13:56:27 -0500 Subject: [PATCH 7/7] Code review Co-Authored-By: Garrick Aden-Buie --- .github/workflows/pytest.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 25b3fb700..ffdb99196 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -118,7 +118,7 @@ jobs: path: test-results/ retention-days: 5 - playwright-deploys-local: + playwright-deploys-precheck: if: github.event_name != 'release' runs-on: ${{ matrix.os }} strategy: @@ -135,15 +135,15 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Test deploys example apps locally - timeout-minutes: 5 + - name: Test that deployable example apps work + timeout-minutes: 5 # ~10s locally env: DEPLOY_APPS: "false" run: | make playwright-deploys SUB_FILE=". -vv" playwright-deploys: - needs: [playwright-deploys-local] + needs: [playwright-deploys-precheck] if: github.event_name != 'release' && (github.event_name == 'push' || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'deploy'))) # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR) # Only one is allowed to run at a time because it is deploying to the same server location. @@ -163,8 +163,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - # Make sure the apps pass locally - - name: Test deploys example apps locally + - name: Test that deployable example apps work timeout-minutes: 5 # ~10s locally env: DEPLOY_APPS: "false"