test: Speed up tests by using single page. Do not attempt deploys job unless conditions are valid

.github/workflows/pytest.yaml

@@ -118,9 +118,35 @@ jobs:
           path: test-results/
           retention-days: 5
 
-  playwright-deploys:
+  playwright-deploys-precheck:
     if: github.event_name != 'release'
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        # Matches deploy server python version
+        python-version: ["3.10"]
+        os: [ubuntu-latest]
+      fail-fast: false
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Setup py-shiny
+        uses: ./.github/py-shiny/setup
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Test that deployable example apps work
+        timeout-minutes: 5 # ~10s locally
+        env:
+          DEPLOY_APPS: "false"
+        run: |
+          make playwright-deploys SUB_FILE=". -vv"
+
+  playwright-deploys:
+    needs: [playwright-deploys-precheck]
+    if: github.event_name != 'release' && (github.event_name == 'push' || (github.event_name == 'pull_request' && startsWith(github.head_ref, 'deploy')))
     # Only allow one `playwright-deploys` job to run at a time. (Independent of branch / PR)
+    # Only one is allowed to run at a time because it is deploying to the same server location.
     concurrency: playwright-deploys
     runs-on: ${{ matrix.os }}
     strategy:
@@ -137,8 +163,8 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Test deploys example apps locally
-        timeout-minutes: 5
+      - name: Test that deployable example apps work
+        timeout-minutes: 5 # ~10s locally
         env:
           DEPLOY_APPS: "false"
         run: |

tests/playwright/conftest.py

@@ -39,6 +39,25 @@
     "retry_with_timeout",
 )
 
+from playwright.sync_api import BrowserContext, Page
+
+
+# Make a single page fixture that can be used by all tests
+@pytest.fixture(scope="session")
+# By using a single page, the browser is only launched once and all tests run in the same tab / page.
+def session_page(browser: BrowserContext) -> Page:
+    return browser.new_page()
+
+
+@pytest.fixture(scope="function")
+# By going to `about:blank`, we _reset_ the page to a known state before each test.
+# It is not perfect, but it is faster than making a new page for each test.
+# This must be done before each test
+def page(session_page: Page) -> Page:
+    session_page.goto("about:blank")
+    return session_page
+
+
 here = PurePath(__file__).parent
 here_root = here.parent.parent
 

tests/playwright/playwright-pytest.ini

@@ -7,8 +7,10 @@ asyncio_mode=strict
 # --numprocesses auto: number of testing workers. auto is number of (virtual) cores
 # --video=retain-on-failure: playwright saves recording of any failed test
 # --tracing=retain-on-failure: playwright saves trace of any failed test
+# --video=on: Always save playwright recording for every test
+# --tracing=on: Always save playwright trace for every test
 # -vv: Extra extra verbose output
 # # --headed: Headed browser testing
 # # -r P: Show extra test summary info: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed (p/P), or (A)ll. (w)arnings...
 # --maxfail=1: Stop after 1 failure has occurred
-addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=retain-on-failure --video=retain-on-failure
+addopts = --strict-markers --durations=6 --durations-min=5.0 --browser chromium --numprocesses auto -vvv --maxfail=1 --headed --tracing=on --video=on --slowmo=250

tests/playwright/shiny/bugs/0696-resolve-id/test_0696_resolve_id.py

@@ -32,6 +32,7 @@
     OutputTextVerbatim,
     OutputUi,
 )
+from examples.example_apps import reruns, reruns_delay
 from mod_state import expect_default_mod_state, expect_mod_state
 from playwright.sync_api import Page
 
@@ -107,6 +108,7 @@ def expect_default_outputs(page: Page, module_id: str):
 
 
 # Sidebars do not seem to work on webkit. Skipping test on webkit
+@pytest.mark.flaky(reruns=reruns, reruns_delay=reruns_delay)
 @pytest.mark.skip_browser("webkit")
 def test_module_support(page: Page, local_app: ShinyAppProc) -> None:
     page.set_viewport_size({"width": 3000, "height": 6000})