diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml
new file mode 100644
index 0000000..cf3af7c
--- /dev/null
+++ b/.github/workflows/ci-tests.yml
@@ -0,0 +1,42 @@
+name: CI Tests for Shiny Assistant
+
+
+on:
+    pull_request:
+        branches:
+        - main
+    push:
+        branches:
+            - "testing**"
+
+
+jobs:
+  smoke-test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r tests/requirements.txt
+          playwright install chromium --with-deps
+
+      - name: Run playwright test
+        env:
+            ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        id: playwright-test
+        run: pytest test_local_app.py
diff --git a/.github/workflows/nightly-tests.yml b/.github/workflows/nightly-tests.yml
new file mode 100644
index 0000000..e8e6af2
--- /dev/null
+++ b/.github/workflows/nightly-tests.yml
@@ -0,0 +1,38 @@
+name: Nightly Tests for Shiny Assistant
+
+on:
+  schedule:
+    - cron: "0 3 * * *"
+  push:
+    branches:
+      - "testing**"
+  workflow_dispatch:
+
+jobs:
+  nightly-tests:
+    runs-on: ubuntu-latest
+    permissions: write-all
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.x"
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r tests/requirements.txt
+          playwright install chromium --with-deps
+
+      - name: Run playwright test
+        id: playwright-test
+        run: python tests/run_smoke_test.py
diff --git a/test_local_app.py b/test_local_app.py
new file mode 100644
index 0000000..fbb811c
--- /dev/null
+++ b/test_local_app.py
@@ -0,0 +1,12 @@
+from playwright.sync_api import Page
+from shiny.run import ShinyAppProc
+
+from tests.utils import send_message_and_verify
+
+
+def test_shiny_assistant(page: Page, local_app: ShinyAppProc) -> None:
+    page.goto(local_app.url)
+
+    send_message_and_verify(
+        page, "Help me write a shiny app with orange cards", "orange"
+    )
diff --git a/tests/requirements.txt b/tests/requirements.txt
new file mode 100644
index 0000000..3e7b297
--- /dev/null
+++ b/tests/requirements.txt
@@ -0,0 +1,3 @@
+playwright
+pytest-playwright
+
diff --git a/tests/run_smoke_test.py b/tests/run_smoke_test.py
new file mode 100644
index 0000000..c50641b
--- /dev/null
+++ b/tests/run_smoke_test.py
@@ -0,0 +1,20 @@
+from playwright.sync_api import Playwright, sync_playwright
+from utils import send_message_and_verify
+
+
+def run(playwright: Playwright):
+    chromium = playwright.chromium
+    browser = chromium.launch()
+    page = browser.new_page()
+    page.goto("https://gallery.shinyapps.io/assistant")
+
+    send_message_and_verify(
+        page,
+        "Help me write a shiny app with orange cards",
+        "orange"
+    )
+    browser.close()
+
+
+with sync_playwright() as playwright:
+    run(playwright)
diff --git a/tests/utils.py b/tests/utils.py
new file mode 100644
index 0000000..c11ebc0
--- /dev/null
+++ b/tests/utils.py
@@ -0,0 +1,18 @@
+import time
+from playwright.sync_api import Page
+
+
+def send_message_and_verify(page: Page, message: str, expected_text: str) -> None:
+    page.get_by_label("Python").check()
+    page.get_by_text("Who can see my activity?").click()
+    page.get_by_text("Who can see my activity?").click()
+    page.get_by_role("textbox", name="Enter a message...").click()
+    page.get_by_role("textbox", name="Enter a message...").fill(message)
+    page.get_by_label("Send message").click()
+    time.sleep(12)  # required since we are streaming responses
+    message_contents = page.query_selector_all(".message-content")
+    last_message_content = message_contents[-1].text_content()
+    if expected_text not in last_message_content:
+        raise AssertionError(
+            f"Expected '{expected_text}' in last message content but got: {last_message_content}"
+        )