From 8ea0abf9e0c5c2ed3f8bee157a6e4695ba729669 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 13:23:20 +0300 Subject: [PATCH 01/12] add playwright and fix db port for running tests outside container --- .devcontainer/docker-compose.yaml | 3 +++ .gitignore | 2 ++ pyproject.toml | 4 ++-- requirements-dev.txt | 5 +++-- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index cd408336..552bc0e7 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,6 +27,9 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres + ports: + - "5432:5432" + # For local developemnt, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/.gitignore b/.gitignore index 20dda899..18609cf6 100644 --- a/.gitignore +++ b/.gitignore @@ -146,3 +146,5 @@ npm-debug.log* node_modules static/ +# Playwright test trace +test-results/ diff --git a/pyproject.toml b/pyproject.toml index 5905aa19..1f168be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,9 @@ python_version = 3.12 exclude = [".venv/*"] [tool.pytest.ini_options] -addopts = "-ra --cov" +addopts = "-ra" testpaths = ["tests"] -pythonpath = ['src'] +pythonpath = ['src/backend'] filterwarnings = ["ignore::DeprecationWarning"] [[tool.mypy.overrides]] diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d7ad271..5ebb470c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,9 +4,10 @@ mypy pre-commit pip-tools pip-compile-cross-platform +playwright pytest -pytest-cov pytest-asyncio +pytest-cov +pytest-playwright pytest-snapshot -mypy locust From b67dbc97f4b543d3bbec255a8e94471c8f3b2fa9 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:03:16 +0300 Subject: [PATCH 02/12] change 'session_state' to 'sessionState' Change to conform with microsoft chat protocol --- src/backend/fastapi_app/api_models.py | 5 +++-- src/frontend/src/pages/chat/Chat.tsx | 3 ++- .../test_advanced_chat_flow/advanced_chat_flow_response.json | 2 +- .../advanced_chat_streaming_flow_response.jsonlines | 4 ++-- .../test_simple_chat_flow/simple_chat_flow_response.json | 2 +- .../simple_chat_streaming_flow_response.jsonlines | 4 ++-- 6 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index c98ca76d..616275c2 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -37,6 +37,7 @@ class ChatRequestContext(BaseModel): class ChatRequest(BaseModel): messages: list[ChatCompletionMessageParam] context: ChatRequestContext + sessionState: Any | None = None class ThoughtStep(BaseModel): @@ -54,13 +55,13 @@ class RAGContext(BaseModel): class RetrievalResponse(BaseModel): message: Message context: RAGContext - session_state: Any | None = None + sessionState: Any | None = None class RetrievalResponseDelta(BaseModel): delta: Message | None = None context: RAGContext | None = None - session_state: Any | None = None + sessionState: Any | None = None class ItemPublic(BaseModel): diff --git a/src/frontend/src/pages/chat/Chat.tsx b/src/frontend/src/pages/chat/Chat.tsx index da0b6934..f583f012 100644 --- a/src/frontend/src/pages/chat/Chat.tsx +++ b/src/frontend/src/pages/chat/Chat.tsx @@ -108,7 +108,8 @@ const Chat = () => { prompt_template: promptTemplate.length === 0 ? undefined : promptTemplate, temperature: temperature } - } + }, + sessionState: answers.length ? answers[answers.length - 1][1].sessionState : null }; const chatClient: AIChatProtocolClient = new AIChatProtocolClient("/chat"); if (shouldStream) { diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json index 2e9eb3ae..c7692bd1 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json @@ -64,5 +64,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines index 8b65342f..b7e4efa3 100644 --- a/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Prompt to generate search arguments","description":["{'role': 'system', 'content': 'Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching database rows.\\nYou have access to an Azure PostgreSQL database with an items table that has columns for title, description, brand, price, and type.\\nGenerate a search query based on the conversation and the new question.\\nIf the question is not in English, translate the question to English before generating the search query.\\nIf you cannot generate a search query, return the original user question.\\nDO NOT return anything besides the query.'}","{'role': 'user', 'content': 'What is the capital of France?'}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}},{"title":"Search using generated search arguments","description":"The capital of France is Paris. [Benefit_Options-2.pdf].","props":{"top":1,"vector_search":true,"text_search":true,"filters":[]}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} diff --git a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json index d5ecba21..a73ff24d 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json +++ b/tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json @@ -52,5 +52,5 @@ ], "followup_questions": null }, - "session_state": null + "sessionState": null } \ No newline at end of file diff --git a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines index 6251bd52..fc63aea9 100644 --- a/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines +++ b/tests/snapshots/test_api_routes/test_simple_chat_streaming_flow/simple_chat_streaming_flow_response.jsonlines @@ -1,2 +1,2 @@ -{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"session_state":null} -{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"session_state":null} +{"delta":null,"context":{"data_points":{"1":{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}},"thoughts":[{"title":"Search query for database","description":"What is the capital of France?","props":{"top":1,"vector_search":true,"text_search":true}},{"title":"Search results","description":[{"id":1,"type":"Footwear","brand":"Daybird","name":"Wanderer Black Hiking Boots","description":"Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long.","price":109.99}],"props":{}},{"title":"Prompt to generate answer","description":["{'role': 'system', 'content': \"Assistant helps customers with questions about products.\\nRespond as if you are a salesperson helping a customer in a store. Do NOT respond with tables.\\nAnswer ONLY with the product details listed in the products.\\nIf there isn't enough information below, say you don't know.\\nDo not generate answers that don't use the sources below.\\nEach product has an ID in brackets followed by colon and the product details.\\nAlways include the product ID for each product you use in the response.\\nUse square brackets to reference the source, for example [52].\\nDon't combine citations, list each product separately, for example [27][51].\"}","{'role': 'user', 'content': \"What is the capital of France?\\n\\nSources:\\n[1]:Name:Wanderer Black Hiking Boots Description:Daybird's Wanderer Hiking Boots in sleek black are perfect for all your outdoor adventures. These boots are made with a waterproof leather upper and a durable rubber sole for superior traction. With their cushioned insole and padded collar, these boots will keep you comfortable all day long. Price:109.99 Brand:Daybird Type:Footwear\\n\\n\"}"],"props":{"model":"gpt-35-turbo","deployment":"gpt-35-turbo"}}],"followup_questions":null},"sessionState":null} +{"delta":{"content":"The capital of France is Paris. [Benefit_Options-2.pdf].","role":"assistant"},"context":null,"sessionState":null} From 367a86d852d60eac936db4b3423d01fe6d0b1193 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:08:05 +0300 Subject: [PATCH 03/12] add playwright tests and workflows --- .github/workflows/app-tests.yaml | 14 ++- tests/e2e.py | 192 +++++++++++++++++++++++++++++++ 2 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 tests/e2e.py diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index ec3e5e1b..c1c14bd1 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -73,4 +73,16 @@ jobs: - name: Run MyPy run: python3 -m mypy . - name: Run Pytest - run: python3 -m pytest + run: python3 -m pytest -s -vv --cov --cov-fail-under=85 + - name: Run E2E tests with Playwright + id: e2e + if: runner.os != 'Windows' + run: | + playwright install chromium --with-deps + python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Upload test artifacts + if: ${{ failure() && steps.e2e.conclusion == 'failure' }} + uses: actions/upload-artifact@v4 + with: + name: playwright-traces${{ matrix.python_version }} + path: test-results diff --git a/tests/e2e.py b/tests/e2e.py new file mode 100644 index 00000000..ace7f591 --- /dev/null +++ b/tests/e2e.py @@ -0,0 +1,192 @@ +import socket +import time +from collections.abc import Generator +from contextlib import closing +from multiprocessing import Process + +import pytest +import requests +import uvicorn +from playwright.sync_api import Page, Route, expect + +import fastapi_app as app + +expect.set_options(timeout=10_000) + + +def wait_for_server_ready(url: str, timeout: float = 10.0, check_interval: float = 0.5) -> bool: + """Make requests to provided url until it responds without error.""" + conn_error = None + for _ in range(int(timeout / check_interval)): + try: + requests.get(url) + except requests.ConnectionError as exc: + time.sleep(check_interval) + conn_error = str(exc) + else: + return True + raise RuntimeError(conn_error) + + +@pytest.fixture(scope="session") +def free_port() -> int: + """Returns a free port for the test server to bind.""" + with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: + s.bind(("", 0)) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + return s.getsockname()[1] + + +def run_server(port: int): + uvicorn.run(app.create_app(testing=True), port=port) + + +@pytest.fixture() +def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, None]: + proc = Process(target=run_server, args=(free_port,), daemon=True) + proc.start() + url = f"http://localhost:{free_port}/" + wait_for_server_ready(url, timeout=10.0, check_interval=0.5) + yield url + proc.kill() + + +@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) +def sized_page(page: Page, request): + size = request.param + page.set_viewport_size({"width": size[0], "height": size[1]}) + yield page + + +def test_home(page: Page, live_server_url: str): + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + +def test_chat(sized_page: Page, live_server_url: str): + page = sized_page + + # Set up a mock route to the /chat endpoint with streaming results + def handle(route: Route): + # Assert that session_state is specified in the request (None for now) + if route.request.post_data_json: + session_state = route.request.post_data_json["sessionState"] + assert session_state is None + # Read the JSONL from our snapshot results and return as the response + f = open( + "tests/snapshots/test_api_routes/test_advanced_chat_streaming_flow/advanced_chat_streaming_flow_response.jsonlines" + ) + jsonl = f.read() + f.close() + route.fulfill(body=jsonl, status=200, headers={"Transfer-encoding": "Chunked"}) + + page.route("*/**/chat/stream", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("heading", name="Product chat")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + # Show the thought process + page.get_by_label("Show thought process").click() + expect(page.get_by_title("Thought process")).to_be_visible() + expect(page.get_by_text("Prompt to generate search arguments")).to_be_visible() + + # Clear the chat + page.get_by_role("button", name="Clear chat").click() + expect(page.get_by_text("Whats the dental plan?")).not_to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).not_to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_disabled() + + +def test_chat_customization(page: Page, live_server_url: str): + # Set up a mock route to the /chat endpoint + def handle(route: Route): + if route.request.post_data_json: + overrides = route.request.post_data_json["context"]["overrides"] + assert overrides["use_advanced_flow"] is False + assert overrides["retrieval_mode"] == "vectors" + assert overrides["top"] == 1 + assert overrides["prompt_template"] == "You are a cat and only talk about tuna." + + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_simple_chat_flow/simple_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + + # Customize all the settings + page.get_by_role("button", name="Developer settings").click() + page.get_by_text( + "Use advanced flow with query rewriting and filter formulation. Not compatible with Ollama models." + ).click() + page.get_by_label("Retrieve this many matching rows:").click() + page.get_by_label("Retrieve this many matching rows:").fill("1") + page.get_by_text("Vectors + Text (Hybrid)").click() + page.get_by_role("option", name="Vectors", exact=True).click() + page.get_by_label("Override prompt template").click() + page.get_by_label("Override prompt template").fill("You are a cat and only talk about tuna.") + + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_role("button", name="Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() + + +def test_chat_nonstreaming(page: Page, live_server_url: str): + # Set up a mock route to the /chat_stream endpoint + def handle(route: Route): + # Read the JSON from our snapshot results and return as the response + f = open("tests/snapshots/test_api_routes/test_advanced_chat_flow/advanced_chat_flow_response.json") + json = f.read() + f.close() + route.fulfill(body=json, status=200) + + page.route("*/**/chat", handle) + + # Check initial page state + page.goto(live_server_url) + expect(page).to_have_title("RAG on PostgreSQL") + expect(page.get_by_role("button", name="Developer settings")).to_be_enabled() + page.get_by_role("button", name="Developer settings").click() + page.get_by_text("Stream chat completion responses").click() + page.locator("button").filter(has_text="Close").click() + + # Ask a question and wait for the message to appear + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").click() + page.get_by_placeholder("Type a new question (e.g. does my plan cover annual eye exams?)").fill( + "Whats the dental plan?" + ) + page.get_by_label("Ask question button").click() + + expect(page.get_by_text("Whats the dental plan?")).to_be_visible() + expect(page.get_by_text("The capital of France is Paris.")).to_be_visible() + expect(page.get_by_role("button", name="Clear chat")).to_be_enabled() From 8f27e2396a5946b9841cce7302852eb9755d0dc1 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Wed, 24 Jul 2024 14:08:23 +0300 Subject: [PATCH 04/12] add types for mypy --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 5ebb470c..08ce71aa 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,7 @@ -r src/backend/requirements.txt ruff mypy +types-requests pre-commit pip-tools pip-compile-cross-platform From 4c7bcaa04c777467c9316a3559e5f11924cffaf9 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Wed, 24 Jul 2024 14:50:25 -0700 Subject: [PATCH 05/12] Update .devcontainer/docker-compose.yaml --- .devcontainer/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index 552bc0e7..e1c04e14 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -29,7 +29,7 @@ services: ports: - "5432:5432" - # For local developemnt, we need to forward the database port here too. + # For local development, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) From 8d6abfd9003d1cec2d070372f30822a304458d69 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 15:26:35 +0300 Subject: [PATCH 06/12] apply feedback from PR review --- .devcontainer/docker-compose.yaml | 3 --- tests/e2e.py | 11 +---------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml index e1c04e14..cd408336 100644 --- a/.devcontainer/docker-compose.yaml +++ b/.devcontainer/docker-compose.yaml @@ -27,9 +27,6 @@ services: POSTGRES_USER: admin POSTGRES_PASSWORD: postgres - ports: - - "5432:5432" - # For local development, we need to forward the database port here too. # Add "forwardPorts": ["5432"] to **devcontainer.json** to forward PostgreSQL locally. # (Adding the "ports" property to this file will not forward from a Codespace.) diff --git a/tests/e2e.py b/tests/e2e.py index ace7f591..56f6023d 100644 --- a/tests/e2e.py +++ b/tests/e2e.py @@ -51,21 +51,12 @@ def live_server_url(mock_session_env, free_port: int) -> Generator[str, None, No proc.kill() -@pytest.fixture(params=[(480, 800), (600, 1024), (768, 1024), (992, 1024), (1024, 768)]) -def sized_page(page: Page, request): - size = request.param - page.set_viewport_size({"width": size[0], "height": size[1]}) - yield page - - def test_home(page: Page, live_server_url: str): page.goto(live_server_url) expect(page).to_have_title("RAG on PostgreSQL") -def test_chat(sized_page: Page, live_server_url: str): - page = sized_page - +def test_chat(page: Page, live_server_url: str): # Set up a mock route to the /chat endpoint with streaming results def handle(route: Route): # Assert that session_state is specified in the request (None for now) From 7522ae6cfd32f12d47f4c85675785929d293c87e Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 16:00:20 +0300 Subject: [PATCH 07/12] testing without trace --- .github/workflows/app-tests.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index c1c14bd1..09c50485 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -80,8 +80,13 @@ jobs: run: | playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure + - name: Run E2E tests on Windows with Playwright + if: runner.os == 'Windows' + run: | + playwright install chromium --with-deps + python3 -m pytest tests/e2e.py - name: Upload test artifacts - if: ${{ failure() && steps.e2e.conclusion == 'failure' }} + if: ${{ failure() && steps.e2e.conclusion == 'failure' && runner.os != 'Windows' }} uses: actions/upload-artifact@v4 with: name: playwright-traces${{ matrix.python_version }} From acca2146193186753065e0bde293a56096061d47 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 16:10:05 +0300 Subject: [PATCH 08/12] Update app-tests.yaml --- .github/workflows/app-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 09c50485..bd9f9be2 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest-xlarge", "macos-13"] + os: ["windows-latest"] python_version: ["3.10", "3.11", "3.12"] exclude: - os: macos-latest-xlarge From 276f73a84b621183b305ec4d3b42c88c71cb64c2 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 16:37:25 +0300 Subject: [PATCH 09/12] Update app-tests.yaml --- .github/workflows/app-tests.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index bd9f9be2..20833f82 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -78,15 +78,17 @@ jobs: id: e2e if: runner.os != 'Windows' run: | + echo "PLAYWRIGHT_BROWSERS_PATH=$HOME/.cache/playwright-bin" >> $GITHUB_ENV playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure - name: Run E2E tests on Windows with Playwright if: runner.os == 'Windows' run: | + echo "PLAYWRIGHT_BROWSERS_PATH=$HOME\.cache\playwright-bin" >> $GITHUB_ENV playwright install chromium --with-deps - python3 -m pytest tests/e2e.py + python3 -m pytest tests/e2e.py --tracing=retain-on-failure - name: Upload test artifacts - if: ${{ failure() && steps.e2e.conclusion == 'failure' && runner.os != 'Windows' }} + if: ${{ failure() && steps.e2e.conclusion == 'failure' }} uses: actions/upload-artifact@v4 with: name: playwright-traces${{ matrix.python_version }} From fd8815ee42962202fc4b3c6b2ea0b53057befc12 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 16:51:39 +0300 Subject: [PATCH 10/12] update to run on windows and speed up: --- .github/workflows/app-tests.yaml | 43 +------------------------------- 1 file changed, 1 insertion(+), 42 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 20833f82..612437d1 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -19,32 +19,8 @@ jobs: matrix: os: ["windows-latest"] python_version: ["3.10", "3.11", "3.12"] - exclude: - - os: macos-latest-xlarge - python_version: "3.10" steps: - uses: actions/checkout@v4 - - name: Check for MacOS Runner - if: matrix.os == 'macos-latest-xlarge' - run: brew install postgresql@14 - - name: Install pgvector on Windows using install-pgvector.bat - if: matrix.os == 'windows-latest' - shell: cmd - run: .github\workflows\install-pgvector.bat - - name: Install PostgreSQL development libraries - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt update - sudo apt install postgresql-server-dev-14 - - name: Setup postgres - uses: ikalnytskyi/action-setup-postgres@v6 - with: - username: admin - password: postgres - database: postgres - - name: Install pgvector on MacOS/Linux using install-pgvector.sh - if: matrix.os != 'windows-latest' - run: .github/workflows/install-pgvector.sh - name: Setup python uses: actions/setup-python@v5 with: @@ -56,11 +32,6 @@ jobs: - name: Install app as editable app run: | python -m pip install -e src/backend - - name: Setup local database with seed data - run: | - cp .env.sample .env - python ./src/backend/fastapi_app/setup_postgres_database.py - python ./src/backend/fastapi_app/setup_postgres_seeddata.py - name: Setup node uses: actions/setup-node@v4 with: @@ -70,21 +41,9 @@ jobs: cd ./src/frontend npm install npm run build - - name: Run MyPy - run: python3 -m mypy . - - name: Run Pytest - run: python3 -m pytest -s -vv --cov --cov-fail-under=85 - - name: Run E2E tests with Playwright - id: e2e - if: runner.os != 'Windows' - run: | - echo "PLAYWRIGHT_BROWSERS_PATH=$HOME/.cache/playwright-bin" >> $GITHUB_ENV - playwright install chromium --with-deps - python3 -m pytest tests/e2e.py --tracing=retain-on-failure - name: Run E2E tests on Windows with Playwright - if: runner.os == 'Windows' run: | - echo "PLAYWRIGHT_BROWSERS_PATH=$HOME\.cache\playwright-bin" >> $GITHUB_ENV + export PLAYWRIGHT_BROWSERS_PATH="$HOME\AppData\Local\ms-playwright" playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure - name: Upload test artifacts From 6446800311952c6316e67fde89cb507bb6f8642c Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 17:04:46 +0300 Subject: [PATCH 11/12] fix shell type --- .github/workflows/app-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 612437d1..c199373a 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -42,6 +42,7 @@ jobs: npm install npm run build - name: Run E2E tests on Windows with Playwright + shell: bash run: | export PLAYWRIGHT_BROWSERS_PATH="$HOME\AppData\Local\ms-playwright" playwright install chromium --with-deps From 6e5ac160904813e486aa3c6ee0d904cd5fdd05e2 Mon Sep 17 00:00:00 2001 From: John Aziz Date: Thu, 25 Jul 2024 17:19:35 +0300 Subject: [PATCH 12/12] attempt for fixing the path --- .github/workflows/app-tests.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index c199373a..ddc18537 100755 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -42,9 +42,8 @@ jobs: npm install npm run build - name: Run E2E tests on Windows with Playwright - shell: bash run: | - export PLAYWRIGHT_BROWSERS_PATH="$HOME\AppData\Local\ms-playwright" + set PLAYWRIGHT_BROWSERS_PATH=%USERPROFILE%\AppData\Local\ms-playwright playwright install chromium --with-deps python3 -m pytest tests/e2e.py --tracing=retain-on-failure - name: Upload test artifacts