From c9fd5a2ee588557e54a36d4eb4f851df316b0a87 Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 20:15:48 +0000 Subject: [PATCH 1/4] Modify prompt --- src/backend/fastapi_app/prompts/answer.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/fastapi_app/prompts/answer.txt b/src/backend/fastapi_app/prompts/answer.txt index 9d8878b1..0c627c8a 100644 --- a/src/backend/fastapi_app/prompts/answer.txt +++ b/src/backend/fastapi_app/prompts/answer.txt @@ -1,5 +1,5 @@ Assistant helps customers with questions about products. -Respond as if you are a salesperson helping a customer in a store. Do NOT respond with tables. +Respond in a very friendly way as if you are a salesperson helping a customer in a store. Do NOT respond with tables. Answer ONLY with the product details listed in the products. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. From f2c6df743f2748bf559f0f745e3567a46ddd4c6d Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 20:16:20 +0000 Subject: [PATCH 2/4] Modify prompt --- src/backend/fastapi_app/api_models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index 35068a80..632b66f0 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -24,7 +24,7 @@ class RetrievalMode(str, Enum): class ChatRequestOverrides(BaseModel): top: int = 3 - temperature: float = 0.3 + temperature: float = 1.0 retrieval_mode: RetrievalMode = RetrievalMode.HYBRID use_advanced_flow: bool = True prompt_template: str | None = None From be04034bc61fc6d121b914793b3e17ece0cc0d2d Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 20:25:43 +0000 Subject: [PATCH 3/4] Change temp in eval config --- .github/workflows/app-tests.yaml | 2 +- evals/eval_config.json | 2 +- src/backend/fastapi_app/api_models.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/app-tests.yaml b/.github/workflows/app-tests.yaml index 220297f4..300c07e5 100644 --- a/.github/workflows/app-tests.yaml +++ b/.github/workflows/app-tests.yaml @@ -28,7 +28,7 @@ jobs: fail-fast: false matrix: os: ["ubuntu-latest", "macos-latest-xlarge", "macos-13", "windows-latest"] - python_version: ["3.10", "3.11", "3.12"] + python_version: ["3.12"] exclude: - os: macos-latest-xlarge python_version: "3.10" diff --git a/evals/eval_config.json b/evals/eval_config.json index c51f9e79..d9761afa 100644 --- a/evals/eval_config.json +++ b/evals/eval_config.json @@ -8,7 +8,7 @@ "use_advanced_flow": true, "top": 3, "retrieval_mode": "hybrid", - "temperature": 0.3 + "temperature": 1.0 } }, "target_response_answer_jmespath": "message.content", diff --git a/src/backend/fastapi_app/api_models.py b/src/backend/fastapi_app/api_models.py index 632b66f0..35068a80 100644 --- a/src/backend/fastapi_app/api_models.py +++ b/src/backend/fastapi_app/api_models.py @@ -24,7 +24,7 @@ class RetrievalMode(str, Enum): class ChatRequestOverrides(BaseModel): top: int = 3 - temperature: float = 1.0 + temperature: float = 0.3 retrieval_mode: RetrievalMode = RetrievalMode.HYBRID use_advanced_flow: bool = True prompt_template: str | None = None From 9ba9aea09a0813b05e3fd3993c8ad69c73787b9e Mon Sep 17 00:00:00 2001 From: Pamela Fox Date: Tue, 22 Oct 2024 20:34:50 +0000 Subject: [PATCH 4/4] Try to make it worse --- evals/eval_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evals/eval_config.json b/evals/eval_config.json index d9761afa..8536d71a 100644 --- a/evals/eval_config.json +++ b/evals/eval_config.json @@ -5,8 +5,8 @@ "target_url": "http://127.0.0.1:8000/chat", "target_parameters": { "overrides": { - "use_advanced_flow": true, - "top": 3, + "use_advanced_flow": false, + "top": 10, "retrieval_mode": "hybrid", "temperature": 1.0 }