diff --git a/backend/score.py b/backend/score.py index 7e7ceff7b..a2c42c982 100644 --- a/backend/score.py +++ b/backend/score.py @@ -159,13 +159,14 @@ async def create_source_knowledge_graph_url( # Set the status "Success" becuase we are treating these error already handled by application as like custom errors. json_obj = {'error_message':error_message, 'status':'Success','db_url':uri, 'userName':userName, 'database':database,'success_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc)),'email':email} logger.log_struct(json_obj, "INFO") + logging.exception(f'File Failed in upload: {e}') return create_api_response('Failed',message=message + error_message[:80],error=error_message,file_source=source_type) except Exception as e: error_message = str(e) message = f" Unable to create source node for source type: {source_type} and source: {source}" json_obj = {'error_message':error_message, 'status':'Failed','db_url':uri, 'userName':userName, 'database':database,'failed_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc)),'email':email} logger.log_struct(json_obj, "ERROR") - logging.exception(f'Exception Stack trace:') + logging.exception(f'Exception Stack trace upload:{e}') return create_api_response('Failed',message=message + error_message[:80],error=error_message,file_source=source_type) finally: gc.collect() @@ -284,6 +285,7 @@ async def extract_knowledge_graph_from_file( json_obj = {'api_name':'extract','message':error_message,'file_created_at':formatted_time(node_detail[0]['created_time']),'error_message':error_message, 'file_name': file_name,'status':'Completed', 'db_url':uri, 'userName':userName, 'database':database,'success_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc)),'email':email} logger.log_struct(json_obj, "INFO") + logging.exception(f'File Failed in extraction: {e}') return create_api_response("Failed", message = error_message, error=error_message, file_name=file_name) except Exception as e: message=f"Failed To Process File:{file_name} or LLM Unable To Parse Content " @@ -295,26 +297,32 @@ async def extract_knowledge_graph_from_file( json_obj = {'api_name':'extract','message':message,'file_created_at':formatted_time(node_detail[0]['created_time']),'error_message':error_message, 'file_name': file_name,'status':'Failed', 'db_url':uri, 'userName':userName, 'database':database,'failed_count':1, 'source_type': source_type, 'source_url':source_url, 'wiki_query':wiki_query, 'logging_time': formatted_time(datetime.now(timezone.utc)),'email':email} logger.log_struct(json_obj, "ERROR") + logging.exception(f'File Failed in extraction: {e}') return create_api_response('Failed', message=message + error_message[:100], error=error_message, file_name = file_name) finally: gc.collect() -@app.get("/sources_list") -async def get_source_list(uri:str=None, userName:str=None, password:str=None, email:str=None, database:str=None): +@app.post("/sources_list") +async def get_source_list( + uri=Form(None), + userName=Form(None), + password=Form(None), + database=Form(None), + email=Form(None)): """ Calls 'get_source_list_from_graph' which returns list of sources which already exist in databse """ try: start = time.time() - if password is not None and password != "null": - decoded_password = decode_password(password) - else: - decoded_password = None - userName = None - database = None - if " " in uri: - uri = uri.replace(" ","+") - result = await asyncio.to_thread(get_source_list_from_graph,uri,userName,decoded_password,database) + # if password is not None and password != "null": + # decoded_password = decode_password(password) + # else: + # decoded_password = None + # userName = None + # database = None + # if " " in uri: + # uri = uri.replace(" ","+") + result = await asyncio.to_thread(get_source_list_from_graph,uri,userName,password,database) end = time.time() elapsed_time = end - start json_obj = {'api_name':'sources_list','db_url':uri, 'userName':userName, 'database':database, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}','email':email} diff --git a/backend/src/document_sources/gcs_bucket.py b/backend/src/document_sources/gcs_bucket.py index d50635571..aec6df330 100644 --- a/backend/src/document_sources/gcs_bucket.py +++ b/backend/src/document_sources/gcs_bucket.py @@ -115,7 +115,7 @@ def merge_file_gcs(bucket_name, original_file_name: str, folder_name_sha1_hashed if blob.exists(): logging.info(f'Blob Name: {blob.name}') chunks.append(blob.download_as_bytes()) - blob.delete() + blob.delete() merged_file = b"".join(chunks) file_name_with__hashed_folder = folder_name_sha1_hashed +'/'+original_file_name diff --git a/backend/src/graphDB_dataAccess.py b/backend/src/graphDB_dataAccess.py index 3bf4f73a0..cb8a9879d 100644 --- a/backend/src/graphDB_dataAccess.py +++ b/backend/src/graphDB_dataAccess.py @@ -20,9 +20,10 @@ def update_exception_db(self, file_name, exp_msg, retry_condition): try: job_status = "Failed" result = self.get_current_status_document_node(file_name) - is_cancelled_status = result[0]['is_cancelled'] - if bool(is_cancelled_status) == True: - job_status = 'Cancelled' + if len(result) > 0: + is_cancelled_status = result[0]['is_cancelled'] + if bool(is_cancelled_status) == True: + job_status = 'Cancelled' if retry_condition is not None: retry_condition = None self.graph.query("""MERGE(d:Document {fileName :$fName}) SET d.status = $status, d.errorMessage = $error_msg, d.retry_condition = $retry_condition""", diff --git a/backend/src/llm.py b/backend/src/llm.py index c1a5f3eeb..1cfb0a52b 100644 --- a/backend/src/llm.py +++ b/backend/src/llm.py @@ -194,24 +194,20 @@ async def get_graph_document_list( return graph_document_list async def get_graph_from_llm(model, chunkId_chunkDoc_list, allowedNodes, allowedRelationship, chunks_to_combine, additional_instructions=None): - try: - llm, model_name = get_llm(model) - combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list, chunks_to_combine) + + llm, model_name = get_llm(model) + combined_chunk_document_list = get_combined_chunks(chunkId_chunkDoc_list, chunks_to_combine) + + if allowedNodes is None or allowedNodes=="": + allowedNodes =[] + else: + allowedNodes = allowedNodes.split(',') + if allowedRelationship is None or allowedRelationship=="": + allowedRelationship=[] + else: + allowedRelationship = allowedRelationship.split(',') - if allowedNodes is None or allowedNodes=="": - allowedNodes =[] - else: - allowedNodes = allowedNodes.split(',') - if allowedRelationship is None or allowedRelationship=="": - allowedRelationship=[] - else: - allowedRelationship = allowedRelationship.split(',') - - graph_document_list = await get_graph_document_list( - llm, combined_chunk_document_list, allowedNodes, allowedRelationship, additional_instructions - ) - return graph_document_list - except Exception as e: - err = f"Error during extracting graph with llm: {e}" - logging.error(err) - raise + graph_document_list = await get_graph_document_list( + llm, combined_chunk_document_list, allowedNodes, allowedRelationship, additional_instructions + ) + return graph_document_list \ No newline at end of file diff --git a/backend/src/main.py b/backend/src/main.py index 4359ad51f..59bff1935 100644 --- a/backend/src/main.py +++ b/backend/src/main.py @@ -233,7 +233,7 @@ async def extract_graph_from_file_local_file(uri, userName, password, database, file_name, pages, file_extension = get_documents_from_file_by_path(merged_file_path,fileName) if pages==None or len(pages)==0: raise LLMGraphBuilderException(f'File content is not available for file : {file_name}') - return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, True, merged_file_path) + return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, True, merged_file_path, additional_instructions=additional_instructions) else: return await processing_source(uri, userName, password, database, model, fileName, [], allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, True, merged_file_path, retry_condition, additional_instructions=additional_instructions) @@ -247,7 +247,7 @@ async def extract_graph_from_file_s3(uri, userName, password, database, model, s if pages==None or len(pages)==0: raise LLMGraphBuilderException(f'File content is not available for file : {file_name}') - return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine) + return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, additional_instructions=additional_instructions) else: return await processing_source(uri, userName, password, database, model, file_name, [], allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, retry_condition=retry_condition, additional_instructions=additional_instructions) @@ -256,7 +256,7 @@ async def extract_graph_from_web_page(uri, userName, password, database, model, file_name, pages = get_documents_from_web_page(source_url) if pages==None or len(pages)==0: raise LLMGraphBuilderException(f'Content is not available for given URL : {file_name}') - return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine) + return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, additional_instructions=additional_instructions) else: return await processing_source(uri, userName, password, database, model, file_name, [], allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, retry_condition=retry_condition, additional_instructions=additional_instructions) @@ -266,7 +266,7 @@ async def extract_graph_from_file_youtube(uri, userName, password, database, mod if pages==None or len(pages)==0: raise LLMGraphBuilderException(f'Youtube transcript is not available for file : {file_name}') - return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine) + return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, additional_instructions=additional_instructions) else: return await processing_source(uri, userName, password, database, model, file_name, [], allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, retry_condition=retry_condition, additional_instructions=additional_instructions) @@ -275,7 +275,7 @@ async def extract_graph_from_file_Wikipedia(uri, userName, password, database, m file_name, pages = get_documents_from_Wikipedia(wiki_query, language) if pages==None or len(pages)==0: raise LLMGraphBuilderException(f'Wikipedia page is not available for file : {file_name}') - return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine) + return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, additional_instructions=additional_instructions) else: return await processing_source(uri, userName, password, database, model, file_name,[], allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, retry_condition=retry_condition, additional_instructions=additional_instructions) @@ -284,7 +284,7 @@ async def extract_graph_from_file_gcs(uri, userName, password, database, model, file_name, pages = get_documents_from_gcs(gcs_project_id, gcs_bucket_name, gcs_bucket_folder, gcs_blob_filename, access_token) if pages==None or len(pages)==0: raise LLMGraphBuilderException(f'File content is not available for file : {file_name}') - return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine) + return await processing_source(uri, userName, password, database, model, file_name, pages, allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, additional_instructions=additional_instructions) else: return await processing_source(uri, userName, password, database, model, file_name, [], allowedNodes, allowedRelationship, token_chunk_size, chunk_overlap, chunks_to_combine, retry_condition=retry_condition, additional_instructions=additional_instructions) diff --git a/docker-compose.yml b/docker-compose.yml index fb7a1e19a..4b166f490 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,8 +34,6 @@ services: # - LLM_MODEL_CONFIG_bedrock_claude_3_5_sonnet=${LLM_MODEL_CONFIG_bedrock_claude_3_5_sonnet-} # - LLM_MODEL_CONFIG_fireworks_qwen_72b=${LLM_MODEL_CONFIG_fireworks_qwen_72b-} - LLM_MODEL_CONFIG_ollama_llama3=${LLM_MODEL_CONFIG_ollama_llama3-} - # env_file: - # - ./backend/.env container_name: backend extra_hosts: - host.docker.internal:host-gateway diff --git a/example.env b/example.env deleted file mode 100644 index 71c9dd36b..000000000 --- a/example.env +++ /dev/null @@ -1,36 +0,0 @@ -# Optional Backend -EMBEDDING_MODEL="all-MiniLM-L6-v2" -IS_EMBEDDING="true" -KNN_MIN_SCORE="0.94" -# Enable Gemini (default is False) | Can be False or True -GEMINI_ENABLED=False -# LLM_MODEL_CONFIG_ollama_llama3="llama3,http://host.docker.internal:11434" - -# Enable Google Cloud logs (default is False) | Can be False or True -GCP_LOG_METRICS_ENABLED=False -NUMBER_OF_CHUNKS_TO_COMBINE=6 -UPDATE_GRAPH_CHUNKS_PROCESSED=20 -NEO4J_URI="neo4j://database:7687" -NEO4J_USERNAME="neo4j" -NEO4J_PASSWORD="password" -LANGCHAIN_API_KEY="" -LANGCHAIN_PROJECT="" -LANGCHAIN_TRACING_V2="true" -LANGCHAIN_ENDPOINT="https://api.smith.langchain.com" -GCS_FILE_CACHE=False -ENTITY_EMBEDDING=True - -# Optional Frontend -VITE_BACKEND_API_URL="http://localhost:8000" -VITE_BLOOM_URL="https://workspace-preview.neo4j.io/workspace/explore?connectURL={CONNECT_URL}&search=Show+me+a+graph&featureGenAISuggestions=true&featureGenAISuggestionsInternal=true" -VITE_REACT_APP_SOURCES="local,youtube,wiki,s3,web" -VITE_ENV="DEV" -VITE_TIME_PER_PAGE=50 -VITE_CHUNK_SIZE=5242880 -VITE_CHUNK_OVERLAP=20 -VITE_TOKENS_PER_CHUNK=100 -VITE_CHUNK_TO_COMBINE=1 -VITE_GOOGLE_CLIENT_ID="" -VITE_CHAT_MODES="" -VITE_BATCH_SIZE=2 -VITE_LLM_MODELS_PROD="openai_gpt_4o,openai_gpt_4o_mini,diffbot,gemini_1.5_flash" diff --git a/frontend/package.json b/frontend/package.json index 8c134f654..f60327cd5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -36,22 +36,22 @@ "uuid": "^9.0.1" }, "devDependencies": { + "@tailwindcss/postcss": "^4.0.7", "@types/node": "^20.11.10", "@types/react": "^18.2.15", "@types/react-dom": "^18.2.7", - "@typescript-eslint/eslint-plugin": "^6.0.0", + "@typescript-eslint/eslint-plugin": "^7.0.0", "@typescript-eslint/parser": "^6.0.0", "@vitejs/plugin-react": "^4.0.3", - "autoprefixer": "^10.4.17", "eslint": "^8.45.0", "eslint-config-prettier": "^8.5.0", - "eslint-plugin-react-hooks": "^4.6.0", + "eslint-plugin-react-hooks": "^5.1.0", "eslint-plugin-react-refresh": "^0.4.3", "postcss": "^8.4.33", "prettier": "^2.7.1", - "react-dropzone": "^14.2.3", - "tailwindcss": "^4.0.6", - "typescript": "^5.0.2", + "react-dropzone": "^14.3.5", + "tailwindcss": "^4.0.7", + "typescript": "^5.7.3", "vite": "^4.5.3" } } diff --git a/frontend/postcss.config.js b/frontend/postcss.config.js index 2e7af2b7f..a7f73a2d1 100644 --- a/frontend/postcss.config.js +++ b/frontend/postcss.config.js @@ -1,6 +1,5 @@ export default { plugins: { - tailwindcss: {}, - autoprefixer: {}, + '@tailwindcss/postcss': {}, }, } diff --git a/frontend/src/App.css b/frontend/src/App.css index bbc0fc44f..b60fb5acf 100644 --- a/frontend/src/App.css +++ b/frontend/src/App.css @@ -335,7 +335,7 @@ .layout-wrapper{ display: grid; grid-template-rows: auto; - grid-template-columns: 64px 1fr minmax(min-content,4fr) 1.4fr 64px; + grid-template-columns: 64px 1fr minmax(min-content,4fr) 1.1fr 64px; max-height: calc(100vh - 58px); max-width: 100%; } @@ -356,3 +356,8 @@ position: relative; height: 100%; } +.sidenav-container{ + height: calc(100vh - 58px); + min-height: 200px; + display: flex +} \ No newline at end of file diff --git a/frontend/src/components/ChatBot/Chatbot.tsx b/frontend/src/components/ChatBot/Chatbot.tsx index e6b6a321a..bebe2f2ee 100644 --- a/frontend/src/components/ChatBot/Chatbot.tsx +++ b/frontend/src/components/ChatBot/Chatbot.tsx @@ -436,7 +436,7 @@ const Chatbot: FC = (props) => { className='-ml-4' hasStatus name='KM' - size='x-large' + size='large' source={ChatBotAvatar} status={connectionStatus ? 'online' : 'offline'} shape='square' @@ -447,7 +447,7 @@ const Chatbot: FC = (props) => { className='' hasStatus name='KM' - size='x-large' + size='large' status={connectionStatus ? 'online' : 'offline'} shape='square' type='image' @@ -457,7 +457,7 @@ const Chatbot: FC = (props) => { @@ -612,7 +612,7 @@ const Chatbot: FC = (props) => { }} > - + "" diff --git a/frontend/src/components/ChatBot/ExpandedChatButtonContainer.tsx b/frontend/src/components/ChatBot/ExpandedChatButtonContainer.tsx index 934e4c624..136eeef15 100644 --- a/frontend/src/components/ChatBot/ExpandedChatButtonContainer.tsx +++ b/frontend/src/components/ChatBot/ExpandedChatButtonContainer.tsx @@ -22,7 +22,7 @@ const ExpandedChatButtonContainer: React.FC = ({ closeChatBot, delete menuAnchor={chatAnchor} isRoot={false} /> -
+
{ diff --git a/frontend/src/components/ChatBot/MultiModeMetrics.tsx b/frontend/src/components/ChatBot/MultiModeMetrics.tsx index f3391d9e4..1b5127cc0 100644 --- a/frontend/src/components/ChatBot/MultiModeMetrics.tsx +++ b/frontend/src/components/ChatBot/MultiModeMetrics.tsx @@ -336,7 +336,7 @@ export default function MultiModeMetrics({ }} isAutoResizingColumns={true} isLoading={metricsLoading} - // rootProps={{ className: isWithAdditionalMetrics === false ? '!w-[465px]' : 'auto' }} + // rootProps={{ className: isWithAdditionalMetrics === false ? 'w-[465px]!' : 'auto' }} components={{ Body: () => ( = ({ placement='top' text='Enhance graph quality' label='Graph Enhancemnet Settings' - className='mr-2.5' + className='mr-2!' onClick={toggleEnhancementDialog} disabled={!connectionStatus || isReadOnlyUser} size={isTablet ? 'small' : 'medium'} @@ -919,7 +919,7 @@ const Content: React.FC = ({ {!connectionStatus ? (