From 76547bec50b76a39cd4a0c3a3ed72b79520907eb Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Mon, 23 Sep 2024 12:52:14 +0000 Subject: [PATCH 01/22] added global search+vector+fulltext mode --- backend/src/QA_integration.py | 62 +++++++++++++++++-------------- backend/src/shared/constants.py | 66 +++++++++++++++++++++++++++++---- 2 files changed, 93 insertions(+), 35 deletions(-) diff --git a/backend/src/QA_integration.py b/backend/src/QA_integration.py index 651440437..643e22062 100644 --- a/backend/src/QA_integration.py +++ b/backend/src/QA_integration.py @@ -158,14 +158,17 @@ def format_documents(documents, model): formatted_docs = [] sources = set() - lc_entities = {'entities':list()} + local_search_entities = {'entities':list()} + global_communities = {"communitydetails":list()} + for doc in sorted_documents: try: source = doc.metadata.get('source', "unknown") sources.add(source) - lc_entities = doc.metadata if 'entities'in doc.metadata.keys() else lc_entities + local_search_entities = doc.metadata if 'entities'in doc.metadata.keys() else local_search_entities + global_communities = doc.metadata if 'communitydetails'in doc.metadata.keys() else global_communities formatted_doc = ( "Document start\n" @@ -178,13 +181,13 @@ def format_documents(documents, model): except Exception as e: logging.error(f"Error formatting document: {e}") - return "\n\n".join(formatted_docs), sources,lc_entities + return "\n\n".join(formatted_docs), sources,local_search_entities,global_communities def process_documents(docs, question, messages, llm, model,chat_mode_settings): start_time = time.time() try: - formatted_docs, sources,lc_entities = format_documents(docs, model) + formatted_docs, sources, local_search_entities,global_communities = format_documents(docs, model) rag_chain = get_rag_chain(llm=llm) @@ -193,12 +196,16 @@ def process_documents(docs, question, messages, llm, model,chat_mode_settings): "context": formatted_docs, "input": question }) + + result = {'sources': [], 'chunkdetails': [], 'entities': [], 'communitydetails': []} + if chat_mode_settings["mode"] == "entity search+vector": - result = {'sources': list(), - 'chunkdetails': list()} - result.update(lc_entities) + result.update(local_search_entities) + elif chat_mode_settings["mode"] == "global search+vector+fulltext": + result.update(global_communities) else: result = get_sources_and_chunks(sources, docs) + content = ai_response.content total_tokens = get_total_tokens(ai_response, llm) @@ -269,10 +276,13 @@ def create_document_retriever_chain(llm, retriever): def initialize_neo4j_vector(graph, chat_mode_settings): try: - mode = chat_mode_settings.get('mode', 'undefined') retrieval_query = chat_mode_settings.get("retrieval_query") index_name = chat_mode_settings.get("index_name") keyword_index = chat_mode_settings.get("keyword_index", "") + node_label = chat_mode_settings.get("node_label") + embedding_node_property = chat_mode_settings.get("embedding_node_property") + text_node_properties = chat_mode_settings.get("text_node_properties") + if not retrieval_query or not index_name: raise ValueError("Required settings 'retrieval_query' or 'index_name' are missing.") @@ -284,28 +294,21 @@ def initialize_neo4j_vector(graph, chat_mode_settings): retrieval_query=retrieval_query, graph=graph, search_type="hybrid", - node_label="Chunk", - embedding_node_property="embedding", - text_node_properties=["text"], + node_label=node_label, + embedding_node_property=embedding_node_property, + text_node_properties=text_node_properties, keyword_index_name=keyword_index ) logging.info(f"Successfully retrieved Neo4jVector Fulltext index '{index_name}' and keyword index '{keyword_index}'") - elif mode == "entity search+vector": - neo_db = Neo4jVector.from_existing_index( - embedding=EMBEDDING_FUNCTION, - index_name=index_name, - retrieval_query=retrieval_query, - graph=graph - ) else: neo_db = Neo4jVector.from_existing_graph( embedding=EMBEDDING_FUNCTION, index_name=index_name, retrieval_query=retrieval_query, graph=graph, - node_label="Chunk", - embedding_node_property="embedding", - text_node_properties=["text"] + node_label=node_label, + embedding_node_property=embedding_node_property, + text_node_properties=text_node_properties ) logging.info(f"Successfully retrieved Neo4jVector index '{index_name}'") except Exception as e: @@ -333,12 +336,12 @@ def create_retriever(neo_db, document_names, chat_mode_settings,search_k, score_ logging.info(f"Successfully created retriever with search_k={search_k}, score_threshold={score_threshold}") return retriever -def get_neo4j_retriever(graph, document_names,chat_mode_settings, search_k=CHAT_SEARCH_KWARG_K, score_threshold=CHAT_SEARCH_KWARG_SCORE_THRESHOLD): +def get_neo4j_retriever(graph, document_names,chat_mode_settings, score_threshold=CHAT_SEARCH_KWARG_SCORE_THRESHOLD): try: - + neo_db = initialize_neo4j_vector(graph, chat_mode_settings) document_names= list(map(str.strip, json.loads(document_names))) - search_k = LOCAL_COMMUNITY_TOP_K if chat_mode_settings["mode"] == "entity search+vector" else CHAT_SEARCH_KWARG_K + search_k = chat_mode_settings["top_k"] retriever = create_retriever(neo_db, document_names,chat_mode_settings, search_k, score_threshold) return retriever except Exception as e: @@ -371,12 +374,13 @@ def process_chat_response(messages, history, question, model, graph, document_na try: llm, doc_retriever, model_version = setup_chat(model, graph, document_names, chat_mode_settings) - docs = retrieve_documents(doc_retriever, messages) + docs = retrieve_documents(doc_retriever, messages) + if docs: content, result, total_tokens = process_documents(docs, question, messages, llm, model, chat_mode_settings) else: content = "I couldn't find any relevant documents to answer your question." - result = {"sources": [], "chunkdetails": [], "entities": []} + result = {"sources": [], "chunkdetails": [], "entities": [],"communitydetails": []} total_tokens = 0 ai_response = AIMessage(content=content) @@ -397,7 +401,8 @@ def process_chat_response(messages, history, question, model, graph, document_na "total_tokens": total_tokens, "response_time": 0, "mode": chat_mode_settings["mode"], - "entities": result["entities"] + "entities": result["entities"], + "communitydetails":result["communitydetails"], }, "user": "chatbot" } @@ -414,7 +419,8 @@ def process_chat_response(messages, history, question, model, graph, document_na "response_time": 0, "error": f"{type(e).__name__}: {str(e)}", "mode": chat_mode_settings["mode"], - "entities": [] + "entities": [], + "communitydetails":[], }, "user": "chatbot" } diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index bbf579520..631791c77 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -87,7 +87,6 @@ ## CHAT SETUP CHAT_MAX_TOKENS = 1000 -CHAT_SEARCH_KWARG_K = 10 CHAT_SEARCH_KWARG_SCORE_THRESHOLD = 0.5 CHAT_DOC_SPLIT_SIZE = 3000 CHAT_EMBEDDING_FILTER_SCORE_THRESHOLD = 0.10 @@ -149,7 +148,11 @@ QUESTION_TRANSFORM_TEMPLATE = "Given the below conversation, generate a search query to look up in order to get information relevant to the conversation. Only respond with the query, nothing else." -## CHAT QUERIES +## CHAT QUERIES + + +VECTOR_SEARCH_TOP_K = 10 + VECTOR_SEARCH_QUERY = """ WITH node AS chunk, score MATCH (chunk)-[:PART_OF]->(d:Document) @@ -391,38 +394,87 @@ ] AS entities """ +GOBAL_SEARCH_TOP_K = 10 + +GLOBAL_VECTOR_SEARCH_QUERY = """ +WITH collect(distinct {community: node, score: score}) AS communities, + avg(score) AS avg_score + +WITH avg_score, + [c IN communities | c.community.summary] AS texts, + [c IN communities | {id: elementId(c.community), score: c.score}] AS communityDetails + +WITH avg_score, communityDetails, + apoc.text.join(texts, "\n----\n") AS text + +RETURN text, + avg_score AS score, + {communitydetails: communityDetails} AS metadata +""" + CHAT_MODE_CONFIG_MAP= { "vector": { "retrieval_query": VECTOR_SEARCH_QUERY, + "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", "keyword_index": None, - "document_filter": True + "document_filter": True, + "node_label": "Chunk", + "embedding_node_property":"embedding", + "text_node_properties":["text"], + }, "fulltext": { "retrieval_query": VECTOR_SEARCH_QUERY, + "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", "keyword_index": "keyword", - "document_filter": False + "document_filter": False, + "node_label": "Chunk", + "embedding_node_property":"embedding", + "text_node_properties":["text"], }, "entity search+vector": { "retrieval_query": LOCAL_COMMUNITY_SEARCH_QUERY.format(topChunks=LOCAL_COMMUNITY_TOP_CHUNKS, topCommunities=LOCAL_COMMUNITY_TOP_COMMUNITIES, topOutsideRels=LOCAL_COMMUNITY_TOP_OUTSIDE_RELS)+LOCAL_COMMUNITY_SEARCH_QUERY_SUFFIX, + "top_k": LOCAL_COMMUNITY_TOP_K, "index_name": "entity_vector", "keyword_index": None, - "document_filter": False + "document_filter": False, + "node_label": "__Entity__", + "embedding_node_property":"embedding", + "text_node_properties":["id"], }, "graph+vector": { "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY.format(no_of_entites=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT), + "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", "keyword_index": None, - "document_filter": True + "document_filter": True, + "node_label": "Chunk", + "embedding_node_property":"embedding", + "text_node_properties":["text"], }, "graph+vector+fulltext": { "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY.format(no_of_entites=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT), + "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", "keyword_index": "keyword", - "document_filter": False + "document_filter": False, + "node_label": "Chunk", + "embedding_node_property":"embedding", + "text_node_properties":["text"], + }, + "global search+vector+fulltext": { + "retrieval_query": GLOBAL_VECTOR_SEARCH_QUERY, + "top_k": GOBAL_SEARCH_TOP_K, + "index_name": "community_vector", + "keyword_index": "community_keyword", + "document_filter": False, + "node_label": "__Community__", + "embedding_node_property":"embedding", + "text_node_properties":["summary"], }, "default": { "retrieval_query": VECTOR_SEARCH_QUERY, From 0db3df51f6e451a84819b3aa211864d182c7a3e0 Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Mon, 23 Sep 2024 13:32:56 +0000 Subject: [PATCH 02/22] added community details in chunk entities --- backend/score.py | 4 +- backend/src/chunkid_entities.py | 83 ++++++++++++++++++++++++--------- backend/src/shared/constants.py | 9 ++++ 3 files changed, 72 insertions(+), 24 deletions(-) diff --git a/backend/score.py b/backend/score.py index 63a26ec5b..c156a4a7d 100644 --- a/backend/score.py +++ b/backend/score.py @@ -318,10 +318,10 @@ async def chat_bot(uri=Form(),model=Form(None),userName=Form(), password=Form(), gc.collect() @app.post("/chunk_entities") -async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), chunk_ids=Form(None),is_entity=Form()): +async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), chunk_ids=Form(None),type=Form()): try: logging.info(f"URI: {uri}, Username: {userName}, chunk_ids: {chunk_ids}") - result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,chunk_ids=chunk_ids,is_entity=json.loads(is_entity.lower())) + result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,node_ids=chunk_ids,type=type) json_obj = {'api_name':'chunk_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} logger.log_struct(json_obj, "INFO") return create_api_response('Success',data=result) diff --git a/backend/src/chunkid_entities.py b/backend/src/chunkid_entities.py index aa8d1d4ca..441fa9c72 100644 --- a/backend/src/chunkid_entities.py +++ b/backend/src/chunkid_entities.py @@ -118,13 +118,13 @@ def remove_duplicate_nodes(nodes,property="element_id"): return unique_nodes -def process_entityids(driver, chunk_ids): +def process_entityids(driver, entity_ids): """ Processes entity IDs to retrieve local community data. """ try: - logging.info(f"Starting graph query process for entity ids: {chunk_ids}") - entity_ids_list = chunk_ids.split(",") + logging.info(f"Starting graph query process for entity ids: {entity_ids}") + entity_ids_list = entity_ids.split(",") query_body = LOCAL_COMMUNITY_SEARCH_QUERY.format( topChunks=LOCAL_COMMUNITY_TOP_CHUNKS, topCommunities=LOCAL_COMMUNITY_TOP_COMMUNITIES, @@ -138,19 +138,44 @@ def process_entityids(driver, chunk_ids): if records: result["nodes"].extend(records[0]["nodes"]) result["nodes"] = remove_duplicate_nodes(result["nodes"]) + logging.info(f"Nodes and relationships are processed") + result["chunk_data"] = records[0]["chunks"] result["community_data"] = records[0]["communities"] else: result["chunk_data"] = list() result["community_data"] = list() - logging.info(f"Query process completed successfully for chunk ids: {chunk_ids}") + logging.info(f"Query process completed successfully for chunk ids: {entity_ids}") return result except Exception as e: - logging.error(f"chunkid_entities module: Error processing entity ids: {chunk_ids}. Error: {e}") + logging.error(f"chunkid_entities module: Error processing entity ids: {entity_ids}. Error: {e}") raise -def get_entities_from_chunkids(uri, username, password, database ,chunk_ids,is_entity=False): +def process_communityids(driver, community_ids): + """ + Processes community IDs to retrieve local community data. + """ + try: + logging.info(f"Starting graph query process for community ids: {community_ids}") + community_ids_list = community_ids.split(",") + query = GLOBAL_COMMUNITY_DETAILS_QUERY + + records, summary, keys = driver.execute_query(query, communityIds=community_ids_list) + + result = process_records(records) + result = {"nodes": [],"relationships": [],"chunk_data":[]} + if records: + result["community_data"] = records[0]["communities"] + else: + result["community_data"] = list() + logging.info(f"Query process completed successfully for community ids: {community_ids}") + return result + except Exception as e: + logging.error(f"chunkid_entities module: Error processing community ids: {community_ids}. Error: {e}") + raise + +def get_entities_from_chunkids(uri, username, password, database ,node_ids,type): """ Retrieve and process nodes and relationships from a graph database given a list of chunk IDs. @@ -158,7 +183,7 @@ def get_entities_from_chunkids(uri, username, password, database ,chunk_ids,is_e uri (str): The URI of the graph database. username (str): The username for the database authentication. password (str): The password for the database authentication. - chunk_ids (str): A comma-separated string of chunk IDs. + node_ids (str): A comma-separated string of Node IDs. Returns: dict: A dictionary with 'nodes' and 'relationships' keys containing processed data, or an error message. @@ -166,10 +191,10 @@ def get_entities_from_chunkids(uri, username, password, database ,chunk_ids,is_e try: driver = get_graphDB_driver(uri, username, password,database) - if not is_entity: - if chunk_ids: - logging.info(f"chunkid_entities module: Starting for chunk ids : {chunk_ids}") - result = process_chunkids(driver,chunk_ids) + if type == "chunk": + if node_ids: + logging.info(f"chunkid_entities module: Starting for chunk ids : {node_ids}") + result = process_chunkids(driver,node_ids) else: logging.info(f"chunkid_entities module: No chunk ids are passed") result = { @@ -178,18 +203,32 @@ def get_entities_from_chunkids(uri, username, password, database ,chunk_ids,is_e "chunk_data":[] } return result - if chunk_ids: - result = process_entityids(driver,chunk_ids) - logging.info(f"chunkid_entities module: Starting for entity ids : {chunk_ids}") + elif type == "entity": + if node_ids: + result = process_entityids(driver,node_ids) + logging.info(f"chunkid_entities module: Starting for entity ids : {node_ids}") + else: + logging.info(f"chunkid_entities module: No entity ids are passed") + result = { + "nodes": [], + "relationships": [], + "chunk_data":[], + "community_data":[] + } + return result else: - logging.info(f"chunkid_entities module: No entity ids are passed") - result = { - "nodes": [], - "relationships": [], - "chunk_data":[], - "community_data":[] - } - return result + if node_ids: + result = process_communityids(driver,node_ids) + logging.info(f"chunkid_entities module: Starting for community ids : {node_ids}") + else: + logging.info(f"chunkid_entities module: No community ids are passed") + result = { + "nodes": [], + "relationships": [], + "chunk_data":[], + "community_data":[] + } + return result except Exception as e: logging.error(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Error: {str(e)}") diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 631791c77..38b576ab0 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -412,6 +412,15 @@ {communitydetails: communityDetails} AS metadata """ + + +GLOBAL_COMMUNITY_DETAILS_QUERY = """ +MATCH (community:__Community__) +WHERE elementId(community) IN $communityIds +WITH collect(distinct community) AS communities +RETURN [community IN communities | + community {.*, embedding: null, elementid: elementId(community)}] AS communities +""" CHAT_MODE_CONFIG_MAP= { "vector": { "retrieval_query": VECTOR_SEARCH_QUERY, From ffe7a6876794f63553e9e491c51ab31c987009ca Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Tue, 24 Sep 2024 09:51:59 +0000 Subject: [PATCH 03/22] added node ids --- backend/score.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/score.py b/backend/score.py index c156a4a7d..001abb07b 100644 --- a/backend/score.py +++ b/backend/score.py @@ -318,10 +318,10 @@ async def chat_bot(uri=Form(),model=Form(None),userName=Form(), password=Form(), gc.collect() @app.post("/chunk_entities") -async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), chunk_ids=Form(None),type=Form()): +async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), node_ids=Form(None),type=Form()): try: - logging.info(f"URI: {uri}, Username: {userName}, chunk_ids: {chunk_ids}") - result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,node_ids=chunk_ids,type=type) + logging.info(f"URI: {uri}, Username: {userName}, node_ids: {node_ids}") + result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,node_ids=node_ids,type=type) json_obj = {'api_name':'chunk_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} logger.log_struct(json_obj, "INFO") return create_api_response('Success',data=result) From 46168e43e0c4b7b4b0242a6b1e7e5d8a28d0a8ca Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Tue, 24 Sep 2024 16:11:59 +0000 Subject: [PATCH 04/22] updated vector graph query --- backend/src/QA_integration.py | 3 +- backend/src/chunkid_entities.py | 6 +- backend/src/shared/constants.py | 198 +++++++++++++++++++++----------- 3 files changed, 133 insertions(+), 74 deletions(-) diff --git a/backend/src/QA_integration.py b/backend/src/QA_integration.py index 643e22062..e4b93b104 100644 --- a/backend/src/QA_integration.py +++ b/backend/src/QA_integration.py @@ -204,7 +204,8 @@ def process_documents(docs, question, messages, llm, model,chat_mode_settings): elif chat_mode_settings["mode"] == "global search+vector+fulltext": result.update(global_communities) else: - result = get_sources_and_chunks(sources, docs) + source_details = get_sources_and_chunks(sources, docs) + result.update(source_details) content = ai_response.content total_tokens = get_total_tokens(ai_response, llm) diff --git a/backend/src/chunkid_entities.py b/backend/src/chunkid_entities.py index 441fa9c72..d407de5ae 100644 --- a/backend/src/chunkid_entities.py +++ b/backend/src/chunkid_entities.py @@ -160,10 +160,8 @@ def process_communityids(driver, community_ids): logging.info(f"Starting graph query process for community ids: {community_ids}") community_ids_list = community_ids.split(",") query = GLOBAL_COMMUNITY_DETAILS_QUERY + records, summary, keys = driver.execute_query(query, communityids=community_ids_list) - records, summary, keys = driver.execute_query(query, communityIds=community_ids_list) - - result = process_records(records) result = {"nodes": [],"relationships": [],"chunk_data":[]} if records: result["community_data"] = records[0]["communities"] @@ -218,8 +216,8 @@ def get_entities_from_chunkids(uri, username, password, database ,node_ids,type) return result else: if node_ids: - result = process_communityids(driver,node_ids) logging.info(f"chunkid_entities module: Starting for community ids : {node_ids}") + result = process_communityids(driver,node_ids) else: logging.info(f"chunkid_entities module: No community ids are passed") result = { diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 38b576ab0..749b61aa5 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -149,88 +149,145 @@ QUESTION_TRANSFORM_TEMPLATE = "Given the below conversation, generate a search query to look up in order to get information relevant to the conversation. Only respond with the query, nothing else." ## CHAT QUERIES - - VECTOR_SEARCH_TOP_K = 10 VECTOR_SEARCH_QUERY = """ WITH node AS chunk, score MATCH (chunk)-[:PART_OF]->(d:Document) -WITH d, collect(distinct {chunk: chunk, score: score}) as chunks, avg(score) as avg_score +WITH d, + collect(distinct {chunk: chunk, score: score}) AS chunks, + avg(score) AS avg_score + WITH d, avg_score, - [c in chunks | c.chunk.text] as texts, - [c in chunks | {id: c.chunk.id, score: c.score}] as chunkdetails -WITH d, avg_score, chunkdetails, - apoc.text.join(texts, "\n----\n") as text -RETURN text, avg_score AS score, - {source: COALESCE(CASE WHEN d.url CONTAINS "None" THEN d.fileName ELSE d.url END, d.fileName), chunkdetails: chunkdetails} as metadata + [c IN chunks | c.chunk.text] AS texts, + [c IN chunks | {id: c.chunk.id, score: c.score}] AS chunkdetails + +WITH d, avg_score, chunkdetails, + apoc.text.join(texts, "\n----\n") AS text + +RETURN text, + avg_score AS score, + {source: COALESCE(CASE WHEN d.url CONTAINS "None" + THEN d.fileName + ELSE d.url + END, + d.fileName), + chunkdetails: chunkdetails} AS metadata """ -VECTOR_GRAPH_SEARCH_ENTITY_LIMIT = 25 +### Vector graph search +VECTOR_GRAPH_SEARCH_ENTITY_LIMIT = 50 +VECTOR_GRAPH_SEARCH_EMBEDDING_MIN_MATCH = 0.3 +VECTOR_GRAPH_SEARCH_EMBEDDING_MAX_MATCH = 0.9 +VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MINMAX_CASE = 20 +VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MAX_CASE = 40 -VECTOR_GRAPH_SEARCH_QUERY = """ +VECTOR_GRAPH_SEARCH_QUERY_PREFIX = """ WITH node as chunk, score // find the document of the chunk MATCH (chunk)-[:PART_OF]->(d:Document) - // aggregate chunk-details -WITH d, collect(DISTINCT {{chunk: chunk, score: score}}) AS chunks, avg(score) as avg_score +WITH d, collect(DISTINCT {chunk: chunk, score: score}) AS chunks, avg(score) as avg_score // fetch entities -CALL {{ WITH chunks +CALL { WITH chunks UNWIND chunks as chunkScore WITH chunkScore.chunk as chunk -// entities connected to the chunk -// todo only return entities that are actually in the chunk, remember we connect all extracted entities to all chunks -// todo sort by relevancy (embeddding comparision?) cut off after X (e.g. 25) nodes? -OPTIONAL MATCH (chunk)-[:HAS_ENTITY]->(e) -WITH e, count(*) as numChunks -ORDER BY numChunks DESC LIMIT {no_of_entites} -// depending on match to query embedding either 1 or 2 step expansion -WITH CASE WHEN true // vector.similarity.cosine($embedding, e.embedding ) <= 0.95 -THEN -collect {{ OPTIONAL MATCH path=(e)(()-[rels:!HAS_ENTITY&!PART_OF]-()){{0,1}}(:!Chunk&!Document) RETURN path }} -ELSE -collect {{ OPTIONAL MATCH path=(e)(()-[rels:!HAS_ENTITY&!PART_OF]-()){{0,2}}(:!Chunk&!Document) RETURN path }} -END as paths, e -WITH apoc.coll.toSet(apoc.coll.flatten(collect(distinct paths))) as paths, collect(distinct e) as entities -// de-duplicate nodes and relationships across chunks -RETURN collect{{ unwind paths as p unwind relationships(p) as r return distinct r}} as rels, -collect{{ unwind paths as p unwind nodes(p) as n return distinct n}} as nodes, entities -}} +""" + +VECTOR_GRAPH_SEARCH_ENTITY_QUERY = """ + OPTIONAL MATCH (chunk)-[:HAS_ENTITY]->(e) + WITH e, count(*) AS numChunks + ORDER BY numChunks DESC + LIMIT {no_of_entites} -// generate metadata and text components for chunks, nodes and relationships + WITH + CASE + WHEN e.embedding IS NULL OR ({embedding_match_min} <= vector.similarity.cosine($embedding, e.embedding) AND vector.similarity.cosine($embedding, e.embedding) <= {embedding_match_max}) THEN + collect {{ + OPTIONAL MATCH path=(e)(()-[rels:!HAS_ENTITY&!PART_OF]-()){{0,1}}(:!Chunk&!Document&!__Community__) + RETURN path LIMIT {entity_limit_minmax_case} + }} + WHEN e.embedding IS NOT NULL AND vector.similarity.cosine($embedding, e.embedding) > {embedding_match_max} THEN + collect {{ + OPTIONAL MATCH path=(e)(()-[rels:!HAS_ENTITY&!PART_OF]-()){{0,2}}(:!Chunk&!Document&!__Community__) + RETURN path LIMIT {entity_limit_max_case} + }} + ELSE + collect {{ + MATCH path=(e) + RETURN path + }} + END AS paths, e +""" + +VECTOR_GRAPH_SEARCH_QUERY_SUFFIX = """ + WITH apoc.coll.toSet(apoc.coll.flatten(collect(DISTINCT paths))) AS paths, + collect(DISTINCT e) AS entities + + // De-duplicate nodes and relationships across chunks + RETURN + collect { + UNWIND paths AS p + UNWIND relationships(p) AS r + RETURN DISTINCT r + } AS rels, + collect { + UNWIND paths AS p + UNWIND nodes(p) AS n + RETURN DISTINCT n + } AS nodes, + entities +} + +// Generate metadata and text components for chunks, nodes, and relationships WITH d, avg_score, [c IN chunks | c.chunk.text] AS texts, - [c IN chunks | {{id: c.chunk.id, score: c.score}}] AS chunkdetails, - apoc.coll.sort([n in nodes | - -coalesce(apoc.coll.removeAll(labels(n),['__Entity__'])[0],"") +":"+ -n.id + (case when n.description is not null then " ("+ n.description+")" else "" end)]) as nodeTexts, - apoc.coll.sort([r in rels - // optional filter if we limit the node-set - // WHERE startNode(r) in nodes AND endNode(r) in nodes - | -coalesce(apoc.coll.removeAll(labels(startNode(r)),['__Entity__'])[0],"") +":"+ -startNode(r).id + -" " + type(r) + " " + -coalesce(apoc.coll.removeAll(labels(endNode(r)),['__Entity__'])[0],"") +":" + endNode(r).id -]) as relTexts -, entities -// combine texts into response-text - -WITH d, avg_score,chunkdetails, -"Text Content:\\n" + -apoc.text.join(texts,"\\n----\\n") + -"\\n----\\nEntities:\\n"+ -apoc.text.join(nodeTexts,"\\n") + -"\\n----\\nRelationships:\\n" + -apoc.text.join(relTexts,"\\n") - -as text,entities - -RETURN text, avg_score as score, {{length:size(text), source: COALESCE( CASE WHEN d.url CONTAINS "None" THEN d.fileName ELSE d.url END, d.fileName), chunkdetails: chunkdetails}} AS metadata + [c IN chunks | {id: c.chunk.id, score: c.score}] AS chunkdetails, + [n IN nodes | elementId(n)] AS entityIds, + [r IN rels | elementId(r)] AS relIds, + apoc.coll.sort([ + n IN nodes | + coalesce(apoc.coll.removeAll(labels(n), ['__Entity__'])[0], "") + ":" + + n.id + + (CASE WHEN n.description IS NOT NULL THEN " (" + n.description + ")" ELSE "" END) + ]) AS nodeTexts, + apoc.coll.sort([ + r IN rels | + coalesce(apoc.coll.removeAll(labels(startNode(r)), ['__Entity__'])[0], "") + ":" + + startNode(r).id + " " + type(r) + " " + + coalesce(apoc.coll.removeAll(labels(endNode(r)), ['__Entity__'])[0], "") + ":" + endNode(r).id + ]) AS relTexts, + entities + +// Combine texts into response text +WITH d, avg_score, chunkdetails, entityIds, relIds, + "Text Content:\n" + apoc.text.join(texts, "\n----\n") + + "\n----\nEntities:\n" + apoc.text.join(nodeTexts, "\n") + + "\n----\nRelationships:\n" + apoc.text.join(relTexts, "\n") AS text, + entities + +RETURN + text, + avg_score AS score, + { + length: size(text), + source: COALESCE(CASE WHEN d.url CONTAINS "None" THEN d.fileName ELSE d.url END, d.fileName), + chunkdetails: chunkdetails, + entities : { + entityids: entityIds, + relationshipids: relIds + } + } AS metadata """ +VECTOR_GRAPH_SEARCH_QUERY = VECTOR_GRAPH_SEARCH_QUERY_PREFIX+ VECTOR_GRAPH_SEARCH_ENTITY_QUERY.format( + no_of_entites=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT, + embedding_match_min=VECTOR_GRAPH_SEARCH_EMBEDDING_MIN_MATCH, + embedding_match_max=VECTOR_GRAPH_SEARCH_EMBEDDING_MAX_MATCH, + entity_limit_minmax_case=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MINMAX_CASE, + entity_limit_max_case=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MAX_CASE +) + VECTOR_GRAPH_SEARCH_QUERY_SUFFIX + ### Local community search LOCAL_COMMUNITY_TOP_K = 10 LOCAL_COMMUNITY_TOP_CHUNKS = 3 @@ -394,7 +451,12 @@ ] AS entities """ -GOBAL_SEARCH_TOP_K = 10 +LOCAL_COMMUNITY_SEARCH_QUERY_FORMATTED = LOCAL_COMMUNITY_SEARCH_QUERY.format( + topChunks=LOCAL_COMMUNITY_TOP_CHUNKS, + topCommunities=LOCAL_COMMUNITY_TOP_COMMUNITIES, + topOutsideRels=LOCAL_COMMUNITY_TOP_OUTSIDE_RELS)+LOCAL_COMMUNITY_SEARCH_QUERY_SUFFIX + +GLOBAL_SEARCH_TOP_K = 10 GLOBAL_VECTOR_SEARCH_QUERY = """ WITH collect(distinct {community: node, score: score}) AS communities, @@ -416,7 +478,7 @@ GLOBAL_COMMUNITY_DETAILS_QUERY = """ MATCH (community:__Community__) -WHERE elementId(community) IN $communityIds +WHERE elementId(community) IN $communityids WITH collect(distinct community) AS communities RETURN [community IN communities | community {.*, embedding: null, elementid: elementId(community)}] AS communities @@ -444,9 +506,7 @@ "text_node_properties":["text"], }, "entity search+vector": { - "retrieval_query": LOCAL_COMMUNITY_SEARCH_QUERY.format(topChunks=LOCAL_COMMUNITY_TOP_CHUNKS, - topCommunities=LOCAL_COMMUNITY_TOP_COMMUNITIES, - topOutsideRels=LOCAL_COMMUNITY_TOP_OUTSIDE_RELS)+LOCAL_COMMUNITY_SEARCH_QUERY_SUFFIX, + "retrieval_query": LOCAL_COMMUNITY_SEARCH_QUERY_FORMATTED, "top_k": LOCAL_COMMUNITY_TOP_K, "index_name": "entity_vector", "keyword_index": None, @@ -456,7 +516,7 @@ "text_node_properties":["id"], }, "graph+vector": { - "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY.format(no_of_entites=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT), + "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY, "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", "keyword_index": None, @@ -466,7 +526,7 @@ "text_node_properties":["text"], }, "graph+vector+fulltext": { - "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY.format(no_of_entites=VECTOR_GRAPH_SEARCH_ENTITY_LIMIT), + "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY, "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", "keyword_index": "keyword", @@ -477,7 +537,7 @@ }, "global search+vector+fulltext": { "retrieval_query": GLOBAL_VECTOR_SEARCH_QUERY, - "top_k": GOBAL_SEARCH_TOP_K, + "top_k": GLOBAL_SEARCH_TOP_K, "index_name": "community_vector", "keyword_index": "community_keyword", "document_filter": False, From 7ae1d6c326082c3536d221091243366f5595acda Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Wed, 25 Sep 2024 05:16:03 +0000 Subject: [PATCH 05/22] added entities and modified chat response --- backend/src/QA_integration.py | 43 ++++++++++++++++++--------------- backend/src/shared/constants.py | 2 +- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/backend/src/QA_integration.py b/backend/src/QA_integration.py index e4b93b104..f128a7c3c 100644 --- a/backend/src/QA_integration.py +++ b/backend/src/QA_integration.py @@ -121,7 +121,6 @@ def get_sources_and_chunks(sources_used, docs): result = { 'sources': sources_used, 'chunkdetails': chunkdetails_list, - "entities" : list() } return result @@ -156,10 +155,10 @@ def format_documents(documents, model): sorted_documents = sorted(documents, key=lambda doc: doc.state.get("query_similarity_score", 0), reverse=True) sorted_documents = sorted_documents[:prompt_token_cutoff] - formatted_docs = [] + formatted_docs = list() sources = set() - local_search_entities = {'entities':list()} - global_communities = {"communitydetails":list()} + entities = dict() + global_communities = list() for doc in sorted_documents: @@ -167,8 +166,8 @@ def format_documents(documents, model): source = doc.metadata.get('source', "unknown") sources.add(source) - local_search_entities = doc.metadata if 'entities'in doc.metadata.keys() else local_search_entities - global_communities = doc.metadata if 'communitydetails'in doc.metadata.keys() else global_communities + entities = doc.metadata['entities'] if 'entities'in doc.metadata.keys() else entities + global_communities = doc.metadata["communitydetails"] if 'communitydetails'in doc.metadata.keys() else global_communities formatted_doc = ( "Document start\n" @@ -181,13 +180,13 @@ def format_documents(documents, model): except Exception as e: logging.error(f"Error formatting document: {e}") - return "\n\n".join(formatted_docs), sources,local_search_entities,global_communities + return "\n\n".join(formatted_docs), sources,entities,global_communities def process_documents(docs, question, messages, llm, model,chat_mode_settings): start_time = time.time() try: - formatted_docs, sources, local_search_entities,global_communities = format_documents(docs, model) + formatted_docs, sources, entitydetails, communities = format_documents(docs, model) rag_chain = get_rag_chain(llm=llm) @@ -197,15 +196,23 @@ def process_documents(docs, question, messages, llm, model,chat_mode_settings): "input": question }) - result = {'sources': [], 'chunkdetails': [], 'entities': [], 'communitydetails': []} + result = {'sources': list(), 'nodedetails': dict(), 'entities': dict()} + node_details = {"chunkdetails":list(),"entitydetails":list(),"communitydetails":list()} + entities = {'entityids':list(),"relationshipids":list()} if chat_mode_settings["mode"] == "entity search+vector": - result.update(local_search_entities) + node_details["entitydetails"] = entitydetails + elif chat_mode_settings["mode"] == "global search+vector+fulltext": - result.update(global_communities) + node_details["communitydetails"] = communities else: - source_details = get_sources_and_chunks(sources, docs) - result.update(source_details) + sources_and_chunks = get_sources_and_chunks(sources, docs) + result['sources'] = sources_and_chunks['sources'] + node_details["chunkdetails"] = sources_and_chunks["chunkdetails"] + entities.update(entitydetails) + + result["nodedetails"] = node_details + result["entities"] = entities content = ai_response.content total_tokens = get_total_tokens(ai_response, llm) @@ -381,7 +388,7 @@ def process_chat_response(messages, history, question, model, graph, document_na content, result, total_tokens = process_documents(docs, question, messages, llm, model, chat_mode_settings) else: content = "I couldn't find any relevant documents to answer your question." - result = {"sources": [], "chunkdetails": [], "entities": [],"communitydetails": []} + result = {"sources": list(), "nodedetails": list(), "entities": list()} total_tokens = 0 ai_response = AIMessage(content=content) @@ -391,19 +398,18 @@ def process_chat_response(messages, history, question, model, graph, document_na summarization_thread.start() logging.info("Summarization thread started.") # summarize_and_log(history, messages, llm) - + return { "session_id": "", "message": content, "info": { "sources": result["sources"], "model": model_version, - "chunkdetails": result["chunkdetails"], + "nodedetails": result["nodedetails"], "total_tokens": total_tokens, "response_time": 0, "mode": chat_mode_settings["mode"], "entities": result["entities"], - "communitydetails":result["communitydetails"], }, "user": "chatbot" } @@ -415,13 +421,12 @@ def process_chat_response(messages, history, question, model, graph, document_na "message": "Something went wrong", "info": { "sources": [], - "chunkdetails": [], + "nodedetails": [], "total_tokens": 0, "response_time": 0, "error": f"{type(e).__name__}: {str(e)}", "mode": chat_mode_settings["mode"], "entities": [], - "communitydetails":[], }, "user": "chatbot" } diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 749b61aa5..314b2270f 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -176,7 +176,7 @@ """ ### Vector graph search -VECTOR_GRAPH_SEARCH_ENTITY_LIMIT = 50 +VECTOR_GRAPH_SEARCH_ENTITY_LIMIT = 25 VECTOR_GRAPH_SEARCH_EMBEDDING_MIN_MATCH = 0.3 VECTOR_GRAPH_SEARCH_EMBEDDING_MAX_MATCH = 0.9 VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MINMAX_CASE = 20 From c100e9d24dc03532bc84ffe829b37966bd91c09d Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Wed, 25 Sep 2024 05:31:31 +0000 Subject: [PATCH 06/22] added params --- backend/src/chunkid_entities.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/chunkid_entities.py b/backend/src/chunkid_entities.py index d407de5ae..22e9618d6 100644 --- a/backend/src/chunkid_entities.py +++ b/backend/src/chunkid_entities.py @@ -173,7 +173,7 @@ def process_communityids(driver, community_ids): logging.error(f"chunkid_entities module: Error processing community ids: {community_ids}. Error: {e}") raise -def get_entities_from_chunkids(uri, username, password, database ,node_ids,type): +def get_entities_from_chunkids(uri, username, password, database ,nodedetails,entities,mode): """ Retrieve and process nodes and relationships from a graph database given a list of chunk IDs. From 2afc3aedb79835489405cbc0264a03797b87c9e4 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Wed, 25 Sep 2024 08:00:28 +0000 Subject: [PATCH 07/22] api response changes --- .../src/components/ChatBot/ChatInfoModal.tsx | 14 +++++----- .../src/components/ChatBot/ChatModeToggle.tsx | 4 +-- frontend/src/components/ChatBot/Chatbot.tsx | 27 +++++++++++++------ frontend/src/services/ChunkEntitiesInfo.ts | 10 ++++--- frontend/src/types.ts | 23 ++++++++++++++++ frontend/src/utils/Constants.ts | 5 ++++ frontend/src/utils/Utils.ts | 2 ++ 7 files changed, 63 insertions(+), 22 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index 74823520a..c32806599 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -43,6 +43,7 @@ const ChatInfoModal: React.FC = ({ cypher_query, graphonly_entities, error, + entities_ids }) => { const { breakpoints } = tokens; const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`); @@ -86,11 +87,8 @@ const ChatInfoModal: React.FC = ({ (async () => { setLoading(true); try { - const response = await chunkEntitiesAPI( - userCredentials as UserCredentials, - chunk_ids.map((c) => c.id).join(','), - userCredentials?.database, - mode === chatModeLables.entity_vector + const response = await chunkEntitiesAPI(userCredentials as UserCredentials, userCredentials?.database, chunk_ids.join(','),entities_ids.join(','), + mode, ); if (response.data.status === 'Failure') { throw new Error(response.data.error); @@ -178,9 +176,9 @@ const ChatInfoModal: React.FC = ({ {mode != chatModeLables.graph ? Sources used : <>} {mode != chatModeLables.graph ? Chunks : <>} {mode === chatModeLables.graph_vector || - mode === chatModeLables.graph || - mode === chatModeLables.graph_vector_fulltext || - mode === chatModeLables.entity_vector ? ( + mode === chatModeLables.graph || + mode === chatModeLables.graph_vector_fulltext || + mode === chatModeLables.entity_vector ? ( Top Entities used ) : ( <> diff --git a/frontend/src/components/ChatBot/ChatModeToggle.tsx b/frontend/src/components/ChatBot/ChatModeToggle.tsx index 187c37178..1d4d4799a 100644 --- a/frontend/src/components/ChatBot/ChatModeToggle.tsx +++ b/frontend/src/components/ChatBot/ChatModeToggle.tsx @@ -8,7 +8,7 @@ import { capitalizeWithPlus } from '../../utils/Utils'; import { useCredentials } from '../../context/UserCredentials'; export default function ChatModeToggle({ menuAnchor, - closeHandler = () => {}, + closeHandler = () => { }, open, anchorPortal = true, disableBackdrop = false, @@ -34,7 +34,7 @@ export default function ChatModeToggle({ const memoizedChatModes = useMemo(() => { return isGdsActive && isCommunityAllowed ? chatModes - : chatModes?.filter((m) => !m.mode.includes(chatModeLables.entity_vector)); + : chatModes?.filter((m) => !m.mode.includes(chatModeLables.entity_vector) && !m.mode.includes(chatModeLables.global_vector)); }, [isGdsActive, isCommunityAllowed]); const menuItems = useMemo(() => { return memoizedChatModes?.map((m) => { diff --git a/frontend/src/components/ChatBot/Chatbot.tsx b/frontend/src/components/ChatBot/Chatbot.tsx index 25cb944d3..54b33dd2b 100644 --- a/frontend/src/components/ChatBot/Chatbot.tsx +++ b/frontend/src/components/ChatBot/Chatbot.tsx @@ -7,7 +7,7 @@ import { SpeakerXMarkIconOutline, } from '@neo4j-ndl/react/icons'; import ChatBotAvatar from '../../assets/images/chatbot-ai.png'; -import { ChatbotProps, CustomFile, UserCredentials, chunk } from '../../types'; +import { ChatbotProps, CustomFile, UserCredentials, chunk, entity } from '../../types'; import { useCredentials } from '../../context/UserCredentials'; import { chatBotAPI } from '../../services/QnaAPI'; import { v4 as uuidv4 } from 'uuid'; @@ -49,6 +49,7 @@ const Chatbot: FC = (props) => { const [chatsMode, setChatsMode] = useState(chatModeLables.graph_vector_fulltext); const [graphEntitites, setgraphEntitites] = useState<[]>([]); const [messageError, setmessageError] = useState(''); + const [entitiesModal, setEntitiesModal] = useState([]); const [value, copy] = useCopyToClipboard(); const { speak, cancel } = useSpeechSynthesis({ @@ -172,6 +173,7 @@ const Chatbot: FC = (props) => { let chatbotReply; let chatSources; let chatModel; + let chatnodedetails; let chatChunks; let chatTimeTaken; let chatTokensUsed; @@ -180,6 +182,7 @@ const Chatbot: FC = (props) => { let graphonly_entities; let error; let entitiysearchonly_entities; + let chatEntities; const datetime = `${date.toLocaleDateString()} ${date.toLocaleTimeString()}`; const userMessage = { id: Date.now(), user: 'user', message: inputMessage, datetime: datetime, mode: chatMode }; setListMessages([...listMessages, userMessage]); @@ -198,7 +201,11 @@ const Chatbot: FC = (props) => { chatbotReply = chatresponse?.data?.data?.message; chatSources = chatresponse?.data?.data?.info.sources; chatModel = chatresponse?.data?.data?.info.model; - chatChunks = chatresponse?.data?.data?.info.chunkdetails; + chatnodedetails = chatresponse?.data?.data?.info.nodedetails; + chatChunks = chatnodedetails?.chunkdetails?.map((chunk: chunk) => ({ + id: chunk.id, + score: chunk.score, + })); chatTokensUsed = chatresponse?.data?.data?.info.total_tokens; chatTimeTaken = chatresponse?.data?.data?.info.response_time; chatingMode = chatresponse?.data?.data?.info?.mode; @@ -206,11 +213,12 @@ const Chatbot: FC = (props) => { graphonly_entities = chatresponse?.data.data.info.context ?? []; entitiysearchonly_entities = chatresponse?.data.data.info.entities; error = chatresponse.data.data.info.error ?? ''; + chatEntities = chatresponse.data.data.info.entities; const finalbotReply = { reply: chatbotReply, sources: chatSources, model: chatModel, - chunk_ids: chatChunks, + chunk_ids: chatChunks.map((chunk: chunk) => chunk.id), // Extract chunk IDs here total_tokens: chatTokensUsed, response_time: chatTimeTaken, speaking: false, @@ -220,6 +228,7 @@ const Chatbot: FC = (props) => { graphonly_entities, error, entitiysearchonly_entities, + chatEntities }; simulateTypingEffect(finalbotReply); } catch (error) { @@ -326,7 +335,7 @@ const Chatbot: FC = (props) => { isElevated={true} className={`p-4 self-start ${isFullScreen ? 'max-w-[55%]' : ''} ${ chat.user === 'chatbot' ? 'n-bg-palette-neutral-bg-strong' : 'n-bg-palette-primary-bg-weak' - } `} + } `} subheader={ chat.user !== 'chatbot' && chat.mode?.length ? ( @@ -340,9 +349,9 @@ const Chatbot: FC = (props) => {
{chat.message}
@@ -374,6 +383,7 @@ const Chatbot: FC = (props) => { setShowInfoModal(true); setChatsMode(chat.mode ?? ''); setgraphEntitites(chat.graphonly_entities ?? []); + setEntitiesModal(chat.entities ?? []); setmessageError(chat.error ?? ''); }} > @@ -430,7 +440,7 @@ const Chatbot: FC = (props) => { = (props) => { sources={sourcesModal} model={modelModal} chunk_ids={chunkModal} + entities_ids={entitiesModal} response_time={responseTime} total_tokens={tokensUsed} mode={chatsMode} diff --git a/frontend/src/services/ChunkEntitiesInfo.ts b/frontend/src/services/ChunkEntitiesInfo.ts index f69ddd3aa..79502dd21 100644 --- a/frontend/src/services/ChunkEntitiesInfo.ts +++ b/frontend/src/services/ChunkEntitiesInfo.ts @@ -3,18 +3,20 @@ import api from '../API/Index'; const chunkEntitiesAPI = async ( userCredentials: UserCredentials, - chunk_ids: string, database: string = 'neo4j', - is_entity: boolean = false + chunk_ids: string, + entities:string, + mode: string, ) => { try { const formData = new FormData(); formData.append('uri', userCredentials?.uri ?? ''); formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); - formData.append('chunk_ids', chunk_ids); formData.append('database', database); - formData.append('is_entity', String(is_entity)); + formData.append('nodedetails', chunk_ids); + formData.append('entities', entities); + formData.append('mode', mode); const response: ChatInfo_APIResponse = await api.post(`/chunk_entities`, formData, { headers: { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 9631335a9..475fdcd31 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -215,6 +215,7 @@ export interface Messages { model?: string; isLoading?: boolean; response_time?: number; + nodedetails?: nodeDetailsProps[]; chunk_ids?: chunk[]; total_tokens?: number; speaking?: boolean; @@ -420,6 +421,7 @@ export interface chatInfoMessage extends Partial { cypher_query?: string; graphonly_entities: []; error: string; + entities_ids:chunk[]; } export interface eventResponsetypes extends Omit { @@ -668,3 +670,24 @@ export type CommunitiesProps = { loading: boolean; communities: Community[]; }; + +export interface entity { + id: string; + score: number; +}; + +export interface community { + id: string; + score: number; +} + +export type nodeDetailsProps = { + chunkDetails?: chunk[], + entitydetails?: entity[], + communitydetails?: community[] +} + +export type entityProps = { + entityids: [], + relationshipids:[] +} \ No newline at end of file diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index cf37e9313..fb0379b78 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -74,6 +74,7 @@ export const chatModeLables = { entity_vector: 'entity search+vector', unavailableChatMode: 'Chat mode is unavailable when rows are selected', selected: 'Selected', + global_vector: 'global search+vector+fulltext' }; export const chatModes = process.env?.VITE_CHAT_MODES?.trim() != '' @@ -106,6 +107,10 @@ export const chatModes = mode: chatModeLables.entity_vector, description: 'Uses vector indexing on entity nodes for highly relevant entity-based search.', }, + { + mode : chatModeLables.global_vector, + description: 'Use vector and full-text indexing on community nodes to provide accurate, context-aware answers globally.' + } ]; export const chunkSize = process.env.VITE_CHUNK_SIZE ? parseInt(process.env.VITE_CHUNK_SIZE) : 1 * 1024 * 1024; diff --git a/frontend/src/utils/Utils.ts b/frontend/src/utils/Utils.ts index 97e7bda07..a1f8e7b8e 100644 --- a/frontend/src/utils/Utils.ts +++ b/frontend/src/utils/Utils.ts @@ -428,6 +428,8 @@ export const getDescriptionForChatMode = (mode: string): string => { return 'Merges vector indexing, graph connections, and fulltext indexing for a comprehensive search approach, combining semantic similarity, contextual relevance, and keyword-based search for optimal results.'; case chatModeLables.entity_vector: return 'Combines entity node vector indexing with graph connections for accurate entity-based search, providing the most relevant response.'; + case chatModeLables.global_vector: + return 'Use vector and full-text indexing on community nodes to provide accurate, context-aware answers globally.' default: return 'Chat mode description not available'; // Fallback description } From 492f56423b08e1ed159d37b75114fac2819f1365 Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Wed, 25 Sep 2024 09:48:55 +0000 Subject: [PATCH 08/22] added chunk entity query --- backend/src/QA_integration.py | 8 +-- backend/src/chunkid_entities.py | 104 +++++++++++------------------ backend/src/shared/constants.py | 114 +++++++++++++++++++++++--------- 3 files changed, 127 insertions(+), 99 deletions(-) diff --git a/backend/src/QA_integration.py b/backend/src/QA_integration.py index f128a7c3c..296905cfe 100644 --- a/backend/src/QA_integration.py +++ b/backend/src/QA_integration.py @@ -200,10 +200,10 @@ def process_documents(docs, question, messages, llm, model,chat_mode_settings): node_details = {"chunkdetails":list(),"entitydetails":list(),"communitydetails":list()} entities = {'entityids':list(),"relationshipids":list()} - if chat_mode_settings["mode"] == "entity search+vector": + if chat_mode_settings["mode"] == CHAT_ENTITY_VECTOR_MODE: node_details["entitydetails"] = entitydetails - elif chat_mode_settings["mode"] == "global search+vector+fulltext": + elif chat_mode_settings["mode"] == CHAT_GLOBAL_VECTOR_FULLTEXT_MODE: node_details["communitydetails"] = communities else: sources_and_chunks = get_sources_and_chunks(sources, docs) @@ -576,7 +576,7 @@ def create_neo4j_chat_message_history(graph, session_id): raise def get_chat_mode_settings(mode,settings_map=CHAT_MODE_CONFIG_MAP): - default_settings = settings_map["default"] + default_settings = settings_map[CHAT_DEFAULT_MODE] try: chat_mode_settings = settings_map.get(mode, default_settings) chat_mode_settings["mode"] = mode @@ -598,7 +598,7 @@ def QA_RAG(graph, model, question, document_names, session_id, mode): user_question = HumanMessage(content=question) messages.append(user_question) - if mode == "graph": + if mode == CHAT_GRAPH_MODE: result = process_graph_response(model, graph, question, messages, history) else: chat_mode_settings = get_chat_mode_settings(mode=mode) diff --git a/backend/src/chunkid_entities.py b/backend/src/chunkid_entities.py index 22e9618d6..966af6118 100644 --- a/backend/src/chunkid_entities.py +++ b/backend/src/chunkid_entities.py @@ -81,16 +81,16 @@ def process_chunk_data(chunk_data): except Exception as e: logging.error(f"chunkid_entities module: An error occurred while extracting the Chunk text from records: {e}") -def process_chunkids(driver, chunk_ids): +def process_chunkids(driver, chunk_ids, entities): """ Processes chunk IDs to retrieve chunk data. """ try: logging.info(f"Starting graph query process for chunk ids: {chunk_ids}") - chunk_ids_list = chunk_ids.split(",") - - records, summary, keys = driver.execute_query(CHUNK_QUERY, chunksIds=chunk_ids_list) + records, summary, keys = driver.execute_query(CHUNK_QUERY, chunksIds=chunk_ids,entityIds=entities["entityids"], relationshipIds=entities["relationshipids"]) result = process_records(records) + result["nodes"].extend(records[0]["nodes"]) + result["nodes"] = remove_duplicate_nodes(result["nodes"]) logging.info(f"Nodes and relationships are processed") result["chunk_data"] = process_chunk_data(records) @@ -124,7 +124,6 @@ def process_entityids(driver, entity_ids): """ try: logging.info(f"Starting graph query process for entity ids: {entity_ids}") - entity_ids_list = entity_ids.split(",") query_body = LOCAL_COMMUNITY_SEARCH_QUERY.format( topChunks=LOCAL_COMMUNITY_TOP_CHUNKS, topCommunities=LOCAL_COMMUNITY_TOP_COMMUNITIES, @@ -132,7 +131,7 @@ def process_entityids(driver, entity_ids): ) query = LOCAL_COMMUNITY_DETAILS_QUERY_PREFIX + query_body + LOCAL_COMMUNITY_DETAILS_QUERY_SUFFIX - records, summary, keys = driver.execute_query(query, entityIds=entity_ids_list) + records, summary, keys = driver.execute_query(query, entityIds=entity_ids) result = process_records(records) if records: @@ -153,81 +152,58 @@ def process_entityids(driver, entity_ids): raise def process_communityids(driver, community_ids): - """ - Processes community IDs to retrieve local community data. - """ + """Processes community IDs to retrieve community data.""" try: logging.info(f"Starting graph query process for community ids: {community_ids}") - community_ids_list = community_ids.split(",") query = GLOBAL_COMMUNITY_DETAILS_QUERY - records, summary, keys = driver.execute_query(query, communityids=community_ids_list) + records, summary, keys = driver.execute_query(query, communityids=community_ids) + + result = {"nodes": [], "relationships": [], "chunk_data": []} + result["community_data"] = records[0]["communities"] if records else [] - result = {"nodes": [],"relationships": [],"chunk_data":[]} - if records: - result["community_data"] = records[0]["communities"] - else: - result["community_data"] = list() logging.info(f"Query process completed successfully for community ids: {community_ids}") return result except Exception as e: logging.error(f"chunkid_entities module: Error processing community ids: {community_ids}. Error: {e}") raise -def get_entities_from_chunkids(uri, username, password, database ,nodedetails,entities,mode): - """ - Retrieve and process nodes and relationships from a graph database given a list of chunk IDs. - - Parameters: - uri (str): The URI of the graph database. - username (str): The username for the database authentication. - password (str): The password for the database authentication. - node_ids (str): A comma-separated string of Node IDs. - - Returns: - dict: A dictionary with 'nodes' and 'relationships' keys containing processed data, or an error message. - """ +def get_entities_from_chunkids(uri, username, password, database ,nodedetails,entities,mode): try: driver = get_graphDB_driver(uri, username, password,database) - if type == "chunk": - if node_ids: - logging.info(f"chunkid_entities module: Starting for chunk ids : {node_ids}") - result = process_chunkids(driver,node_ids) + default_response = {"nodes": list(),"relationships": list(),"chunk_data": list(),"community_data": list(),} + + if mode == CHAT_GLOBAL_VECTOR_FULLTEXT_MODE: + + if "communitydetails" in nodedetails and nodedetails["communitydetails"]: + community_ids = [item["id"] for item in nodedetails["communitydetails"]] + logging.info(f"chunkid_entities module: Starting for community ids: {community_ids}") + return process_communityids(driver, community_ids) else: - logging.info(f"chunkid_entities module: No chunk ids are passed") - result = { - "nodes": [], - "relationships": [], - "chunk_data":[] - } - return result - elif type == "entity": - if node_ids: - result = process_entityids(driver,node_ids) - logging.info(f"chunkid_entities module: Starting for entity ids : {node_ids}") + logging.info("chunkid_entities module: No community ids are passed") + return default_response + + elif mode == CHAT_ENTITY_VECTOR_MODE: + + if "entitydetails" in nodedetails and nodedetails["entitydetails"]: + entity_ids = [item["id"] for item in nodedetails["entitydetails"]] + logging.info(f"chunkid_entities module: Starting for entity ids: {entity_ids}") + return process_entityids(driver, entity_ids) else: - logging.info(f"chunkid_entities module: No entity ids are passed") - result = { - "nodes": [], - "relationships": [], - "chunk_data":[], - "community_data":[] - } - return result + logging.info("chunkid_entities module: No entity ids are passed") + return default_response + else: - if node_ids: - logging.info(f"chunkid_entities module: Starting for community ids : {node_ids}") - result = process_communityids(driver,node_ids) + + if "chunkdetails" in nodedetails and nodedetails["chunkdetails"]: + chunk_ids = [item["id"] for item in nodedetails["chunkdetails"]] + logging.info(f"chunkid_entities module: Starting for chunk ids: {chunk_ids}") + return process_chunkids(driver, chunk_ids, entities) else: - logging.info(f"chunkid_entities module: No community ids are passed") - result = { - "nodes": [], - "relationships": [], - "chunk_data":[], - "community_data":[] - } - return result + logging.info("chunkid_entities module: No chunk ids are passed") + return default_response except Exception as e: logging.error(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Error: {str(e)}") - raise Exception(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Please check the logs for more details.") from e \ No newline at end of file + raise Exception(f"chunkid_entities module: An error occurred in get_entities_from_chunkids. Please check the logs for more details.") from e + diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 314b2270f..09777daac 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -69,20 +69,72 @@ """ CHUNK_QUERY = """ -match (chunk:Chunk) where chunk.id IN $chunksIds - +MATCH (chunk:Chunk) +WHERE chunk.id IN $chunksIds MATCH (chunk)-[:PART_OF]->(d:Document) -CALL {WITH chunk -MATCH (chunk)-[:HAS_ENTITY]->(e) -MATCH path=(e)(()-[rels:!HAS_ENTITY&!PART_OF]-()){0,2}(:!Chunk &! Document &! `__Community__`) -UNWIND rels as r -RETURN collect(distinct r) as rels -} -WITH d, collect(distinct chunk) as chunks, apoc.coll.toSet(apoc.coll.flatten(collect(rels))) as rels -RETURN d as doc, [chunk in chunks | chunk {.*, embedding:null}] as chunks, - [r in rels | {startNode:{element_id:elementId(startNode(r)), labels:labels(startNode(r)), properties:{id:startNode(r).id,description:startNode(r).description}}, - endNode:{element_id:elementId(endNode(r)), labels:labels(endNode(r)), properties:{id:endNode(r).id,description:endNode(r).description}}, - relationship: {type:type(r), element_id:elementId(r)}}] as entities + +WITH d, + collect(distinct chunk) AS chunks + +// Collect relationships and nodes +WITH d, chunks, + collect { + MATCH ()-[r]->() + WHERE elementId(r) IN $relationshipIds + RETURN r + } AS rels, + collect { + MATCH (e) + WHERE elementId(e) IN $entityIds + RETURN e + } AS nodes + +WITH d, + chunks, + apoc.coll.toSet(apoc.coll.flatten(rels)) AS rels, + nodes + +RETURN + d AS doc, + [chunk IN chunks | + chunk {.*, embedding: null} + ] AS chunks, + [ + node IN nodes | + { + element_id: elementId(node), + labels: labels(node), + properties: { + id: node.id, + description: node.description + } + } + ] AS nodes, + [ + r IN rels | + { + startNode: { + element_id: elementId(startNode(r)), + labels: labels(startNode(r)), + properties: { + id: startNode(r).id, + description: startNode(r).description + } + }, + endNode: { + element_id: elementId(endNode(r)), + labels: labels(endNode(r)), + properties: { + id: endNode(r).id, + description: endNode(r).description + } + }, + relationship: { + type: type(r), + element_id: elementId(r) + } + } + ] AS entities """ ## CHAT SETUP @@ -90,12 +142,6 @@ CHAT_SEARCH_KWARG_SCORE_THRESHOLD = 0.5 CHAT_DOC_SPLIT_SIZE = 3000 CHAT_EMBEDDING_FILTER_SCORE_THRESHOLD = 0.10 -CHAT_TOKEN_CUT_OFF = { - ("openai-gpt-3.5",'azure_ai_gpt_35',"gemini-1.0-pro","gemini-1.5-pro","groq-llama3",'groq_llama3_70b','anthropic_claude_3_5_sonnet','fireworks_llama_v3_70b','bedrock_claude_3_5_sonnet', ) : 4, - ("openai-gpt-4","diffbot" ,'azure_ai_gpt_4o',"openai-gpt-4o", "openai-gpt-4o-mini") : 28, - ("ollama_llama3") : 2 -} - CHAT_TOKEN_CUT_OFF = { ("openai-gpt-3.5",'azure_ai_gpt_35',"gemini-1.0-pro","gemini-1.5-pro","groq-llama3",'groq_llama3_70b','anthropic_claude_3_5_sonnet','fireworks_llama_v3_70b','bedrock_claude_3_5_sonnet', ) : 4, @@ -483,8 +529,20 @@ RETURN [community IN communities | community {.*, embedding: null, elementid: elementId(community)}] AS communities """ + +## CHAT MODES + +CHAT_VECTOR_MODE = "vector" +CHAT_FULLTEXT_MODE = "fulltext" +CHAT_ENTITY_VECTOR_MODE = "entity search+vector" +CHAT_VECTOR_GRAPH_MODE = "graph+vector" +CHAT_VECTOR_GRAPH_FULLTEXT_MODE = "graph+vector+fulltext" +CHAT_GLOBAL_VECTOR_FULLTEXT_MODE = "global search+vector+fulltext" +CHAT_GRAPH_MODE = "graph" +CHAT_DEFAULT_MODE = "graph+vector+fulltext" + CHAT_MODE_CONFIG_MAP= { - "vector": { + CHAT_VECTOR_MODE : { "retrieval_query": VECTOR_SEARCH_QUERY, "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", @@ -495,7 +553,7 @@ "text_node_properties":["text"], }, - "fulltext": { + CHAT_FULLTEXT_MODE : { "retrieval_query": VECTOR_SEARCH_QUERY, "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", @@ -505,7 +563,7 @@ "embedding_node_property":"embedding", "text_node_properties":["text"], }, - "entity search+vector": { + CHAT_ENTITY_VECTOR_MODE : { "retrieval_query": LOCAL_COMMUNITY_SEARCH_QUERY_FORMATTED, "top_k": LOCAL_COMMUNITY_TOP_K, "index_name": "entity_vector", @@ -515,7 +573,7 @@ "embedding_node_property":"embedding", "text_node_properties":["id"], }, - "graph+vector": { + CHAT_VECTOR_GRAPH_MODE : { "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY, "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", @@ -525,7 +583,7 @@ "embedding_node_property":"embedding", "text_node_properties":["text"], }, - "graph+vector+fulltext": { + CHAT_VECTOR_GRAPH_FULLTEXT_MODE : { "retrieval_query": VECTOR_GRAPH_SEARCH_QUERY, "top_k": VECTOR_SEARCH_TOP_K, "index_name": "vector", @@ -535,7 +593,7 @@ "embedding_node_property":"embedding", "text_node_properties":["text"], }, - "global search+vector+fulltext": { + CHAT_GLOBAL_VECTOR_FULLTEXT_MODE : { "retrieval_query": GLOBAL_VECTOR_SEARCH_QUERY, "top_k": GLOBAL_SEARCH_TOP_K, "index_name": "community_vector", @@ -545,12 +603,6 @@ "embedding_node_property":"embedding", "text_node_properties":["summary"], }, - "default": { - "retrieval_query": VECTOR_SEARCH_QUERY, - "index_name": "vector", - "keyword_index": None, - "document_filter": True - } } YOUTUBE_CHUNK_SIZE_SECONDS = 60 From 87874bdf25f26fce9d401e0249f0c59b3c0e9ed1 Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Wed, 25 Sep 2024 11:30:36 +0000 Subject: [PATCH 09/22] modifies query --- backend/src/shared/constants.py | 71 ++++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 09777daac..82f32d214 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -21,49 +21,72 @@ GRAPH_QUERY = """ MATCH docs = (d:Document) WHERE d.fileName IN $document_names -WITH docs, d ORDER BY d.createdAt DESC -// fetch chunks for documents, currently with limit +WITH docs, d +ORDER BY d.createdAt DESC + +// Fetch chunks for documents, currently with limit CALL {{ WITH d - OPTIONAL MATCH chunks=(d)<-[:PART_OF|FIRST_CHUNK]-(c:Chunk) + OPTIONAL MATCH chunks = (d)<-[:PART_OF|FIRST_CHUNK]-(c:Chunk) RETURN c, chunks LIMIT {graph_chunk_limit} }} -WITH collect(distinct docs) as docs, collect(distinct chunks) as chunks, collect(distinct c) as selectedChunks -WITH docs, chunks, selectedChunks -// select relationships between selected chunks +WITH collect(distinct docs) AS docs, + collect(distinct chunks) AS chunks, + collect(distinct c) AS selectedChunks + +// Select relationships between selected chunks WITH *, -[ c in selectedChunks | [p=(c)-[:NEXT_CHUNK|SIMILAR]-(other) WHERE other IN selectedChunks | p]] as chunkRels + [c IN selectedChunks | + [p = (c)-[:NEXT_CHUNK|SIMILAR]-(other) + WHERE other IN selectedChunks | p]] AS chunkRels -// fetch entities and relationships between entities +// Fetch entities and relationships between entities CALL {{ WITH selectedChunks - UNWIND selectedChunks as c - - OPTIONAL MATCH entities=(c:Chunk)-[:HAS_ENTITY]->(e) - OPTIONAL MATCH entityRels=(e)--(e2:!Chunk) WHERE exists {{ + UNWIND selectedChunks AS c + OPTIONAL MATCH entities = (c:Chunk)-[:HAS_ENTITY]->(e) + OPTIONAL MATCH entityRels = (e)--(e2:!Chunk) + WHERE exists {{ (e2)<-[:HAS_ENTITY]-(other) WHERE other IN selectedChunks }} - RETURN entities , entityRels, collect(DISTINCT e) as entity + RETURN entities, entityRels, collect(DISTINCT e) AS entity }} -WITH docs,chunks,chunkRels, collect(entities) as entities, collect(entityRels) as entityRels, entity + +WITH docs, chunks, chunkRels, + collect(entities) AS entities, + collect(entityRels) AS entityRels, + entity WITH * CALL {{ - with entity - unwind entity as n - OPTIONAL MATCH community=(n:__Entity__)-[:IN_COMMUNITY]->(p:__Community__) - OPTIONAL MATCH parentcommunity=(p)-[:PARENT_COMMUNITY*]->(p2:__Community__) - return collect(community) as communities , collect(parentcommunity) as parentCommunities + WITH entity + UNWIND entity AS n + OPTIONAL MATCH community = (n:__Entity__)-[:IN_COMMUNITY]->(p:__Community__) + OPTIONAL MATCH parentcommunity = (p)-[:PARENT_COMMUNITY*]->(p2:__Community__) + RETURN collect(community) AS communities, + collect(parentcommunity) AS parentCommunities }} -WITH apoc.coll.flatten(docs + chunks + chunkRels + entities + entityRels + communities + parentCommunities, true) as paths +WITH apoc.coll.flatten(docs + chunks + chunkRels + entities + entityRels + communities + parentCommunities, true) AS paths + +// Distinct nodes and relationships +CALL {{ + WITH paths + UNWIND paths AS path + UNWIND nodes(path) AS node + WITH distinct node + RETURN collect(node /* {{.*, labels:labels(node), elementId:elementId(node), embedding:null, text:null}} */) AS nodes +}} + +CALL {{ + WITH paths + UNWIND paths AS path + UNWIND relationships(path) AS rel + RETURN collect(distinct rel) AS rels +}} -// distinct nodes and rels -CALL {{ WITH paths UNWIND paths AS path UNWIND nodes(path) as node WITH distinct node - RETURN collect(node /* {{.*, labels:labels(node), elementId:elementId(node), embedding:null, text:null}} */) AS nodes }} -CALL {{ WITH paths UNWIND paths AS path UNWIND relationships(path) as rel RETURN collect(distinct rel) AS rels }} RETURN nodes, rels """ From cbb91b1de0f5050098e3610832c20a0bd867cef0 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:50:50 +0000 Subject: [PATCH 10/22] labels cahnge for nodes --- .../components/Graph/CheckboxSelection.tsx | 20 ++++++----- .../src/components/Graph/GraphViewModal.tsx | 34 ++++++++++--------- frontend/src/types.ts | 2 ++ frontend/src/utils/Utils.ts | 11 ++++++ 4 files changed, 42 insertions(+), 25 deletions(-) diff --git a/frontend/src/components/Graph/CheckboxSelection.tsx b/frontend/src/components/Graph/CheckboxSelection.tsx index b335a4324..738a1dd01 100644 --- a/frontend/src/components/Graph/CheckboxSelection.tsx +++ b/frontend/src/components/Graph/CheckboxSelection.tsx @@ -3,21 +3,23 @@ import React from 'react'; import { CheckboxSectionProps } from '../../types'; import { graphLabels } from '../../utils/Constants'; -const CheckboxSelection: React.FC = ({ graphType, loading, handleChange, isgds }) => ( +const CheckboxSelection: React.FC = ({ graphType, loading, handleChange, isgds, isDocChunk, isEntity }) => (
- handleChange('DocumentChunk')} - /> - handleChange('Entities')} - /> + />)} + {isEntity && ( + handleChange('Entities')} + /> + )} {isgds && ( = ({ graphType.includes('DocumentChunk') && graphType.includes('Entities') ? queryMap.DocChunkEntities : graphType.includes('DocumentChunk') - ? queryMap.DocChunks - : graphType.includes('Entities') - ? queryMap.Entities - : ''; + ? queryMap.DocChunks + : graphType.includes('Entities') + ? queryMap.Entities + : ''; // fit graph to original position const handleZoomToFit = () => { @@ -136,10 +136,10 @@ const GraphViewModal: React.FunctionComponent = ({ const nodeRelationshipData = viewPoint === graphLabels.showGraphView ? await graphQueryAPI( - userCredentials as UserCredentials, - graphQuery, - selectedRows?.map((f) => f.name) - ) + userCredentials as UserCredentials, + graphQuery, + selectedRows?.map((f) => f.name) + ) : await graphQueryAPI(userCredentials as UserCredentials, graphQuery, [inspectedName ?? '']); return nodeRelationshipData; } catch (error: any) { @@ -152,9 +152,11 @@ const GraphViewModal: React.FunctionComponent = ({ try { const result = await fetchData(); if (result && result.data.data.nodes.length > 0) { - const neoNodes = result.data.data.nodes.map((f: Node) => f); - const neoRels = result.data.data.relationships.map((f: Relationship) => f); + const neoNodes = result.data.data.nodes.map((f: Node) => f).filter((node: ExtendedNode) => node.labels.length === 1); + const nodeIds = new Set(neoNodes.map((node:any) => node.element_id)); + const neoRels = result.data.data.relationships.map((f: Relationship) => f).filter((rel: any) => nodeIds.has(rel.end_node_element_id) && nodeIds.has(rel.start_node_element_id)); const { finalNodes, finalRels, schemeVal } = processGraphData(neoNodes, neoRels); + if (mode === 'refreshMode') { initGraph(graphType, finalNodes, finalRels, schemeVal); } else { @@ -246,8 +248,8 @@ const GraphViewModal: React.FunctionComponent = ({ match && viewPoint === graphLabels.showGraphView ? 100 : match && viewPoint !== graphLabels.showGraphView - ? 50 - : graphLabels.nodeSize, + ? 50 + : graphLabels.nodeSize, }; }); // deactivating any active relationships @@ -356,8 +358,8 @@ const GraphViewModal: React.FunctionComponent = ({ isActive && viewPoint === graphLabels.showGraphView ? 100 : isActive && viewPoint !== graphLabels.showGraphView - ? 50 - : graphLabels.nodeSize, + ? 50 + : graphLabels.nodeSize, }; }); // deactivating any active relationships @@ -421,7 +423,7 @@ const GraphViewModal: React.FunctionComponent = ({ graphType={graphType} loading={loading} handleChange={handleCheckboxChange} - isgds={allNodes.some((n) => n.labels.includes('__Community__'))} + {...getCheckboxConditions(allNodes)} /> )} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 9631335a9..1e3a6f7aa 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -257,6 +257,8 @@ export interface CheckboxSectionProps { loading: boolean; handleChange: (graph: GraphType) => void; isgds: boolean; + isDocChunk: boolean; + isEntity:boolean; } export interface fileName { diff --git a/frontend/src/utils/Utils.ts b/frontend/src/utils/Utils.ts index 97e7bda07..0f5a33972 100644 --- a/frontend/src/utils/Utils.ts +++ b/frontend/src/utils/Utils.ts @@ -178,6 +178,7 @@ export const processGraphData = (neoNodes: ExtendedNode[], neoRels: ExtendedRela }; }); const finalNodes = newNodes.flat(); + // Process relationships const newRels: Relationship[] = neoRels.map((relations: any) => { return { id: relations.element_id, @@ -213,6 +214,7 @@ export const filterData = ( (type) => type !== 'Document' && type !== 'Chunk' && type !== '__Community__' ); // Only Document + Chunk + // const processedEntities = entityTypes.flatMap(item => item.includes(',') ? item.split(',') : item); if ( graphType.includes('DocumentChunk') && !graphType.includes('Entities') && @@ -245,6 +247,7 @@ export const filterData = ( nodeIds.has(rel.to) ); filteredScheme = Object.fromEntries(entityTypes.map((key) => [key, scheme[key]])) as Scheme; + console.log('labels', entityNodes); // Only Communities } else if ( graphType.includes('Communities') && @@ -336,6 +339,7 @@ export const filterData = ( filteredNodes = allNodes; filteredRelations = allRelationships; filteredScheme = scheme; + console.log('entity', filteredScheme); } return { filteredNodes, filteredRelations, filteredScheme }; }; @@ -469,3 +473,10 @@ export function isAllowedHost(url: string, allowedHosts: string[]) { return false; } } + +export const getCheckboxConditions = (allNodes: ExtendedNode[]) => { + const isDocChunk = allNodes.some((n) => n.labels?.includes('Document')); + const isEntity = allNodes.some((n) => !n.labels?.includes('Document') || !n.labels?.includes('Chunk')); + const isgds = allNodes.some((n) => n.labels?.includes('__Community__')); + return { isDocChunk, isEntity, isgds }; +}; From d6fb4394b39b89623d1736d6123cbb12c3751ff2 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Wed, 25 Sep 2024 14:25:52 +0000 Subject: [PATCH 11/22] payload changes --- frontend/src/components/ChatBot/ChatInfoModal.tsx | 2 +- frontend/src/components/ChatBot/Chatbot.tsx | 10 +++++----- frontend/src/services/ChunkEntitiesInfo.ts | 8 ++++---- frontend/src/types.ts | 8 ++++---- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index c32806599..86991f449 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -87,7 +87,7 @@ const ChatInfoModal: React.FC = ({ (async () => { setLoading(true); try { - const response = await chunkEntitiesAPI(userCredentials as UserCredentials, userCredentials?.database, chunk_ids.join(','),entities_ids.join(','), + const response = await chunkEntitiesAPI(userCredentials as UserCredentials, userCredentials?.database, chunk_ids,entities_ids, mode, ); if (response.data.status === 'Failure') { diff --git a/frontend/src/components/ChatBot/Chatbot.tsx b/frontend/src/components/ChatBot/Chatbot.tsx index 54b33dd2b..7a95af322 100644 --- a/frontend/src/components/ChatBot/Chatbot.tsx +++ b/frontend/src/components/ChatBot/Chatbot.tsx @@ -42,14 +42,14 @@ const Chatbot: FC = (props) => { const [sourcesModal, setSourcesModal] = useState([]); const [modelModal, setModelModal] = useState(''); const [responseTime, setResponseTime] = useState(0); - const [chunkModal, setChunkModal] = useState([]); + const [chunkModal, setChunkModal] = useState([]); const [tokensUsed, setTokensUsed] = useState(0); const [cypherQuery, setcypherQuery] = useState(''); const [copyMessageId, setCopyMessageId] = useState(null); const [chatsMode, setChatsMode] = useState(chatModeLables.graph_vector_fulltext); const [graphEntitites, setgraphEntitites] = useState<[]>([]); const [messageError, setmessageError] = useState(''); - const [entitiesModal, setEntitiesModal] = useState([]); + const [entitiesModal, setEntitiesModal] = useState([]); const [value, copy] = useCopyToClipboard(); const { speak, cancel } = useSpeechSynthesis({ @@ -83,7 +83,7 @@ const Chatbot: FC = (props) => { reply: string; sources?: string[]; model?: string; - chunk_ids?: chunk[]; + chunk_ids?: string[]; total_tokens?: number; response_time?: number; speaking?: boolean; @@ -92,7 +92,7 @@ const Chatbot: FC = (props) => { cypher_query?: string; graphonly_entities?: []; error?: string; - entitiysearchonly_entities?: chunk[]; + entitiysearchonly_entities?: string[]; }, index = 0 ) => { @@ -218,7 +218,7 @@ const Chatbot: FC = (props) => { reply: chatbotReply, sources: chatSources, model: chatModel, - chunk_ids: chatChunks.map((chunk: chunk) => chunk.id), // Extract chunk IDs here + chunk_ids: chatnodedetails, // Extract chunk IDs here total_tokens: chatTokensUsed, response_time: chatTimeTaken, speaking: false, diff --git a/frontend/src/services/ChunkEntitiesInfo.ts b/frontend/src/services/ChunkEntitiesInfo.ts index 79502dd21..eff7094bc 100644 --- a/frontend/src/services/ChunkEntitiesInfo.ts +++ b/frontend/src/services/ChunkEntitiesInfo.ts @@ -4,8 +4,8 @@ import api from '../API/Index'; const chunkEntitiesAPI = async ( userCredentials: UserCredentials, database: string = 'neo4j', - chunk_ids: string, - entities:string, + chunk_ids: (string)[], + entities:(string)[], mode: string, ) => { try { @@ -14,8 +14,8 @@ const chunkEntitiesAPI = async ( formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); formData.append('database', database); - formData.append('nodedetails', chunk_ids); - formData.append('entities', entities); + formData.append('nodedetails', JSON.stringify(chunk_ids)); + formData.append('entities', JSON.stringify(entities)); formData.append('mode', mode); const response: ChatInfo_APIResponse = await api.post(`/chunk_entities`, formData, { diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 475fdcd31..f918d271c 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -216,7 +216,7 @@ export interface Messages { isLoading?: boolean; response_time?: number; nodedetails?: nodeDetailsProps[]; - chunk_ids?: chunk[]; + chunk_ids?: string[]; total_tokens?: number; speaking?: boolean; copying?: boolean; @@ -224,7 +224,7 @@ export interface Messages { cypher_query?: string; graphonly_entities?: []; error?: string; - entities?: chunk[]; + entities?: string[]; } export type ChatbotProps = { @@ -415,13 +415,13 @@ export interface chatInfoMessage extends Partial { sources: string[]; model: string; response_time: number; - chunk_ids: chunk[]; + chunk_ids: string[]; total_tokens: number; mode: string; cypher_query?: string; graphonly_entities: []; error: string; - entities_ids:chunk[]; + entities_ids:string[]; } export interface eventResponsetypes extends Omit { From da71b895f8c5e4252b849a308ade8bca1c1694c8 Mon Sep 17 00:00:00 2001 From: vasanthasaikalluri <165021735+vasanthasaikalluri@users.noreply.github.com> Date: Wed, 25 Sep 2024 15:12:15 +0000 Subject: [PATCH 12/22] added nodetails properties --- backend/score.py | 5 ++--- backend/src/chunkid_entities.py | 3 +++ backend/src/shared/constants.py | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/score.py b/backend/score.py index 001abb07b..dc309741f 100644 --- a/backend/score.py +++ b/backend/score.py @@ -318,10 +318,9 @@ async def chat_bot(uri=Form(),model=Form(None),userName=Form(), password=Form(), gc.collect() @app.post("/chunk_entities") -async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), node_ids=Form(None),type=Form()): +async def chunk_entities(uri=Form(),userName=Form(), password=Form(), database=Form(), nodedetails=Form(None),entities=Form(),mode=Form()): try: - logging.info(f"URI: {uri}, Username: {userName}, node_ids: {node_ids}") - result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,node_ids=node_ids,type=type) + result = await asyncio.to_thread(get_entities_from_chunkids,uri=uri, username=userName, password=password, database=database,nodedetails=nodedetails,entities=entities,mode=mode) json_obj = {'api_name':'chunk_entities','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} logger.log_struct(json_obj, "INFO") return create_api_response('Success',data=result) diff --git a/backend/src/chunkid_entities.py b/backend/src/chunkid_entities.py index 966af6118..8bb9c2198 100644 --- a/backend/src/chunkid_entities.py +++ b/backend/src/chunkid_entities.py @@ -173,6 +173,9 @@ def get_entities_from_chunkids(uri, username, password, database ,nodedetails,en driver = get_graphDB_driver(uri, username, password,database) default_response = {"nodes": list(),"relationships": list(),"chunk_data": list(),"community_data": list(),} + nodedetails = json.loads(nodedetails) + entities = json.loads(entities) + if mode == CHAT_GLOBAL_VECTOR_FULLTEXT_MODE: if "communitydetails" in nodedetails and nodedetails["communitydetails"]: diff --git a/backend/src/shared/constants.py b/backend/src/shared/constants.py index 82f32d214..e2e2a85a0 100644 --- a/backend/src/shared/constants.py +++ b/backend/src/shared/constants.py @@ -248,8 +248,8 @@ VECTOR_GRAPH_SEARCH_ENTITY_LIMIT = 25 VECTOR_GRAPH_SEARCH_EMBEDDING_MIN_MATCH = 0.3 VECTOR_GRAPH_SEARCH_EMBEDDING_MAX_MATCH = 0.9 -VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MINMAX_CASE = 20 -VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MAX_CASE = 40 +VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MINMAX_CASE = 10 +VECTOR_GRAPH_SEARCH_ENTITY_LIMIT_MAX_CASE = 25 VECTOR_GRAPH_SEARCH_QUERY_PREFIX = """ WITH node as chunk, score From 86d2b8ecdbeec6827547be768bfa33fcdf17bc8c Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Wed, 25 Sep 2024 18:23:59 +0000 Subject: [PATCH 13/22] payload new changes --- .../src/components/ChatBot/ChatInfoModal.tsx | 20 +++++++-------- frontend/src/components/ChatBot/Chatbot.tsx | 25 +++++++------------ frontend/src/services/ChunkEntitiesInfo.ts | 6 ++--- frontend/src/types.ts | 12 ++++----- 4 files changed, 28 insertions(+), 35 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index 86991f449..b78f886ee 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -38,7 +38,7 @@ const ChatInfoModal: React.FC = ({ model, total_tokens, response_time, - chunk_ids, + nodeDetails, mode, cypher_query, graphonly_entities, @@ -87,7 +87,7 @@ const ChatInfoModal: React.FC = ({ (async () => { setLoading(true); try { - const response = await chunkEntitiesAPI(userCredentials as UserCredentials, userCredentials?.database, chunk_ids,entities_ids, + const response = await chunkEntitiesAPI(userCredentials as UserCredentials, userCredentials?.database, nodeDetails, entities_ids, mode, ); if (response.data.status === 'Failure') { @@ -125,13 +125,13 @@ const ChatInfoModal: React.FC = ({ setChunks( chunksData .map((chunk: any) => { - const chunkScore = chunk_ids.find((chunkdetail) => chunkdetail.id === chunk.id); - return ( - { - ...chunk, - score: chunkScore?.score, - } ?? [] - ); + const chunkScore = nodeDetails?.chunkdetails?.find((c: any) => + c.id + === chunk.id); + return { + ...chunk, + score: chunkScore?.score + }; }) .sort((a: any, b: any) => b.score - a.score) ); @@ -145,7 +145,7 @@ const ChatInfoModal: React.FC = ({ () => { setcopiedText(false); }; - }, [chunk_ids, mode, error]); + }, [nodeDetails, mode, error]); const onChangeTabs = (tabId: number) => { setActiveTab(tabId); diff --git a/frontend/src/components/ChatBot/Chatbot.tsx b/frontend/src/components/ChatBot/Chatbot.tsx index 7a95af322..013bd8db4 100644 --- a/frontend/src/components/ChatBot/Chatbot.tsx +++ b/frontend/src/components/ChatBot/Chatbot.tsx @@ -7,7 +7,7 @@ import { SpeakerXMarkIconOutline, } from '@neo4j-ndl/react/icons'; import ChatBotAvatar from '../../assets/images/chatbot-ai.png'; -import { ChatbotProps, CustomFile, UserCredentials, chunk, entity } from '../../types'; +import { ChatbotProps, CustomFile, UserCredentials,nodeDetailsProps } from '../../types'; import { useCredentials } from '../../context/UserCredentials'; import { chatBotAPI } from '../../services/QnaAPI'; import { v4 as uuidv4 } from 'uuid'; @@ -42,7 +42,6 @@ const Chatbot: FC = (props) => { const [sourcesModal, setSourcesModal] = useState([]); const [modelModal, setModelModal] = useState(''); const [responseTime, setResponseTime] = useState(0); - const [chunkModal, setChunkModal] = useState([]); const [tokensUsed, setTokensUsed] = useState(0); const [cypherQuery, setcypherQuery] = useState(''); const [copyMessageId, setCopyMessageId] = useState(null); @@ -50,6 +49,7 @@ const Chatbot: FC = (props) => { const [graphEntitites, setgraphEntitites] = useState<[]>([]); const [messageError, setmessageError] = useState(''); const [entitiesModal, setEntitiesModal] = useState([]); + const [nodeDetailsModal, setNodeDetailsModal] = useState({}); const [value, copy] = useCopyToClipboard(); const { speak, cancel } = useSpeechSynthesis({ @@ -83,7 +83,6 @@ const Chatbot: FC = (props) => { reply: string; sources?: string[]; model?: string; - chunk_ids?: string[]; total_tokens?: number; response_time?: number; speaking?: boolean; @@ -93,6 +92,7 @@ const Chatbot: FC = (props) => { graphonly_entities?: []; error?: string; entitiysearchonly_entities?: string[]; + nodeDetails?: nodeDetailsProps; }, index = 0 ) => { @@ -114,7 +114,6 @@ const Chatbot: FC = (props) => { isLoading: true, sources: response?.sources, model: response?.model, - chunks: response?.chunk_ids, total_tokens: response.total_tokens, response_time: response?.response_time, speaking: false, @@ -124,6 +123,7 @@ const Chatbot: FC = (props) => { graphonly_entities: response?.graphonly_entities, error: response.error, entitiysearchonly_entities: response.entitiysearchonly_entities, + nodeDetails: response?.nodeDetails }, ]); } else { @@ -137,7 +137,6 @@ const Chatbot: FC = (props) => { lastmsg.isLoading = false; lastmsg.sources = response?.sources; lastmsg.model = response?.model; - lastmsg.chunk_ids = response?.chunk_ids; lastmsg.total_tokens = response?.total_tokens; lastmsg.response_time = response?.response_time; lastmsg.speaking = false; @@ -147,6 +146,7 @@ const Chatbot: FC = (props) => { lastmsg.graphonly_entities = response.graphonly_entities; lastmsg.error = response.error; lastmsg.entities = response.entitiysearchonly_entities; + lastmsg.nodeDetails = response?.nodeDetails; return msgs.map((msg, index) => { if (index === msgs.length - 1) { return lastmsg; @@ -174,7 +174,6 @@ const Chatbot: FC = (props) => { let chatSources; let chatModel; let chatnodedetails; - let chatChunks; let chatTimeTaken; let chatTokensUsed; let chatingMode; @@ -202,10 +201,6 @@ const Chatbot: FC = (props) => { chatSources = chatresponse?.data?.data?.info.sources; chatModel = chatresponse?.data?.data?.info.model; chatnodedetails = chatresponse?.data?.data?.info.nodedetails; - chatChunks = chatnodedetails?.chunkdetails?.map((chunk: chunk) => ({ - id: chunk.id, - score: chunk.score, - })); chatTokensUsed = chatresponse?.data?.data?.info.total_tokens; chatTimeTaken = chatresponse?.data?.data?.info.response_time; chatingMode = chatresponse?.data?.data?.info?.mode; @@ -218,7 +213,6 @@ const Chatbot: FC = (props) => { reply: chatbotReply, sources: chatSources, model: chatModel, - chunk_ids: chatnodedetails, // Extract chunk IDs here total_tokens: chatTokensUsed, response_time: chatTimeTaken, speaking: false, @@ -228,7 +222,8 @@ const Chatbot: FC = (props) => { graphonly_entities, error, entitiysearchonly_entities, - chatEntities + chatEntities, + nodeDetails: chatnodedetails }; simulateTypingEffect(finalbotReply); } catch (error) { @@ -375,9 +370,6 @@ const Chatbot: FC = (props) => { setModelModal(chat.model ?? ''); setSourcesModal(chat.sources ?? []); setResponseTime(chat.response_time ?? 0); - setChunkModal( - chat.mode === 'entity search+vector' ? chat.entities ?? [] : chat.chunk_ids ?? [] - ); setTokensUsed(chat.total_tokens ?? 0); setcypherQuery(chat.cypher_query ?? ''); setShowInfoModal(true); @@ -385,6 +377,7 @@ const Chatbot: FC = (props) => { setgraphEntitites(chat.graphonly_entities ?? []); setEntitiesModal(chat.entities ?? []); setmessageError(chat.error ?? ''); + setNodeDetailsModal(chat.nodeDetails ?? {}) }} > {' '} @@ -483,7 +476,6 @@ const Chatbot: FC = (props) => { = (props) => { cypher_query={cypherQuery} graphonly_entities={graphEntitites} error={messageError} + nodeDetails={nodeDetailsModal} /> diff --git a/frontend/src/services/ChunkEntitiesInfo.ts b/frontend/src/services/ChunkEntitiesInfo.ts index eff7094bc..39d9f8c2e 100644 --- a/frontend/src/services/ChunkEntitiesInfo.ts +++ b/frontend/src/services/ChunkEntitiesInfo.ts @@ -1,10 +1,10 @@ -import { ChatInfo_APIResponse, UserCredentials } from '../types'; +import { ChatInfo_APIResponse, nodeDetailsProps, UserCredentials } from '../types'; import api from '../API/Index'; const chunkEntitiesAPI = async ( userCredentials: UserCredentials, database: string = 'neo4j', - chunk_ids: (string)[], + nodeDetails: (nodeDetailsProps), entities:(string)[], mode: string, ) => { @@ -14,7 +14,7 @@ const chunkEntitiesAPI = async ( formData.append('userName', userCredentials?.userName ?? ''); formData.append('password', userCredentials?.password ?? ''); formData.append('database', database); - formData.append('nodedetails', JSON.stringify(chunk_ids)); + formData.append('nodedetails', JSON.stringify(nodeDetails)); formData.append('entities', JSON.stringify(entities)); formData.append('mode', mode); diff --git a/frontend/src/types.ts b/frontend/src/types.ts index f918d271c..7e04e8597 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -201,7 +201,7 @@ export interface Source { source_name: string; start_time?: string; } -export interface chunk { +export interface ChunkDetail { id: string; score: number; } @@ -215,7 +215,7 @@ export interface Messages { model?: string; isLoading?: boolean; response_time?: number; - nodedetails?: nodeDetailsProps[]; + nodeDetails?: nodeDetailsProps; chunk_ids?: string[]; total_tokens?: number; speaking?: boolean; @@ -415,13 +415,13 @@ export interface chatInfoMessage extends Partial { sources: string[]; model: string; response_time: number; - chunk_ids: string[]; total_tokens: number; mode: string; cypher_query?: string; graphonly_entities: []; error: string; entities_ids:string[]; + nodeDetails: nodeDetailsProps; } export interface eventResponsetypes extends Omit { @@ -671,7 +671,7 @@ export type CommunitiesProps = { communities: Community[]; }; -export interface entity { +export interface entity{ id: string; score: number; }; @@ -681,8 +681,8 @@ export interface community { score: number; } -export type nodeDetailsProps = { - chunkDetails?: chunk[], +export interface nodeDetailsProps { + chunkdetails?: ChunkDetail[], entitydetails?: entity[], communitydetails?: community[] } From fdd9909e9d0f5f493f5e0f9837efe27545f32246 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Thu, 26 Sep 2024 06:57:08 +0000 Subject: [PATCH 14/22] communities check --- .../src/components/ChatBot/ChatInfoModal.tsx | 57 ++++++++++++------- 1 file changed, 38 insertions(+), 19 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index b78f886ee..f72888996 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -47,7 +47,15 @@ const ChatInfoModal: React.FC = ({ }) => { const { breakpoints } = tokens; const isTablet = useMediaQuery(`(min-width:${breakpoints.xs}) and (max-width: ${breakpoints.lg})`); - const [activeTab, setActiveTab] = useState(error?.length ? 10 : mode === chatModeLables.graph ? 4 : 3); + const [activeTab, setActiveTab] = useState( + error?.length + ? 10 + : mode === chatModeLables.global_vector + ? 7 + : mode === chatModeLables.graph + ? 4 + : 3 + ); const [infoEntities, setInfoEntities] = useState([]); const [communities, setCommunities] = useState([]); const [loading, setLoading] = useState(false); @@ -173,22 +181,33 @@ const ChatInfoModal: React.FC = ({ {error} ) : ( - {mode != chatModeLables.graph ? Sources used : <>} - {mode != chatModeLables.graph ? Chunks : <>} - {mode === chatModeLables.graph_vector || - mode === chatModeLables.graph || - mode === chatModeLables.graph_vector_fulltext || - mode === chatModeLables.entity_vector ? ( - Top Entities used - ) : ( - <> - )} - {mode === chatModeLables.graph && cypher_query?.trim()?.length ? ( - Generated Cypher Query + {mode === chatModeLables.global_vector ? ( + // Only show the Communities tab if mode is global + Communities ) : ( - <> + <> + {mode != chatModeLables.graph ? Sources used : <>} + {mode != chatModeLables.graph ? Chunks : <>} + {mode === chatModeLables.graph_vector || + mode === chatModeLables.graph || + mode === chatModeLables.graph_vector_fulltext || + mode === chatModeLables.entity_vector ? ( + Top Entities used + ) : ( + <> + )} + {mode === chatModeLables.graph && cypher_query?.trim()?.length ? ( + Generated Cypher Query + ) : ( + <> + )} + {mode === chatModeLables.entity_vector ? ( + Communities + ) : ( + <> + )} + )} - {mode === chatModeLables.entity_vector ? Communities : <>} )} @@ -196,6 +215,9 @@ const ChatInfoModal: React.FC = ({ + + + = ({ infoEntities={infoEntities} /> - - - = ({ className='min-h-40' /> - {mode === chatModeLables.entity_vector ? ( + {mode === chatModeLables.entity_vector || mode === chatModeLables.global_vector ? ( From a1518a3227a22597a4095cc5bffe6717746c598d Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Thu, 26 Sep 2024 07:13:23 +0000 Subject: [PATCH 15/22] communities selecetion check --- frontend/src/components/ChatBot/ChatInfoModal.tsx | 8 ++++---- frontend/src/components/ChatBot/Communities.tsx | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatInfoModal.tsx b/frontend/src/components/ChatBot/ChatInfoModal.tsx index f72888996..89a4df49e 100644 --- a/frontend/src/components/ChatBot/ChatInfoModal.tsx +++ b/frontend/src/components/ChatBot/ChatInfoModal.tsx @@ -187,7 +187,7 @@ const ChatInfoModal: React.FC = ({ ) : ( <> {mode != chatModeLables.graph ? Sources used : <>} - {mode != chatModeLables.graph ? Chunks : <>} + {mode != chatModeLables.graph ? Chunks : <>} {mode === chatModeLables.graph_vector || mode === chatModeLables.graph || mode === chatModeLables.graph_vector_fulltext || @@ -215,9 +215,6 @@ const ChatInfoModal: React.FC = ({ - - - = ({ infoEntities={infoEntities} /> + + + = ({ loading, communities }) => { + console.log('communities', communities); return ( <> {loading ? ( From 3668a18dea03c1590b1eedf94ac20d467c3d5933 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:57:23 +0000 Subject: [PATCH 16/22] enable communities --- frontend/src/components/ChatBot/ChatModeToggle.tsx | 4 ++-- .../GraphEnhancementDialog/PostProcessingCheckList/index.tsx | 2 +- frontend/src/utils/Constants.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatModeToggle.tsx b/frontend/src/components/ChatBot/ChatModeToggle.tsx index 187c37178..c2666b0d5 100644 --- a/frontend/src/components/ChatBot/ChatModeToggle.tsx +++ b/frontend/src/components/ChatBot/ChatModeToggle.tsx @@ -20,7 +20,7 @@ export default function ChatModeToggle({ disableBackdrop?: boolean; }) { const { setchatMode, chatMode, postProcessingTasks, selectedRows } = useFileContext(); - const isCommunityAllowed = postProcessingTasks.includes('create_communities'); + const isCommunityAllowed = postProcessingTasks.includes('enable_communities'); const { isGdsActive } = useCredentials(); useEffect(() => { @@ -29,7 +29,7 @@ export default function ChatModeToggle({ } else { setchatMode(chatModeLables.graph_vector_fulltext); } - }, [selectedRows]); + }, [selectedRows.length]); const memoizedChatModes = useMemo(() => { return isGdsActive && isCommunityAllowed diff --git a/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx b/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx index 28114ced7..b48b6c6a6 100644 --- a/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx +++ b/frontend/src/components/Popups/GraphEnhancementDialog/PostProcessingCheckList/index.tsx @@ -20,7 +20,7 @@ export default function PostProcessingCheckList() { {POST_PROCESSING_JOBS.map((job, idx) => { - const isCreateCommunities = job.title === 'create_communities'; + const isCreateCommunities = job.title === 'enable_communities'; return ( Date: Thu, 26 Sep 2024 13:03:58 +0000 Subject: [PATCH 17/22] removed the selected prop --- frontend/src/components/Layout/SideNav.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/components/Layout/SideNav.tsx b/frontend/src/components/Layout/SideNav.tsx index bdc43c8be..ac05cd88b 100644 --- a/frontend/src/components/Layout/SideNav.tsx +++ b/frontend/src/components/Layout/SideNav.tsx @@ -203,7 +203,6 @@ const SideNav: React.FC = ({ {!isChatModalOpen && ( { setchatModeAnchor(e.currentTarget); setshowChatMode(true); From 703ee0c376b426e22deb53ab04fbe9bddaf67d0d Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Thu, 26 Sep 2024 13:42:46 +0000 Subject: [PATCH 18/22] enable communities label change --- backend/score.py | 2 +- frontend/src/context/UsersFiles.tsx | 2 +- frontend/src/utils/Constants.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/score.py b/backend/score.py index 63a26ec5b..b5d32174b 100644 --- a/backend/score.py +++ b/backend/score.py @@ -271,7 +271,7 @@ async def post_processing(uri=Form(), userName=Form(), password=Form(), database await asyncio.to_thread(create_entity_embedding, graph) json_obj = {'api_name': 'post_processing/create_entity_embedding', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} logging.info(f'Entity Embeddings created') - if "create_communities" in tasks: + if "enable_communities" in tasks: model = "openai-gpt-4o" await asyncio.to_thread(create_communities, uri, userName, password, database,model) josn_obj = {'api_name': 'post_processing/create_communities', 'db_url': uri, 'logging_time': formatted_time(datetime.now(timezone.utc))} diff --git a/frontend/src/context/UsersFiles.tsx b/frontend/src/context/UsersFiles.tsx index fbab1b719..5d5f737d8 100644 --- a/frontend/src/context/UsersFiles.tsx +++ b/frontend/src/context/UsersFiles.tsx @@ -68,7 +68,7 @@ const FileContextProvider: FC = ({ children }) => { 'materialize_text_chunk_similarities', 'enable_hybrid_search_and_fulltext_search_in_bloom', 'materialize_entity_similarities', - 'create_communities', + 'enable_communities', ]); const [processedCount, setProcessedCount] = useState(0); useEffect(() => { diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index 5a5b9c2fa..81587f83e 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -233,7 +233,7 @@ export const POST_PROCESSING_JOBS: { title: string; description: string }[] = [ performing similarity-based searches.`, }, { - title: 'Enable community creation across entities to use GraphRAG capabilities both local and global search. ', + title: 'enable_communities', description: 'Enable community creation across entities to use GraphRAG capabilities both local and global search.', }, ]; From beb1ef055172817906029af5780f01eaa0d916e7 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Mon, 30 Sep 2024 09:41:56 +0000 Subject: [PATCH 19/22] communities name change --- .../components/ChatBot/CommunitiesInfo.tsx | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 frontend/src/components/ChatBot/CommunitiesInfo.tsx diff --git a/frontend/src/components/ChatBot/CommunitiesInfo.tsx b/frontend/src/components/ChatBot/CommunitiesInfo.tsx new file mode 100644 index 000000000..9ab23f9b2 --- /dev/null +++ b/frontend/src/components/ChatBot/CommunitiesInfo.tsx @@ -0,0 +1,40 @@ +import { Box, LoadingSpinner, Flex, Typography } from '@neo4j-ndl/react'; +import { FC } from 'react'; +import ReactMarkdown from 'react-markdown'; +import { CommunitiesProps } from '../../types'; + +const CommunitiesInfo: FC = ({ loading, communities }) => { + return ( + <> + {loading ? ( + + + + ) : communities?.length > 0 ? ( +
+
    + {communities.map((community, index) => ( +
  • +
    + + ID : + {community.id} + + + Score : + {community.score && ({community.score})} + + {community.summary} +
    +
    • + ))} +
+
+ ) : ( + No Communities Found + )} + + ); +}; + +export default CommunitiesInfo; From 64d7445ef3e42a9e051b13d49674fdb56309af0e Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Mon, 30 Sep 2024 10:59:45 +0000 Subject: [PATCH 20/22] cred check --- frontend/src/components/ChatBot/ChatModeToggle.tsx | 6 +++--- frontend/src/components/Content.tsx | 4 +++- frontend/src/utils/Constants.ts | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatModeToggle.tsx b/frontend/src/components/ChatBot/ChatModeToggle.tsx index 466516782..e2139ae42 100644 --- a/frontend/src/components/ChatBot/ChatModeToggle.tsx +++ b/frontend/src/components/ChatBot/ChatModeToggle.tsx @@ -21,7 +21,7 @@ export default function ChatModeToggle({ }) { const { setchatMode, chatMode, postProcessingTasks, selectedRows } = useFileContext(); const isCommunityAllowed = postProcessingTasks.includes('enable_communities'); - const { isGdsActive } = useCredentials(); + const { isGdsActive, userCredentials } = useCredentials(); useEffect(() => { // If rows are selected, the mode is valid (either vector or graph+vector) @@ -30,7 +30,7 @@ export default function ChatModeToggle({ setchatMode(chatModeLables.graph_vector); } } - }, [selectedRows.length, chatMode, setchatMode]); + }, [selectedRows.length, chatMode, setchatMode, userCredentials]); const memoizedChatModes = useMemo(() => { return isGdsActive && isCommunityAllowed ? chatModes @@ -86,7 +86,7 @@ export default function ChatModeToggle({ if (!selectedRows.length && !chatMode) { setchatMode(chatModeLables.graph_vector_fulltext); } - }, [setchatMode, selectedRows.length, chatMode]); + }, [setchatMode, selectedRows.length, chatMode, userCredentials]); return ( = ({ queue, processedCount, setProcessedCount, + setchatMode } = useFileContext(); const [viewPoint, setViewPoint] = useState<'tableView' | 'showGraphView' | 'chatInfoView'>('tableView'); const [showDeletePopUp, setshowDeletePopUp] = useState(false); @@ -541,6 +542,7 @@ const Content: React.FC = ({ setSelectedNodes([]); setSelectedRels([]); setClearHistoryData(true); + setchatMode(chatModeLables.graph_vector_fulltext) }; const retryHandler = async (filename: string, retryoption: string) => { diff --git a/frontend/src/utils/Constants.ts b/frontend/src/utils/Constants.ts index a9a41c0a2..c550bb45a 100644 --- a/frontend/src/utils/Constants.ts +++ b/frontend/src/utils/Constants.ts @@ -68,7 +68,7 @@ export const chatModeLables = { fulltext: 'fulltext', graph_vector_fulltext: 'graph+vector+fulltext', entity_vector: 'entity search+vector', - unavailableChatMode: 'Chat mode is unavailable when rows are selected', + unavailableChatMode: 'Chat mode is unavailable when files are selected', selected: 'Selected', global_vector: 'global search+vector+fulltext', }; From ee543876325b17ca309b9f3a01c5b46403083577 Mon Sep 17 00:00:00 2001 From: Prakriti Solankey <156313631+prakriti-solankey@users.noreply.github.com> Date: Thu, 3 Oct 2024 13:47:29 +0000 Subject: [PATCH 21/22] tooltip --- backend/score.py | 1 + frontend/src/components/FileTable.tsx | 78 +++++++++++++++------- frontend/src/components/Layout/SideNav.tsx | 2 +- 3 files changed, 55 insertions(+), 26 deletions(-) diff --git a/backend/score.py b/backend/score.py index 8887d6450..f932946f6 100644 --- a/backend/score.py +++ b/backend/score.py @@ -426,6 +426,7 @@ async def upload_large_file_into_chunks(file:UploadFile = File(...), chunkNumber elapsed_time = end - start json_obj = {'api_name':'upload','db_url':uri, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'} logger.log_struct(json_obj, "INFO") + # result['elapsed_api_time'] = f'{elapsed_time:.2f}' if int(chunkNumber) == int(totalChunks): return create_api_response('Success',data=result, message='Source Node Created Successfully') else: diff --git a/frontend/src/components/FileTable.tsx b/frontend/src/components/FileTable.tsx index 4b6d4bb7e..97f69ac3f 100644 --- a/frontend/src/components/FileTable.tsx +++ b/frontend/src/components/FileTable.tsx @@ -7,7 +7,9 @@ import { ProgressBar, StatusIndicator, TextLink, + Tip, Typography, + useCopyToClipboard, } from '@neo4j-ndl/react'; import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; import { @@ -35,7 +37,7 @@ import { } from '../utils/Utils'; import { SourceNode, CustomFile, FileTableProps, UserCredentials, statusupdate, ChildRef } from '../types'; import { useCredentials } from '../context/UserCredentials'; -import { ArrowPathIconSolid, MagnifyingGlassCircleIconSolid } from '@neo4j-ndl/react/icons'; +import { ArrowPathIconSolid, ClipboardDocumentIconOutline, MagnifyingGlassCircleIconSolid } from '@neo4j-ndl/react/icons'; import CustomProgressBar from './UI/CustomProgressBar'; import subscribe from '../services/PollingAPI'; import { triggerStatusUpdateAPI } from '../services/ServerSideStatusUpdateAPI'; @@ -61,6 +63,7 @@ const FileTable = forwardRef((props, ref) => { const [fileSourceFilter, setFileSourceFilter] = useState(''); const [llmtypeFilter, setLLmtypeFilter] = useState(''); const skipPageResetRef = useRef(false); + const [value, copy] = useCopyToClipboard(); const tableRef = useRef(null); @@ -72,6 +75,10 @@ const FileTable = forwardRef((props, ref) => { showErrorToast(`${fileName} Failed to process`); } ); + + const handleCopy = (message: string) => { + copy(message); + }; const columns = useMemo( () => [ { @@ -140,28 +147,49 @@ const FileTable = forwardRef((props, ref) => { cell: (info) => { if (info.getValue() != 'Processing') { return ( -
- - {info.getValue()} - {(info.getValue() === 'Completed' || info.getValue() === 'Failed' || info.getValue() === 'Cancelled') && - !isReadOnlyUser && ( - - onRetry(info?.row?.id as string)} - > - - - - )} -
+ + +
+ + + {info.getValue()} + + {(info.getValue() === 'Completed' || + info.getValue() === 'Failed' || + (info.getValue() === 'Cancelled' && !isReadOnlyUser)) && ( + + + onRetry(info?.row?.id as string)} + > + + + + )} +
+ + {info.row.original?.status === 'Failed' && ( + + handleCopy(info.row.original?.errorMessage ?? '')} + > + + + ) + } + ); } else if (info.getValue() === 'Processing' && info.row.original.processingProgress === undefined) { return ( @@ -656,8 +684,8 @@ const FileTable = forwardRef((props, ref) => { language: item?.language ?? '', processingProgress: item?.processed_chunk != undefined && - item?.total_chunks != undefined && - !isNaN(Math.floor((item?.processed_chunk / item?.total_chunks) * 100)) + item?.total_chunks != undefined && + !isNaN(Math.floor((item?.processed_chunk / item?.total_chunks) * 100)) ? Math.floor((item?.processed_chunk / item?.total_chunks) * 100) : undefined, access_token: item?.access_token ?? '', diff --git a/frontend/src/components/Layout/SideNav.tsx b/frontend/src/components/Layout/SideNav.tsx index e07979405..a371b3f32 100644 --- a/frontend/src/components/Layout/SideNav.tsx +++ b/frontend/src/components/Layout/SideNav.tsx @@ -113,7 +113,7 @@ const SideNav: React.FC = ({ + } From 0b6b04cb97a011f4214f7a18a1a52cca786dfe47 Mon Sep 17 00:00:00 2001 From: kartikpersistent <101251502+kartikpersistent@users.noreply.github.com> Date: Tue, 8 Oct 2024 05:24:34 +0000 Subject: [PATCH 22/22] fix: Copy Icon Theme Fix --- .../src/components/ChatBot/ChatModeToggle.tsx | 2 +- frontend/src/components/ChatBot/Chatbot.tsx | 10 ++-- .../components/ChatBot/CommunitiesInfo.tsx | 2 +- frontend/src/components/Content.tsx | 15 ++++- frontend/src/components/FileTable.tsx | 58 +++++++++++-------- .../src/components/Graph/GraphViewModal.tsx | 24 ++++---- 6 files changed, 64 insertions(+), 47 deletions(-) diff --git a/frontend/src/components/ChatBot/ChatModeToggle.tsx b/frontend/src/components/ChatBot/ChatModeToggle.tsx index e2139ae42..c84a3f718 100644 --- a/frontend/src/components/ChatBot/ChatModeToggle.tsx +++ b/frontend/src/components/ChatBot/ChatModeToggle.tsx @@ -8,7 +8,7 @@ import { capitalizeWithPlus } from '../../utils/Utils'; import { useCredentials } from '../../context/UserCredentials'; export default function ChatModeToggle({ menuAnchor, - closeHandler = () => { }, + closeHandler = () => {}, open, anchorPortal = true, disableBackdrop = false, diff --git a/frontend/src/components/ChatBot/Chatbot.tsx b/frontend/src/components/ChatBot/Chatbot.tsx index d2c97978b..550d04c97 100644 --- a/frontend/src/components/ChatBot/Chatbot.tsx +++ b/frontend/src/components/ChatBot/Chatbot.tsx @@ -324,7 +324,7 @@ const Chatbot: FC = (props) => { isElevated={true} className={`p-4 self-start ${isFullScreen ? 'max-w-[55%]' : ''} ${ chat.user === 'chatbot' ? 'n-bg-palette-neutral-bg-strong' : 'n-bg-palette-primary-bg-weak' - } `} + } `} subheader={ chat.user !== 'chatbot' && chat.mode?.length ? ( @@ -338,9 +338,9 @@ const Chatbot: FC = (props) => {
{chat.message}
@@ -427,7 +427,7 @@ const Chatbot: FC = (props) => { = ({ loading, communities }) => { Score : - {community.score && ({community.score})} + {community.score && {community.score}} {community.summary}
diff --git a/frontend/src/components/Content.tsx b/frontend/src/components/Content.tsx index a54ba686f..641faf6d0 100644 --- a/frontend/src/components/Content.tsx +++ b/frontend/src/components/Content.tsx @@ -18,7 +18,16 @@ import { postProcessing } from '../services/PostProcessing'; import { triggerStatusUpdateAPI } from '../services/ServerSideStatusUpdateAPI'; import useServerSideEvent from '../hooks/useSse'; import { useSearchParams } from 'react-router-dom'; -import { batchSize, buttonCaptions, chatModeLables, defaultLLM, largeFileSize, llms, RETRY_OPIONS, tooltips } from '../utils/Constants'; +import { + batchSize, + buttonCaptions, + chatModeLables, + defaultLLM, + largeFileSize, + llms, + RETRY_OPIONS, + tooltips, +} from '../utils/Constants'; import ButtonWithToolTip from './UI/ButtonWithToolTip'; import connectAPI from '../services/ConnectAPI'; import DropdownComponent from './Dropdown'; @@ -96,7 +105,7 @@ const Content: React.FC = ({ queue, processedCount, setProcessedCount, - setchatMode + setchatMode, } = useFileContext(); const [viewPoint, setViewPoint] = useState<'tableView' | 'showGraphView' | 'chatInfoView'>('tableView'); const [showDeletePopUp, setshowDeletePopUp] = useState(false); @@ -543,7 +552,7 @@ const Content: React.FC = ({ setSelectedNodes([]); setSelectedRels([]); setClearHistoryData(true); - setchatMode(chatModeLables.graph_vector_fulltext) + setchatMode(chatModeLables.graph_vector_fulltext); }; const retryHandler = async (filename: string, retryoption: string) => { diff --git a/frontend/src/components/FileTable.tsx b/frontend/src/components/FileTable.tsx index 97f69ac3f..4ae9cabc4 100644 --- a/frontend/src/components/FileTable.tsx +++ b/frontend/src/components/FileTable.tsx @@ -11,7 +11,7 @@ import { Typography, useCopyToClipboard, } from '@neo4j-ndl/react'; -import { forwardRef, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; +import { forwardRef, useContext, useEffect, useImperativeHandle, useMemo, useRef, useState } from 'react'; import { useReactTable, getCoreRowModel, @@ -37,7 +37,11 @@ import { } from '../utils/Utils'; import { SourceNode, CustomFile, FileTableProps, UserCredentials, statusupdate, ChildRef } from '../types'; import { useCredentials } from '../context/UserCredentials'; -import { ArrowPathIconSolid, ClipboardDocumentIconOutline, MagnifyingGlassCircleIconSolid } from '@neo4j-ndl/react/icons'; +import { + ArrowPathIconSolid, + ClipboardDocumentIconOutline, + MagnifyingGlassCircleIconSolid, +} from '@neo4j-ndl/react/icons'; import CustomProgressBar from './UI/CustomProgressBar'; import subscribe from '../services/PollingAPI'; import { triggerStatusUpdateAPI } from '../services/ServerSideStatusUpdateAPI'; @@ -49,7 +53,9 @@ import { IconButtonWithToolTip } from './UI/IconButtonToolTip'; import { batchSize, largeFileSize, llms } from '../utils/Constants'; import IndeterminateCheckbox from './UI/CustomCheckBox'; import { showErrorToast, showNormalToast } from '../utils/toasts'; +import { ThemeWrapperContext } from '../context/ThemeWrapper'; let onlyfortheFirstRender = true; + const FileTable = forwardRef((props, ref) => { const { isExpanded, connectionStatus, setConnectionStatus, onInspect, onRetry } = props; const { filesData, setFilesData, model, rowSelection, setRowSelection, setSelectedRows, setProcessedCount, queue } = @@ -63,7 +69,8 @@ const FileTable = forwardRef((props, ref) => { const [fileSourceFilter, setFileSourceFilter] = useState(''); const [llmtypeFilter, setLLmtypeFilter] = useState(''); const skipPageResetRef = useRef(false); - const [value, copy] = useCopyToClipboard(); + const [_, copy] = useCopyToClipboard(); + const { colorMode } = useContext(ThemeWrapperContext); const tableRef = useRef(null); @@ -148,7 +155,6 @@ const FileTable = forwardRef((props, ref) => { if (info.getValue() != 'Processing') { return ( -
((props, ref) => { {(info.getValue() === 'Completed' || info.getValue() === 'Failed' || (info.getValue() === 'Cancelled' && !isReadOnlyUser)) && ( - - - onRetry(info?.row?.id as string)} - > - - - - )} + + onRetry(info?.row?.id as string)} + > + + + + )}
{info.row.original?.status === 'Failed' && ( @@ -185,11 +190,14 @@ const FileTable = forwardRef((props, ref) => { size='small' onClick={() => handleCopy(info.row.original?.errorMessage ?? '')} > - + - ) - } - + + )} + ); } else if (info.getValue() === 'Processing' && info.row.original.processingProgress === undefined) { return ( @@ -538,7 +546,7 @@ const FileTable = forwardRef((props, ref) => { footer: (info) => info.column.id, }), ], - [filesData.length, statusFilter, filetypeFilter, llmtypeFilter, fileSourceFilter, isReadOnlyUser] + [filesData.length, statusFilter, filetypeFilter, llmtypeFilter, fileSourceFilter, isReadOnlyUser, colorMode] ); const table = useReactTable({ @@ -684,8 +692,8 @@ const FileTable = forwardRef((props, ref) => { language: item?.language ?? '', processingProgress: item?.processed_chunk != undefined && - item?.total_chunks != undefined && - !isNaN(Math.floor((item?.processed_chunk / item?.total_chunks) * 100)) + item?.total_chunks != undefined && + !isNaN(Math.floor((item?.processed_chunk / item?.total_chunks) * 100)) ? Math.floor((item?.processed_chunk / item?.total_chunks) * 100) : undefined, access_token: item?.access_token ?? '', diff --git a/frontend/src/components/Graph/GraphViewModal.tsx b/frontend/src/components/Graph/GraphViewModal.tsx index e61919fdb..9dc561553 100644 --- a/frontend/src/components/Graph/GraphViewModal.tsx +++ b/frontend/src/components/Graph/GraphViewModal.tsx @@ -98,10 +98,10 @@ const GraphViewModal: React.FunctionComponent = ({ graphType.includes('DocumentChunk') && graphType.includes('Entities') ? queryMap.DocChunkEntities : graphType.includes('DocumentChunk') - ? queryMap.DocChunks - : graphType.includes('Entities') - ? queryMap.Entities - : ''; + ? queryMap.DocChunks + : graphType.includes('Entities') + ? queryMap.Entities + : ''; // fit graph to original position const handleZoomToFit = () => { @@ -136,10 +136,10 @@ const GraphViewModal: React.FunctionComponent = ({ const nodeRelationshipData = viewPoint === graphLabels.showGraphView ? await graphQueryAPI( - userCredentials as UserCredentials, - graphQuery, - selectedRows?.map((f) => f.name) - ) + userCredentials as UserCredentials, + graphQuery, + selectedRows?.map((f) => f.name) + ) : await graphQueryAPI(userCredentials as UserCredentials, graphQuery, [inspectedName ?? '']); return nodeRelationshipData; } catch (error: any) { @@ -252,8 +252,8 @@ const GraphViewModal: React.FunctionComponent = ({ match && viewPoint === graphLabels.showGraphView ? 100 : match && viewPoint !== graphLabels.showGraphView - ? 50 - : graphLabels.nodeSize, + ? 50 + : graphLabels.nodeSize, }; }); // deactivating any active relationships @@ -362,8 +362,8 @@ const GraphViewModal: React.FunctionComponent = ({ isActive && viewPoint === graphLabels.showGraphView ? 100 : isActive && viewPoint !== graphLabels.showGraphView - ? 50 - : graphLabels.nodeSize, + ? 50 + : graphLabels.nodeSize, }; }); // deactivating any active relationships