Skip to content

Commit 41ae360

Browse files
kartikpersistentpraveshkumar1988
kartikpersistent
and
praveshkumar1988
authored
File expiration alert (#953)
* added the expiration alert * expiration days changes * disabled the continue button for expired files * Added gcs_file_cache in response of connect API and removed unused library import * Added GCS_FILE_CACHE on backend_config API * added the GCS_FILE_CACHE env variable check --------- Co-authored-by: Pravesh Kumar <[email protected]>
1 parent 633ed87 commit 41ae360

File tree

21 files changed

+235
-293
lines changed

21 files changed

+235
-293
lines changed

‎backend/score.py‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,19 @@
1818
from src.communities import create_communities
1919
from src.neighbours import get_neighbour_nodes
2020
import json
21-
from typing import List, Mapping, Union
21+
from typing import List
2222
from starlette.middleware.sessions import SessionMiddleware
23-
import google_auth_oauthlib.flow
2423
from google.oauth2.credentials import Credentials
2524
import os
2625
from src.logger import CustomLogger
2726
from datetime import datetime, timezone
2827
import time
2928
import gc
30-
from Secweb import SecWeb
31-
from Secweb.StrictTransportSecurity import HSTS
32-
from Secweb.ContentSecurityPolicy import ContentSecurityPolicy
3329
from Secweb.XContentTypeOptions import XContentTypeOptions
3430
from Secweb.XFrameOptions import XFrame
3531
from fastapi.middleware.gzip import GZipMiddleware
3632
from src.ragas_eval import *
37-
from starlette.types import ASGIApp, Message, Receive, Scope, Send
38-
import gzip
33+
from starlette.types import ASGIApp, Receive, Scope, Send
3934
from langchain_neo4j import Neo4jGraph
4035

4136
logger = CustomLogger()
@@ -493,11 +488,13 @@ async def connect(uri=Form(), userName=Form(), password=Form(), database=Form())
493488
start = time.time()
494489
graph = create_graph_database_connection(uri, userName, password, database)
495490
result = await asyncio.to_thread(connection_check_and_get_vector_dimensions, graph, database)
491+
gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
496492
end = time.time()
497493
elapsed_time = end - start
498494
json_obj = {'api_name':'connect','db_url':uri, 'userName':userName, 'database':database,'status':result, 'count':1, 'logging_time': formatted_time(datetime.now(timezone.utc)), 'elapsed_api_time':f'{elapsed_time:.2f}'}
499495
logger.log_struct(json_obj, "INFO")
500496
result['elapsed_api_time'] = f'{elapsed_time:.2f}'
497+
result['gcs_file_cache'] = gcs_file_cache
501498
return create_api_response('Success',data=result)
502499
except Exception as e:
503500
job_status = "Failed"
@@ -975,6 +972,7 @@ async def backend_connection_configuration():
975972
username= os.getenv('NEO4J_USERNAME')
976973
database= os.getenv('NEO4J_DATABASE')
977974
password= os.getenv('NEO4J_PASSWORD')
975+
gcs_file_cache = os.environ.get('GCS_FILE_CACHE')
978976
if all([uri, username, database, password]):
979977
print(f'uri:{uri}, usrName:{username}, database :{database}, password: {password}')
980978
graph = Neo4jGraph()
@@ -989,6 +987,7 @@ async def backend_connection_configuration():
989987
result["user_name"] = username
990988
result["database"] = database
991989
result["password"] = encoded_password
990+
result['gcs_file_cache'] = gcs_file_cache
992991
return create_api_response('Success',message=f"Backend connection successful",data=result)
993992
else:
994993
graph_connection = False

‎frontend/src/components/ChatBot/ChatInfoModal.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -393,7 +393,7 @@ const ChatInfoModal: React.FC<chatInfoMessage> = ({
393393
<Typography variant='body-medium'>
394394
Currently ragas evaluation works on{' '}
395395
{supportedLLmsForRagas.map((s, idx) => (
396-
<span className='font-bold'>
396+
<span className='font-bold' key={s}>
397397
{capitalizeWithUnderscore(s) + (idx != supportedLLmsForRagas.length - 1 ? ',' : '')}
398398
</span>
399399
))}

‎frontend/src/components/ChatBot/EntitiesInfo.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -93,11 +93,11 @@ const EntitiesInfo: FC<EntitiesProps> = ({ loading, mode, graphonly_entities, in
9393
</ul>
9494
</li>
9595
))
96-
: sortedLabels.map((label, index) => {
96+
: sortedLabels.map((label) => {
9797
const entity = groupedEntities[label == 'undefined' ? 'Entity' : label];
9898
return (
9999
<li
100-
key={index}
100+
key={label}
101101
className='flex items-center mb-2 text-ellipsis whitespace-nowrap max-w-[100%)] overflow-hidden'
102102
>
103103
<GraphLabel type='node' className='legend' color={`${entity.color}`} isSelected={false}>

‎frontend/src/components/Content.tsx‎

Lines changed: 46 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import { useMessageContext } from '../context/UserMessages';
3535
import PostProcessingToast from './Popups/GraphEnhancementDialog/PostProcessingCheckList/PostProcessingToast';
3636
import { getChunkText } from '../services/getChunkText';
3737
import ChunkPopUp from './Popups/ChunkPopUp';
38+
import { isExpired } from '../utils/Utils';
3839

3940
const ConfirmationDialog = lazy(() => import('./Popups/LargeFilePopUp/ConfirmationDialog'));
4041

@@ -55,8 +56,10 @@ const Content: React.FC<ContentProps> = ({
5556
const [openGraphView, setOpenGraphView] = useState<boolean>(false);
5657
const [inspectedName, setInspectedName] = useState<string>('');
5758
const [documentName, setDocumentName] = useState<string>('');
58-
const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser } = useCredentials();
59+
const { setUserCredentials, userCredentials, setConnectionStatus, isGdsActive, isReadOnlyUser, isGCSActive } =
60+
useCredentials();
5961
const [showConfirmationModal, setshowConfirmationModal] = useState<boolean>(false);
62+
const [showExpirationModal, setshowExpirationModal] = useState<boolean>(false);
6063
const [extractLoading, setextractLoading] = useState<boolean>(false);
6164
const [retryFile, setRetryFile] = useState<string>('');
6265
const [retryLoading, setRetryLoading] = useState<boolean>(false);
@@ -660,7 +663,8 @@ const Content: React.FC<ContentProps> = ({
660663
const onClickHandler = () => {
661664
const selectedRows = childRef.current?.getSelectedRows();
662665
if (selectedRows?.length) {
663-
let selectedLargeFiles: CustomFile[] = [];
666+
const selectedLargeFiles: CustomFile[] = [];
667+
const expiredFiles: CustomFile[] = [];
664668
for (let index = 0; index < selectedRows.length; index++) {
665669
const parsedData: CustomFile = selectedRows[index];
666670
if (
@@ -671,9 +675,18 @@ const Content: React.FC<ContentProps> = ({
671675
) {
672676
selectedLargeFiles.push(parsedData);
673677
}
678+
if (
679+
parsedData.fileSource === 'local file' &&
680+
(parsedData.status === 'New' || parsedData.status == 'Ready to Reprocess') &&
681+
isExpired(parsedData?.createdAt as Date)
682+
) {
683+
expiredFiles.push(parsedData);
684+
}
674685
}
675686
if (selectedLargeFiles.length) {
676687
setshowConfirmationModal(true);
688+
} else if (expiredFiles.length && isGCSActive) {
689+
setshowExpirationModal(true);
677690
} else {
678691
handleGenerateGraph(selectedRows.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess'));
679692
}
@@ -688,16 +701,34 @@ const Content: React.FC<ContentProps> = ({
688701
}
689702
return false;
690703
});
704+
const expiredFiles = filesData.filter((f) => {
705+
console.log(
706+
f.fileSource === 'local file'
707+
? { isExpired: isExpired(f?.createdAt as Date), name: f.name, createdAt: f.createdAt }
708+
: 'Not Local File'
709+
);
710+
if (
711+
f.fileSource === 'local file' &&
712+
(f.status === 'New' || f.status == 'Ready to Reprocess') &&
713+
isExpired(f?.createdAt as Date)
714+
) {
715+
return true;
716+
}
717+
return false;
718+
});
691719
const selectAllNewFiles = filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess');
692720
const stringified = selectAllNewFiles.reduce((accu, f) => {
693721
const key = f.id;
694722
// @ts-ignore
695723
accu[key] = true;
696724
return accu;
697725
}, {});
726+
console.log(expiredFiles);
698727
setRowSelection(stringified);
699728
if (largefiles.length) {
700729
setshowConfirmationModal(true);
730+
} else if (expiredFiles.length && isGCSActive) {
731+
setshowExpirationModal(true);
701732
} else {
702733
handleGenerateGraph(filesData.filter((f) => f.status === 'New' || f.status === 'Ready to Reprocess'));
703734
}
@@ -746,6 +777,19 @@ const Content: React.FC<ContentProps> = ({
746777
></ConfirmationDialog>
747778
</Suspense>
748779
)}
780+
{showExpirationModal && filesForProcessing.length && (
781+
<Suspense fallback={<FallBackDialog />}>
782+
<ConfirmationDialog
783+
open={showExpirationModal}
784+
largeFiles={filesForProcessing}
785+
extractHandler={handleGenerateGraph}
786+
onClose={() => setshowExpirationModal(false)}
787+
loading={extractLoading}
788+
selectedRows={childRef.current?.getSelectedRows() as CustomFile[]}
789+
isLargeDocumentAlert={false}
790+
></ConfirmationDialog>
791+
</Suspense>
792+
)}
749793
{showDeletePopUp && (
750794
<DeletePopUp
751795
open={showDeletePopUp}

‎frontend/src/components/DataSources/Local/DropZone.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ const DropZone: FunctionComponent = () => {
4040
entityEntityRelCount: 0,
4141
communityNodeCount: 0,
4242
communityRelCount: 0,
43+
createdAt: new Date(),
4344
};
4445

4546
const copiedFilesData: CustomFile[] = [...filesData];
@@ -186,6 +187,7 @@ const DropZone: FunctionComponent = () => {
186187
...curfile,
187188
status: 'New',
188189
uploadProgress: 100,
190+
createdAt: new Date(),
189191
};
190192
}
191193
return curfile;
@@ -241,7 +243,7 @@ const DropZone: FunctionComponent = () => {
241243
'application/vnd.ms-powerpoint': ['.pptx'],
242244
'application/vnd.ms-excel': ['.xls'],
243245
'text/markdown': ['.md'],
244-
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':['.xlsx']
246+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
245247
},
246248
onDrop: (f: Partial<globalThis.File>[]) => {
247249
onDropHandler(f);

‎frontend/src/components/DataSources/Local/DropZoneForSmallLayouts.tsx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@ export default function DropZoneForSmallLayouts() {
118118
...curfile,
119119
status: 'New',
120120
uploadprogess: 100,
121+
createdAt: new Date(),
121122
};
122123
}
123124
return curfile;
@@ -141,7 +142,7 @@ export default function DropZoneForSmallLayouts() {
141142
'application/vnd.ms-powerpoint': ['.pptx'],
142143
'application/vnd.ms-excel': ['.xls'],
143144
'text/markdown': ['.md'],
144-
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet':['.xlsx']
145+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
145146
},
146147
onDrop: (f: Partial<globalThis.File>[]) => {
147148
onDropHandler(f);
@@ -176,6 +177,7 @@ export default function DropZoneForSmallLayouts() {
176177
entityEntityRelCount: 0,
177178
communityNodeCount: 0,
178179
communityRelCount: 0,
180+
createdAt: new Date(),
179181
};
180182

181183
const copiedFilesData: CustomFile[] = [...filesData];

‎frontend/src/components/FileTable.tsx‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ import {
3434
getFileSourceStatus,
3535
isProcessingFileValid,
3636
capitalizeWithUnderscore,
37+
getParsedDate,
3738
} from '../utils/Utils';
3839
import { SourceNode, CustomFile, FileTableProps, UserCredentials, statusupdate, ChildRef } from '../types';
3940
import { useCredentials } from '../context/UserCredentials';
@@ -763,6 +764,7 @@ const FileTable = forwardRef<ChildRef, FileTableProps>((props, ref) => {
763764
entityEntityRelCount: item.entityEntityRelCount ?? 0,
764765
communityNodeCount: item.communityNodeCount ?? 0,
765766
communityRelCount: item.communityRelCount ?? 0,
767+
createdAt: item.createdAt != undefined ? getParsedDate(item?.createdAt) : undefined,
766768
});
767769
}
768770
});
@@ -783,6 +785,7 @@ const FileTable = forwardRef<ChildRef, FileTableProps>((props, ref) => {
783785
}
784786
setIsLoading(false);
785787
} catch (error: any) {
788+
console.log(error);
786789
if (error instanceof Error) {
787790
showErrorToast(error.message);
788791
}

‎frontend/src/components/Graph/ResultOverview.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,10 @@ const ResultOverview: React.FunctionComponent<OverViewProps> = ({
143143
</Flex>
144144
<div className='flex gap-2 flex-wrap ml-2'>
145145
<ShowAll initiallyShown={RESULT_STEP_SIZE}>
146-
{nodeCheck.map((nodeLabel, index) => (
146+
{nodeCheck.map((nodeLabel) => (
147147
<LegendsChip
148148
type='node'
149-
key={index}
149+
key={nodeLabel}
150150
label={nodeLabel}
151151
scheme={newScheme}
152152
count={nodeCount(nodes, nodeLabel)}

‎frontend/src/components/Layout/PageLayout.tsx‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const PageLayout: React.FC = () => {
6161
setErrorMessage,
6262
setShowDisconnectButton,
6363
showDisconnectButton,
64+
setIsGCSActive,
6465
} = useCredentials();
6566
const { cancel } = useSpeechSynthesis();
6667

@@ -97,6 +98,7 @@ const PageLayout: React.FC = () => {
9798
});
9899
setGdsActive(parsedConnection.isgdsActive);
99100
setIsReadOnlyUser(parsedConnection.isReadOnlyUser);
101+
setIsGCSActive(parsedConnection.isGCSActive);
100102
} else {
101103
console.error('Invalid parsed session data:', parsedConnection);
102104
}
@@ -105,7 +107,8 @@ const PageLayout: React.FC = () => {
105107
}
106108
};
107109
// To update credentials if environment values differ
108-
const updateSessionIfNeeded = (envCredentials: UserCredentials, storedSession: string) => {
110+
111+
const updateSessionIfNeeded = (envCredentials: any, storedSession: string) => {
109112
try {
110113
const storedCredentials = JSON.parse(storedSession);
111114
const isDiffCreds =
@@ -115,6 +118,7 @@ const PageLayout: React.FC = () => {
115118
envCredentials.database !== storedCredentials.database;
116119
if (isDiffCreds) {
117120
setUserCredentials(envCredentials);
121+
setIsGCSActive(envCredentials.isGCSActive ?? false);
118122
localStorage.setItem(
119123
'neo4j.connection',
120124
JSON.stringify({
@@ -147,7 +151,9 @@ const PageLayout: React.FC = () => {
147151
database: connectionData.data.database,
148152
isReadonlyUser: !connectionData.data.write_access,
149153
isgdsActive: connectionData.data.gds_status,
154+
isGCSActive: connectionData?.data?.gcs_file_cache === 'True',
150155
};
156+
setIsGCSActive(connectionData?.data?.gcs_file_cache === 'True');
151157
if (session) {
152158
const updated = updateSessionIfNeeded(envCredentials, session);
153159
if (!updated) {
@@ -168,6 +174,7 @@ const PageLayout: React.FC = () => {
168174
userDbVectorIndex: 384,
169175
isReadOnlyUser: envCredentials.isReadonlyUser,
170176
isgdsActive: envCredentials.isgdsActive,
177+
isGCSActive: envCredentials.isGCSActive,
171178
})
172179
);
173180
setConnectionStatus(true);

‎frontend/src/components/Popups/ChunkPopUp/index.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,8 @@ const ChunkPopUp = ({
5858
<Loader title='loading...'></Loader>
5959
) : (
6060
<ol className='max-h-80 overflow-y-auto flex flex-col gap-4'>
61-
{sortedChunksData.map((c, idx) => (
62-
<li key={`${idx}${c.position}`} className='flex flex-row gap-1'>
61+
{sortedChunksData.map((c) => (
62+
<li key={`${c.position}`} className='flex flex-row gap-1'>
6363
<Flex flexDirection='column' gap='2'>
6464
<Flex flexDirection='row'>
6565
<Typography variant='label'>Position :</Typography>

0 commit comments

Comments
 (0)