diff --git a/pom.xml b/pom.xml index d9d73ed41f2..6e5eee3b28e 100644 --- a/pom.xml +++ b/pom.xml @@ -58,6 +58,11 @@ websocket-api ${jetty_version} + + com.github.javafaker + javafaker + 0.12 + org.opencampaignlink. report-generator-jvm diff --git a/src/main/java/ca/uhn/fhir/jpa/starter/service/HelperService.java b/src/main/java/ca/uhn/fhir/jpa/starter/service/HelperService.java index 22d1e891ac7..33f5bee317c 100644 --- a/src/main/java/ca/uhn/fhir/jpa/starter/service/HelperService.java +++ b/src/main/java/ca/uhn/fhir/jpa/starter/service/HelperService.java @@ -1184,7 +1184,7 @@ public List getGroupsByUser(String userId) { // return ResponseEntity.ok(dataResult); // } - public void saveInAsyncTable(DataResult dataResult, String id) { + public void saveInAsyncTable(DataResult dataResult, String id, Boolean isApiRequest) { notificationDataSource = NotificationDataSource.getInstance(); byte[] summaryResult = dataResult.getSummaryResult(); List> dailyResult = dataResult.getDailyResult(); @@ -1201,6 +1201,8 @@ public void saveInAsyncTable(DataResult dataResult, String id) { asyncRecord.setLastUpdated(Date.valueOf(LocalDate.now())); asyncRecord.setAnonymousEntry(ISANONYMIZED.NO.name()); notificationDataSource.update(asyncRecord); + if (!isApiRequest) + saveAnonymousDataInAsyncTable(dataResult, id); } catch (Exception e) { logger.warn(ExceptionUtils.getStackTrace(e)); } @@ -1252,7 +1254,7 @@ public Map processCategories(String organizationId, String start if (categories.indexOf(category) == (categories.size() - 1)) { if (!isApiRequest || !hashCodesToBeProcessed.isEmpty()) { - saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig); + saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig, isApiRequest); } else { if (hashCodesToBeProcessed.isEmpty() && fetchAsyncData != null && !fetchAsyncData.isEmpty()) { Date lastUpdated = fetchAsyncData.get(0).getLastUpdated(); @@ -1264,7 +1266,7 @@ public Map processCategories(String organizationId, String start asyncRecord.setStatus(ApiAsyncTaskEntity.Status.PROCESSING.name()); } datasource.updateObjects(entitiesToUpdate); - saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig); + saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig, isApiRequest); } } } @@ -1275,11 +1277,11 @@ public Map processCategories(String organizationId, String start public void saveQueryResultAndHandleException(String organizationId, String startDate, String endDate, LinkedHashMap filters, List hashcodes, String env, - List ancDailySummaryConfig) { + List ancDailySummaryConfig, Boolean isApiRequest) { ThreadPoolTaskExecutor executor = getAsyncExecutor(); executor.submit(() -> { try { - saveQueryResult(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig); + saveQueryResult(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig, isApiRequest); } catch (FileNotFoundException e) { // Handle the exception, e.g., log it or take appropriate action. e.printStackTrace(); @@ -1291,35 +1293,68 @@ public ThreadPoolTaskExecutor getAsyncExecutor() { return asyncConf.asyncExecutor(); } + public void saveAnonymousDataInAsyncTable(DataResult dataResult, String id){ + List> dailyResult = dataResult.getDailyResult(); + if (id.contains("bio")) { + int i; + if (!dailyResult.isEmpty()){ + for (i = 0; i < dailyResult.size(); i++){ + String randomGeneratedName = Utils.generateRandomName(); + String randomGeneratedOclId = Utils.generateRandomPatientOclId(); + String randomDOB = Utils.generateRandomDate(); + Map singleDailyResult = dailyResult.get(i); + singleDailyResult.put("Name", randomGeneratedName); + singleDailyResult.put("Card Number",randomGeneratedOclId); + singleDailyResult.put("Date of Birth",randomDOB); + } + } + byte[] summaryResult = dataResult.getSummaryResult(); + String base64SummaryResult = Base64.getEncoder().encodeToString(summaryResult); + String dailyResultJsonString = new Gson().toJson(dailyResult); + String uuidHash = getFormattedUuid(id); + ApiAsyncTaskEntity apiAsyncTaskEntity = new ApiAsyncTaskEntity( + uuidHash, + ApiAsyncTaskEntity.Status.COMPLETED.name(), + ClobProxy.generateProxy(base64SummaryResult), + ClobProxy.generateProxy(dailyResultJsonString), + Date.valueOf(LocalDate.now()), + ISANONYMIZED.YES.name() + ); + notificationDataSource.insert(apiAsyncTaskEntity); + } + } + public ResponseEntity getAsyncData(Map categoryWithHashCodes, Boolean isAnonymizationEnabled) throws SQLException, IOException { List dataResult = new ArrayList<>(); for (Map.Entry item : categoryWithHashCodes.entrySet()) { - String dateRangeValue = getFormattedUUID(item.getValue()); ArrayList asyncData = datasource.fetchStatus(item.getValue()); - ArrayList anonymousAsyncData = datasource.fetchAnonymousData(dateRangeValue); if (asyncData == null) return ResponseEntity.ok("Searching in Progress"); ApiAsyncTaskEntity asyncRecord = asyncData.get(0); if (asyncRecord.getSummaryResult() == null || asyncRecord.getStatus() == ApiAsyncTaskEntity.Status.PROCESSING.name()) return ResponseEntity.ok("Searching in Progress"); String dailyResultInString = convertClobToString(asyncRecord.getDailyResult()); String summaryResultInString = convertClobToString(asyncRecord.getSummaryResult()); - if (!anonymousAsyncData.isEmpty() && isAnonymizationEnabled) { + List> dailyResult = mapper.readValue(dailyResultInString, new TypeReference>>() { + }); + dataResult.add(new DataResultJava(item.getKey(), summaryResultInString, dailyResult)); + } + if (isAnonymizationEnabled){ + String formattedUuid = getFormattedUuid(categoryWithHashCodes.get("patient-bio-data")); + ArrayList anonymousAsyncData = datasource.fetchAnonymousData(formattedUuid); + if (!anonymousAsyncData.isEmpty()) { + dataResult.removeIf(dataResultJava -> dataResultJava.getCategoryId().contains("bio")); ApiAsyncTaskEntity anonymousAsyncRecord = anonymousAsyncData.get(0); String anonymousDailyResultInString = convertClobToString(anonymousAsyncRecord.getDailyResult()); String anonymousSummaryResultInString = convertClobToString(anonymousAsyncRecord.getSummaryResult()); List> anonymousDailyResult = mapper.readValue(anonymousDailyResultInString, new TypeReference>>() { }); - dataResult.add(new DataResultJava(item.getKey(), anonymousSummaryResultInString, anonymousDailyResult)); - } else{ - List> dailyResult = mapper.readValue(dailyResultInString, new TypeReference>>() { - }); - dataResult.add(new DataResultJava(item.getKey(), summaryResultInString, dailyResult)); + dataResult.add(new DataResultJava("patient-bio-data", anonymousSummaryResultInString, anonymousDailyResult)); } } return ResponseEntity.ok(dataResult); } - public static String getFormattedUUID(String input) { + public static String getFormattedUuid(String input) { // Regular expression to match the date range pattern followed by any text String regex = "(\\d{4}-\\d{2}-\\d{2})(\\d{4}-\\d{2}-\\d{2})(.*)"; @@ -1339,7 +1374,7 @@ public static String getFormattedUUID(String input) { public void saveQueryResult(String organizationId, String startDate, String endDate, LinkedHashMap filters, List hashcodes, String env, - List ancDailySummaryConfig) throws FileNotFoundException { + List ancDailySummaryConfig, Boolean isApiRequest) throws FileNotFoundException { List fhirSearchList = getFhirSearchListByFilters(filters, env); logger.warn("Calling details page function saveQueryResult. StartDate: {} EndDate: {}", startDate, endDate); @@ -1347,7 +1382,7 @@ public void saveQueryResult(String organizationId, String startDate, String endD List dataResult = (List) getReportGen("getAncDailySummaryData", organizationId, startDate, endDate, fhirSearchList, ancDailySummaryConfig); for (String hashcode : hashcodes) { - saveInAsyncTable(dataResult.get(hashcodes.indexOf(hashcode)), hashcode); + saveInAsyncTable(dataResult.get(hashcodes.indexOf(hashcode)), hashcode, isApiRequest); } Long end3 = System.nanoTime(); Double diff3 = ((end3 - start3) / 1e9); // Convert nanoseconds to seconds diff --git a/src/main/java/com/iprd/fhir/utils/Utils.java b/src/main/java/com/iprd/fhir/utils/Utils.java index f8f530327b8..bb32fbeb266 100644 --- a/src/main/java/com/iprd/fhir/utils/Utils.java +++ b/src/main/java/com/iprd/fhir/utils/Utils.java @@ -1,8 +1,10 @@ package com.iprd.fhir.utils; +import com.github.javafaker.Faker; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import com.iprd.report.FhirPath; +import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -573,4 +575,21 @@ public static boolean noneMatchDates(List dates, Date currentDate) { .map(Date::toLocalDate) .noneMatch(date -> date.equals(currentLocalDate)); } + + public static String generateRandomName(){ + Faker faker = new Faker(); + return faker.name().fullName(); + } + + public static String generateRandomPatientOclId(){ + return RandomStringUtils.random(12, true, true); + } + + public static String generateRandomDate(){ + Random random = new Random(); + int minDay = (int) LocalDate.of(1900, 1, 1).toEpochDay(); + int maxDay = (int) LocalDate.of(2015, 1, 1).toEpochDay(); + long randomDay = minDay + random.nextInt(maxDay - minDay); + return LocalDate.ofEpochDay(randomDay).toString(); + } } diff --git a/src/test/java/ca/uhn/fhir/jpa/starter/service/HelperServiceTest.java b/src/test/java/ca/uhn/fhir/jpa/starter/service/HelperServiceTest.java index 1442737045d..4e1bf51da28 100644 --- a/src/test/java/ca/uhn/fhir/jpa/starter/service/HelperServiceTest.java +++ b/src/test/java/ca/uhn/fhir/jpa/starter/service/HelperServiceTest.java @@ -10,11 +10,7 @@ import ca.uhn.fhir.jpa.starter.model.CategoryItem; import ca.uhn.fhir.jpa.starter.model.PatientIdentifierEntity; import ca.uhn.fhir.jpa.starter.model.ScoreCardIndicatorItem; -import ca.uhn.fhir.jpa.starter.model.OrgHierarchy; -import ca.uhn.fhir.jpa.starter.model.OrgIndicatorAverageResult; -import ca.uhn.fhir.jpa.starter.model.ScoreCardResponseItem; import ca.uhn.fhir.rest.client.api.IGenericClient; -import ca.uhn.fhir.jpa.starter.model.ReportType; import ca.uhn.fhir.rest.gclient.IQuery; import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; @@ -32,11 +28,7 @@ import com.iprd.report.model.definition.FhirPathTransformation; import com.iprd.report.model.definition.BarComponent; import com.iprd.report.model.definition.ANCDailySummaryConfig; -import com.iprd.report.model.data.BarChartItemDataCollection; -import com.iprd.report.model.data.LineChartItemCollection; import com.iprd.report.model.definition.IndicatorItem; -import com.iprd.report.model.data.PieChartItemDataCollection; -import com.iprd.report.model.data.ScoreCardItem; import org.hibernate.SessionFactory; import org.hl7.fhir.instance.model.api.IBaseBundle; import org.hl7.fhir.r4.model.Organization; @@ -88,14 +80,13 @@ import java.util.concurrent.TimeUnit; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.doNothing; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doCallRealMethod; import static org.mockito.Mockito.times; @@ -571,7 +562,7 @@ void testSaveInAsyncTable() throws SQLException { doNothing().when(notificationDataSourceMock).update(any()); // Replace the actual instance with the mock in NotificationDataSource Whitebox.setInternalState(NotificationDataSource.class, notificationDataSourceMock); - helperService.saveInAsyncTable(dataResult,"68455"); + helperService.saveInAsyncTable(dataResult,"68455", true); verify(notificationDataSourceMock).update(any(ApiAsyncTaskEntity.class)); // Verify the update method was called with the captured argument ArgumentCaptor argumentCaptor = ArgumentCaptor.forClass(ApiAsyncTaskEntity.class); @@ -645,19 +636,19 @@ void testSaveQueryResultAndHandleException() throws FileNotFoundException, Inter executor.initialize(); when(helperServiceMoc.getAsyncExecutor( )).thenReturn(executor); - doNothing().when(helperServiceMoc).saveQueryResult(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList()); + doNothing().when(helperServiceMoc).saveQueryResult(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList(),eq(false)); doAnswer(invocation -> { latch.countDown(); return null; - }).when(helperServiceMoc).saveQueryResult(anyString(), anyString(), anyString(), any(), anyList(), anyString(), anyList()); + }).when(helperServiceMoc).saveQueryResult(anyString(), anyString(), anyString(), any(), anyList(), anyString(), anyList(),eq(false)); // Call the method - doCallRealMethod().when(helperServiceMoc).saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig); - helperServiceMoc.saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig); + doCallRealMethod().when(helperServiceMoc).saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig, true); + helperServiceMoc.saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig, true); // Wait for the asynchronous operation to complete assertTrue(latch.await(5, TimeUnit.SECONDS)); // Verify that the saveQueryResult method was called verify(helperServiceMoc, times(1)).saveQueryResult( - eq(organizationId), eq(startDate), eq(endDate), eq(filters), eq(hashcodes), eq(env), eq(ancDailySummaryConfig) + eq(organizationId), eq(startDate), eq(endDate), eq(filters), eq(hashcodes), eq(env), eq(ancDailySummaryConfig),eq(false) ); executor.shutdown(); } @@ -685,11 +676,11 @@ void testProcessCategories() throws FileNotFoundException { .thenReturn(new ArrayList()); helperServiceMock.datasource = notificationDataSourceMock; doNothing().when(notificationDataSourceMock).insert(any()); - doNothing().when(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList()); + doNothing().when(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList(), true); doCallRealMethod().when(helperServiceMock).processCategories(organizationId,startDate,endDate,env,filters,true); Map categoryWithHashCodes = helperServiceMock.processCategories(organizationId,startDate,endDate,env,filters,true); ArgumentCaptor> argumentCaptor = ArgumentCaptor.forClass(List.class); - verify(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),argumentCaptor.capture()); + verify(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),argumentCaptor.capture(), true); List capturedList = argumentCaptor.getValue(); assertEquals(ancDailySummaryConfig,capturedList); assertEquals(categoryWithHashCodes.get("antenatal"),"e15b899b-8d94-4279-8cb7-3eb90a14279b2023-11-012023-11-01antenatalV2age-2");