Skip to content

Create Anonymized Data #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: fhir-resources-helper
Choose a base branch
from
Open

Create Anonymized Data #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0fc71ae
create anonymized entries.
AdityaKBhadragond14 Jul 15, 2024
1d36c94
changed to replace the original patient bio data with anonymized pati…
AdityaKBhadragond14 Jul 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,11 @@
<artifactId>websocket-api</artifactId>
<version>${jetty_version}</version>
</dependency>
<dependency>
<groupId>com.github.javafaker</groupId>
<artifactId>javafaker</artifactId>
<version>0.12</version>
</dependency>
<dependency>
<groupId>org.opencampaignlink.</groupId>
<artifactId>report-generator-jvm</artifactId>
Expand Down
67 changes: 51 additions & 16 deletions src/main/java/ca/uhn/fhir/jpa/starter/service/HelperService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1184,7 +1184,7 @@ public List<GroupRepresentation> getGroupsByUser(String userId) {
// return ResponseEntity.ok(dataResult);
// }

public void saveInAsyncTable(DataResult dataResult, String id) {
public void saveInAsyncTable(DataResult dataResult, String id, Boolean isApiRequest) {
notificationDataSource = NotificationDataSource.getInstance();
byte[] summaryResult = dataResult.getSummaryResult();
List<Map<String, String>> dailyResult = dataResult.getDailyResult();
Expand All @@ -1201,6 +1201,8 @@ public void saveInAsyncTable(DataResult dataResult, String id) {
asyncRecord.setLastUpdated(Date.valueOf(LocalDate.now()));
asyncRecord.setAnonymousEntry(ISANONYMIZED.NO.name());
notificationDataSource.update(asyncRecord);
if (!isApiRequest)
saveAnonymousDataInAsyncTable(dataResult, id);
} catch (Exception e) {
logger.warn(ExceptionUtils.getStackTrace(e));
}
Expand Down Expand Up @@ -1252,7 +1254,7 @@ public Map<String, String> processCategories(String organizationId, String start

if (categories.indexOf(category) == (categories.size() - 1)) {
if (!isApiRequest || !hashCodesToBeProcessed.isEmpty()) {
saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig);
saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig, isApiRequest);
} else {
if (hashCodesToBeProcessed.isEmpty() && fetchAsyncData != null && !fetchAsyncData.isEmpty()) {
Date lastUpdated = fetchAsyncData.get(0).getLastUpdated();
Expand All @@ -1264,7 +1266,7 @@ public Map<String, String> processCategories(String organizationId, String start
asyncRecord.setStatus(ApiAsyncTaskEntity.Status.PROCESSING.name());
}
datasource.updateObjects(entitiesToUpdate);
saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig);
saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashCodes, env, ancDailySummaryConfig, isApiRequest);
}
}
}
Expand All @@ -1275,11 +1277,11 @@ public Map<String, String> processCategories(String organizationId, String start

public void saveQueryResultAndHandleException(String organizationId, String startDate, String endDate,
LinkedHashMap<String, String> filters, List<String> hashcodes, String env,
List<ANCDailySummaryConfig> ancDailySummaryConfig) {
List<ANCDailySummaryConfig> ancDailySummaryConfig, Boolean isApiRequest) {
ThreadPoolTaskExecutor executor = getAsyncExecutor();
executor.submit(() -> {
try {
saveQueryResult(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig);
saveQueryResult(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig, isApiRequest);
} catch (FileNotFoundException e) {
// Handle the exception, e.g., log it or take appropriate action.
e.printStackTrace();
Expand All @@ -1291,35 +1293,68 @@ public ThreadPoolTaskExecutor getAsyncExecutor() {
return asyncConf.asyncExecutor();
}

public void saveAnonymousDataInAsyncTable(DataResult dataResult, String id){
List<Map<String, String>> dailyResult = dataResult.getDailyResult();
if (id.contains("bio")) {
int i;
if (!dailyResult.isEmpty()){
for (i = 0; i < dailyResult.size(); i++){
String randomGeneratedName = Utils.generateRandomName();
String randomGeneratedOclId = Utils.generateRandomPatientOclId();
String randomDOB = Utils.generateRandomDate();
Map<String, String> singleDailyResult = dailyResult.get(i);
singleDailyResult.put("Name", randomGeneratedName);
singleDailyResult.put("Card Number",randomGeneratedOclId);
singleDailyResult.put("Date of Birth",randomDOB);
}
}
byte[] summaryResult = dataResult.getSummaryResult();
String base64SummaryResult = Base64.getEncoder().encodeToString(summaryResult);
String dailyResultJsonString = new Gson().toJson(dailyResult);
String uuidHash = getFormattedUuid(id);
ApiAsyncTaskEntity apiAsyncTaskEntity = new ApiAsyncTaskEntity(
uuidHash,
ApiAsyncTaskEntity.Status.COMPLETED.name(),
ClobProxy.generateProxy(base64SummaryResult),
ClobProxy.generateProxy(dailyResultJsonString),
Date.valueOf(LocalDate.now()),
ISANONYMIZED.YES.name()
);
notificationDataSource.insert(apiAsyncTaskEntity);
}
}

public ResponseEntity<?> getAsyncData(Map<String, String> categoryWithHashCodes, Boolean isAnonymizationEnabled) throws SQLException, IOException {
List<DataResultJava> dataResult = new ArrayList<>();
for (Map.Entry<String, String> item : categoryWithHashCodes.entrySet()) {
String dateRangeValue = getFormattedUUID(item.getValue());
ArrayList<ApiAsyncTaskEntity> asyncData = datasource.fetchStatus(item.getValue());
ArrayList<ApiAsyncTaskEntity> anonymousAsyncData = datasource.fetchAnonymousData(dateRangeValue);
if (asyncData == null) return ResponseEntity.ok("Searching in Progress");
ApiAsyncTaskEntity asyncRecord = asyncData.get(0);
if (asyncRecord.getSummaryResult() == null || asyncRecord.getStatus() == ApiAsyncTaskEntity.Status.PROCESSING.name())
return ResponseEntity.ok("Searching in Progress");
String dailyResultInString = convertClobToString(asyncRecord.getDailyResult());
String summaryResultInString = convertClobToString(asyncRecord.getSummaryResult());
if (!anonymousAsyncData.isEmpty() && isAnonymizationEnabled) {
List<Map<String, String>> dailyResult = mapper.readValue(dailyResultInString, new TypeReference<List<Map<String, String>>>() {
});
dataResult.add(new DataResultJava(item.getKey(), summaryResultInString, dailyResult));
}
if (isAnonymizationEnabled){
String formattedUuid = getFormattedUuid(categoryWithHashCodes.get("patient-bio-data"));
ArrayList<ApiAsyncTaskEntity> anonymousAsyncData = datasource.fetchAnonymousData(formattedUuid);
if (!anonymousAsyncData.isEmpty()) {
dataResult.removeIf(dataResultJava -> dataResultJava.getCategoryId().contains("bio"));
ApiAsyncTaskEntity anonymousAsyncRecord = anonymousAsyncData.get(0);
String anonymousDailyResultInString = convertClobToString(anonymousAsyncRecord.getDailyResult());
String anonymousSummaryResultInString = convertClobToString(anonymousAsyncRecord.getSummaryResult());
List<Map<String, String>> anonymousDailyResult = mapper.readValue(anonymousDailyResultInString, new TypeReference<List<Map<String, String>>>() {
});
dataResult.add(new DataResultJava(item.getKey(), anonymousSummaryResultInString, anonymousDailyResult));
} else{
List<Map<String, String>> dailyResult = mapper.readValue(dailyResultInString, new TypeReference<List<Map<String, String>>>() {
});
dataResult.add(new DataResultJava(item.getKey(), summaryResultInString, dailyResult));
dataResult.add(new DataResultJava("patient-bio-data", anonymousSummaryResultInString, anonymousDailyResult));
}
}
return ResponseEntity.ok(dataResult);
}

public static String getFormattedUUID(String input) {
public static String getFormattedUuid(String input) {
// Regular expression to match the date range pattern followed by any text
String regex = "(\\d{4}-\\d{2}-\\d{2})(\\d{4}-\\d{2}-\\d{2})(.*)";

Expand All @@ -1339,15 +1374,15 @@ public static String getFormattedUUID(String input) {

public void saveQueryResult(String organizationId, String startDate, String endDate,
LinkedHashMap<String, String> filters, List<String> hashcodes, String env,
List<ANCDailySummaryConfig> ancDailySummaryConfig) throws FileNotFoundException {
List<ANCDailySummaryConfig> ancDailySummaryConfig, Boolean isApiRequest) throws FileNotFoundException {

List<String> fhirSearchList = getFhirSearchListByFilters(filters, env);
logger.warn("Calling details page function saveQueryResult. StartDate: {} EndDate: {}", startDate, endDate);
Long start3 = System.nanoTime();
List<DataResult> dataResult = (List<DataResult>) getReportGen("getAncDailySummaryData", organizationId, startDate, endDate,
fhirSearchList, ancDailySummaryConfig);
for (String hashcode : hashcodes) {
saveInAsyncTable(dataResult.get(hashcodes.indexOf(hashcode)), hashcode);
saveInAsyncTable(dataResult.get(hashcodes.indexOf(hashcode)), hashcode, isApiRequest);
}
Long end3 = System.nanoTime();
Double diff3 = ((end3 - start3) / 1e9); // Convert nanoseconds to seconds
Expand Down
19 changes: 19 additions & 0 deletions src/main/java/com/iprd/fhir/utils/Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package com.iprd.fhir.utils;

import com.github.javafaker.Faker;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.iprd.report.FhirPath;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -573,4 +575,21 @@ public static boolean noneMatchDates(List<Date> dates, Date currentDate) {
.map(Date::toLocalDate)
.noneMatch(date -> date.equals(currentLocalDate));
}

public static String generateRandomName(){
Faker faker = new Faker();
return faker.name().fullName();
}

public static String generateRandomPatientOclId(){
return RandomStringUtils.random(12, true, true);
}

public static String generateRandomDate(){
Random random = new Random();
int minDay = (int) LocalDate.of(1900, 1, 1).toEpochDay();
int maxDay = (int) LocalDate.of(2015, 1, 1).toEpochDay();
long randomDay = minDay + random.nextInt(maxDay - minDay);
return LocalDate.ofEpochDay(randomDay).toString();
}
}
29 changes: 10 additions & 19 deletions src/test/java/ca/uhn/fhir/jpa/starter/service/HelperServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,7 @@
import ca.uhn.fhir.jpa.starter.model.CategoryItem;
import ca.uhn.fhir.jpa.starter.model.PatientIdentifierEntity;
import ca.uhn.fhir.jpa.starter.model.ScoreCardIndicatorItem;
import ca.uhn.fhir.jpa.starter.model.OrgHierarchy;
import ca.uhn.fhir.jpa.starter.model.OrgIndicatorAverageResult;
import ca.uhn.fhir.jpa.starter.model.ScoreCardResponseItem;
import ca.uhn.fhir.rest.client.api.IGenericClient;
import ca.uhn.fhir.jpa.starter.model.ReportType;
import ca.uhn.fhir.rest.gclient.IQuery;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
Expand All @@ -32,11 +28,7 @@
import com.iprd.report.model.definition.FhirPathTransformation;
import com.iprd.report.model.definition.BarComponent;
import com.iprd.report.model.definition.ANCDailySummaryConfig;
import com.iprd.report.model.data.BarChartItemDataCollection;
import com.iprd.report.model.data.LineChartItemCollection;
import com.iprd.report.model.definition.IndicatorItem;
import com.iprd.report.model.data.PieChartItemDataCollection;
import com.iprd.report.model.data.ScoreCardItem;
import org.hibernate.SessionFactory;
import org.hl7.fhir.instance.model.api.IBaseBundle;
import org.hl7.fhir.r4.model.Organization;
Expand Down Expand Up @@ -88,14 +80,13 @@
import java.util.concurrent.TimeUnit;
import static org.junit.Assert.assertEquals;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.anyList;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.times;
Expand Down Expand Up @@ -571,7 +562,7 @@ void testSaveInAsyncTable() throws SQLException {
doNothing().when(notificationDataSourceMock).update(any());
// Replace the actual instance with the mock in NotificationDataSource
Whitebox.setInternalState(NotificationDataSource.class, notificationDataSourceMock);
helperService.saveInAsyncTable(dataResult,"68455");
helperService.saveInAsyncTable(dataResult,"68455", true);
verify(notificationDataSourceMock).update(any(ApiAsyncTaskEntity.class));
// Verify the update method was called with the captured argument
ArgumentCaptor<ApiAsyncTaskEntity> argumentCaptor = ArgumentCaptor.forClass(ApiAsyncTaskEntity.class);
Expand Down Expand Up @@ -645,19 +636,19 @@ void testSaveQueryResultAndHandleException() throws FileNotFoundException, Inter
executor.initialize();
when(helperServiceMoc.getAsyncExecutor(
)).thenReturn(executor);
doNothing().when(helperServiceMoc).saveQueryResult(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList());
doNothing().when(helperServiceMoc).saveQueryResult(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList(),eq(false));
doAnswer(invocation -> {
latch.countDown();
return null;
}).when(helperServiceMoc).saveQueryResult(anyString(), anyString(), anyString(), any(), anyList(), anyString(), anyList());
}).when(helperServiceMoc).saveQueryResult(anyString(), anyString(), anyString(), any(), anyList(), anyString(), anyList(),eq(false));
// Call the method
doCallRealMethod().when(helperServiceMoc).saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig);
helperServiceMoc.saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig);
doCallRealMethod().when(helperServiceMoc).saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig, true);
helperServiceMoc.saveQueryResultAndHandleException(organizationId, startDate, endDate, filters, hashcodes, env, ancDailySummaryConfig, true);
// Wait for the asynchronous operation to complete
assertTrue(latch.await(5, TimeUnit.SECONDS));
// Verify that the saveQueryResult method was called
verify(helperServiceMoc, times(1)).saveQueryResult(
eq(organizationId), eq(startDate), eq(endDate), eq(filters), eq(hashcodes), eq(env), eq(ancDailySummaryConfig)
eq(organizationId), eq(startDate), eq(endDate), eq(filters), eq(hashcodes), eq(env), eq(ancDailySummaryConfig),eq(false)
);
executor.shutdown();
}
Expand Down Expand Up @@ -685,11 +676,11 @@ void testProcessCategories() throws FileNotFoundException {
.thenReturn(new ArrayList<ApiAsyncTaskEntity>());
helperServiceMock.datasource = notificationDataSourceMock;
doNothing().when(notificationDataSourceMock).insert(any());
doNothing().when(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList());
doNothing().when(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),anyList(), true);
doCallRealMethod().when(helperServiceMock).processCategories(organizationId,startDate,endDate,env,filters,true);
Map<String,String> categoryWithHashCodes = helperServiceMock.processCategories(organizationId,startDate,endDate,env,filters,true);
ArgumentCaptor<List<ANCDailySummaryConfig>> argumentCaptor = ArgumentCaptor.forClass(List.class);
verify(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),argumentCaptor.capture());
verify(helperServiceMock).saveQueryResultAndHandleException(anyString(),anyString(),anyString(),any(),anyList(),anyString(),argumentCaptor.capture(), true);
List<ANCDailySummaryConfig> capturedList = argumentCaptor.getValue();
assertEquals(ancDailySummaryConfig,capturedList);
assertEquals(categoryWithHashCodes.get("antenatal"),"e15b899b-8d94-4279-8cb7-3eb90a14279b2023-11-012023-11-01antenatalV2age-2");
Expand Down