From 571f82abcee1c5c0a2467b649e829110ef438ba5 Mon Sep 17 00:00:00 2001 From: SJ Date: Fri, 28 Oct 2022 22:16:06 +0530 Subject: [PATCH] feat(explore): support qs filters for eds --- .../service/explore/ExploreService.java | 17 +++- .../explore/entity/EntityRequestHandler.java | 80 ++++++++++++++++--- 2 files changed, 87 insertions(+), 10 deletions(-) diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java index 8601c765..38ff6105 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/ExploreService.java @@ -1,6 +1,12 @@ package org.hypertrace.gateway.service.explore; import static org.hypertrace.core.attribute.service.v1.AttributeSource.EDS; +import static org.hypertrace.gateway.service.common.ExpressionLocation.COLUMN_GROUP_BY; +import static org.hypertrace.gateway.service.common.ExpressionLocation.COLUMN_ORDER_BY; +import static org.hypertrace.gateway.service.common.ExpressionLocation.COLUMN_SELECTION; +import static org.hypertrace.gateway.service.common.ExpressionLocation.METRIC_AGGREGATION; +import static org.hypertrace.gateway.service.common.ExpressionLocation.METRIC_ORDER_BY; +import static org.hypertrace.gateway.service.common.ExpressionLocation.TIME_AGGREGATION; import com.google.common.collect.ImmutableMap; import io.micrometer.core.instrument.Timer; @@ -9,6 +15,7 @@ import java.util.Arrays; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import org.hypertrace.core.attribute.service.v1.AttributeMetadata; import org.hypertrace.core.serviceframework.metrics.PlatformMetricsRegistry; @@ -120,7 +127,15 @@ private IRequestHandler getRequestHandler( && !hasTimeAggregations(request) && !request.getGroupByList().isEmpty()) { Optional source = - ExpressionContext.getSingleSourceForAllAttributes(expressionContext); + ExpressionContext.getSingleSourceForAttributes( + expressionContext, + Set.of( + COLUMN_SELECTION, + METRIC_AGGREGATION, + TIME_AGGREGATION, + COLUMN_ORDER_BY, + METRIC_ORDER_BY, + COLUMN_GROUP_BY)); if ((source.isPresent() && EDS.toString().equals(source.get())) || !hasTimeRange(request)) { return entityRequestHandler; } diff --git a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java index 77faaefd..a97ff153 100644 --- a/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java +++ b/gateway-service-impl/src/main/java/org/hypertrace/gateway/service/explore/entity/EntityRequestHandler.java @@ -3,10 +3,13 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Streams; import java.util.Iterator; +import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import org.hypertrace.core.attribute.service.v1.AttributeMetadata; +import org.hypertrace.core.attribute.service.v1.AttributeSource; import org.hypertrace.entity.query.service.client.EntityQueryServiceClient; import org.hypertrace.entity.query.service.v1.ColumnMetadata; import org.hypertrace.entity.query.service.v1.ResultSetChunk; @@ -19,13 +22,16 @@ import org.hypertrace.gateway.service.common.datafetcher.EntityFetcherResponse; import org.hypertrace.gateway.service.common.datafetcher.QueryServiceEntityFetcher; import org.hypertrace.gateway.service.common.util.AttributeMetadataUtil; +import org.hypertrace.gateway.service.common.util.ExpressionReader; import org.hypertrace.gateway.service.common.util.MetricAggregationFunctionUtil; import org.hypertrace.gateway.service.common.util.QueryServiceClient; import org.hypertrace.gateway.service.entity.EntitiesRequestContext; import org.hypertrace.gateway.service.entity.config.EntityIdColumnsConfigs; import org.hypertrace.gateway.service.explore.ExploreRequestContext; import org.hypertrace.gateway.service.explore.RequestHandler; +import org.hypertrace.gateway.service.v1.common.Filter; import org.hypertrace.gateway.service.v1.common.FunctionExpression; +import org.hypertrace.gateway.service.v1.common.Operator; import org.hypertrace.gateway.service.v1.entity.EntitiesRequest; import org.hypertrace.gateway.service.v1.entity.Entity.Builder; import org.hypertrace.gateway.service.v1.explore.ExploreRequest; @@ -92,15 +98,24 @@ public ExploreResponse.Builder handleRequest( requestContext.setHasGroupBy(true); } - Set entityIds = getEntityIds(requestContext, request); + Map attributeMetadataMap = + attributeMetadataProvider.getAttributesMetadata(requestContext, request.getContext()); + + Set entityIds = getEntityIds(requestContext, expressionContext, attributeMetadataMap); ExploreResponse.Builder builder = ExploreResponse.newBuilder(); if (entityIds.isEmpty()) { return builder; } + Optional maybeEdsFilter = + buildFilter(request.getFilter(), AttributeSource.EDS, attributeMetadataMap); + ExploreRequest.Builder exploreEntityRequestBuilder = + ExploreRequest.newBuilder(request).clearFilter(); + maybeEdsFilter.ifPresent(exploreEntityRequestBuilder::setFilter); Iterator resultSetChunkIterator = - entityServiceEntityFetcher.getResults(requestContext, request, entityIds); + entityServiceEntityFetcher.getResults( + requestContext, exploreEntityRequestBuilder.build(), entityIds); while (resultSetChunkIterator.hasNext()) { org.hypertrace.entity.query.service.v1.ResultSetChunk chunk = resultSetChunkIterator.next(); @@ -144,22 +159,69 @@ public ExploreResponse.Builder handleRequest( } private Set getEntityIds( - ExploreRequestContext requestContext, ExploreRequest exploreRequest) { + ExploreRequestContext requestContext, + ExpressionContext expressionContext, + Map attributeMetadataMap) { + ExploreRequest request = requestContext.getRequest(); EntitiesRequestContext entitiesRequestContext = convert(attributeMetadataProvider, requestContext); - EntitiesRequest entitiesRequest = + EntitiesRequest.Builder entitiesRequestBuilder = EntitiesRequest.newBuilder() - .setEntityType(exploreRequest.getContext()) - .setStartTimeMillis(exploreRequest.getStartTimeMillis()) - .setEndTimeMillis(exploreRequest.getEndTimeMillis()) - .build(); + .setEntityType(request.getContext()) + .setStartTimeMillis(request.getStartTimeMillis()) + .setEndTimeMillis(request.getEndTimeMillis()); + // build qs filter from filter in the request + buildFilter(request.getFilter(), AttributeSource.QS, attributeMetadataMap) + .ifPresent(entitiesRequestBuilder::setFilter); EntityFetcherResponse response = - queryServiceEntityFetcher.getEntities(entitiesRequestContext, entitiesRequest); + queryServiceEntityFetcher.getEntities( + entitiesRequestContext, entitiesRequestBuilder.build()); return response.getEntityKeyBuilderMap().values().stream() .map(Builder::getId) .collect(Collectors.toUnmodifiableSet()); } + private Optional buildFilter( + Filter filter, AttributeSource source, Map attributeMetadataMap) { + if (filter.equals(Filter.getDefaultInstance())) { + return Optional.empty(); + } + + Operator operator = filter.getOperator(); + switch (operator) { + case AND: + case OR: + return buildCompositeFilter(filter, source, operator, attributeMetadataMap); + default: + List sources = + attributeMetadataMap + .get( + ExpressionReader.getAttributeIdFromAttributeSelection(filter.getLhs()) + .orElseThrow()) + .getSourcesList(); + return sources.contains(source) + ? Optional.of(Filter.newBuilder(filter).build()) + : Optional.empty(); + } + } + + private Optional buildCompositeFilter( + Filter filter, + AttributeSource source, + Operator operator, + Map attributeMetadataMap) { + Filter.Builder filterBuilder = Filter.newBuilder(); + for (Filter childFilter : filter.getChildFilterList()) { + buildFilter(childFilter, source, attributeMetadataMap).map(filterBuilder::addChildFilter); + } + if (filterBuilder.getChildFilterCount() > 0) { + filterBuilder.setOperator(operator); + return Optional.of(filterBuilder.build()); + } else { + return Optional.empty(); + } + } + private EntitiesRequestContext convert( AttributeMetadataProvider attributeMetadataProvider, ExploreRequestContext requestContext) { String entityType = requestContext.getContext();