From 1d6870fbb87333dd2e058fd7fac5bdba74c47a19 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 23 Aug 2017 12:12:44 -0700 Subject: [PATCH 01/20] implement bucketing for rollouts --- .../ab/bucketing/DecisionService.java | 38 +++++++++++++++++++ .../optimizely/ab/config/ProjectConfig.java | 7 ++++ 2 files changed, 45 insertions(+) diff --git a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java index 50b9241b8..167d59351 100644 --- a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java +++ b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java @@ -20,7 +20,9 @@ import com.optimizely.ab.config.Experiment; import com.optimizely.ab.config.FeatureFlag; import com.optimizely.ab.config.ProjectConfig; +import com.optimizely.ab.config.Rollout; import com.optimizely.ab.config.Variation; +import com.optimizely.ab.config.audience.Audience; import com.optimizely.ab.error.ErrorHandler; import com.optimizely.ab.internal.ExperimentUtils; import org.slf4j.Logger; @@ -165,6 +167,42 @@ public DecisionService(@Nonnull Bucketer bucketer, logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in any experiments."); } + // use rollout to get variation for feature + if (!featureFlag.getRolloutId().isEmpty()) { + Rollout rollout = projectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); + + for (Experiment experiment : rollout.getExperiments()) { + Audience audience = projectConfig.getAudienceIdMapping().get(experiment.getAudienceIds().get(0)); + String audienceName = "Everyone Else"; + if (audience != null) { + audienceName = audience.getName(); + } + if (!experiment.isActive()) { + logger.info("Did not attempt to bucket user into rollout rule for audience \"" + + audienceName + "\" since the rule is not active."); + } + else if (ExperimentUtils.isUserInExperiment(projectConfig, experiment, filteredAttributes)) { + logger.info("Attempting to bucket user \"" + userId + + "\" into rollout rule for audience \"" + audienceName + + "\"."); + Variation variation = bucketer.bucket(experiment, userId); + if (variation == null) { + logger.info("User \"" + userId + + "\" was excluded due to traffic allocation."); + } + return variation; + } + else { + logger.info("User \"" + userId + + "\" did not meet the conditions to be in rollout rule for audience \"" + audienceName + + "\"."); + } + } + } + else { + logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in any experiments."); + } + return null; } diff --git a/core-api/src/main/java/com/optimizely/ab/config/ProjectConfig.java b/core-api/src/main/java/com/optimizely/ab/config/ProjectConfig.java index dc68400c8..539703176 100644 --- a/core-api/src/main/java/com/optimizely/ab/config/ProjectConfig.java +++ b/core-api/src/main/java/com/optimizely/ab/config/ProjectConfig.java @@ -87,6 +87,7 @@ public String toString() { private final Map audienceIdMapping; private final Map experimentIdMapping; private final Map groupIdMapping; + private final Map rolloutIdMapping; // other mappings private final Map> liveVariableIdToExperimentsMapping; @@ -192,6 +193,7 @@ public ProjectConfig(String accountId, this.audienceIdMapping = ProjectConfigUtils.generateIdMapping(audiences); this.experimentIdMapping = ProjectConfigUtils.generateIdMapping(this.experiments); this.groupIdMapping = ProjectConfigUtils.generateIdMapping(groups); + this.rolloutIdMapping = ProjectConfigUtils.generateIdMapping(this.rollouts); if (liveVariables == null) { this.liveVariables = null; @@ -318,6 +320,10 @@ public Map getGroupIdMapping() { return groupIdMapping; } + public Map getRolloutIdMapping() { + return rolloutIdMapping; + } + public Map getLiveVariableKeyMapping() { return liveVariableKeyMapping; } @@ -488,6 +494,7 @@ public String toString() { ", audienceIdMapping=" + audienceIdMapping + ", experimentIdMapping=" + experimentIdMapping + ", groupIdMapping=" + groupIdMapping + + ", rolloutIdMapping=" + rolloutIdMapping + ", liveVariableIdToExperimentsMapping=" + liveVariableIdToExperimentsMapping + ", variationToLiveVariableUsageInstanceMapping=" + variationToLiveVariableUsageInstanceMapping + ", forcedVariationMapping=" + forcedVariationMapping + From 45365016e278e3b66229640c0260df9eb9457c26 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Thu, 24 Aug 2017 11:23:34 -0700 Subject: [PATCH 02/20] log variable name --- core-api/src/main/java/com/optimizely/ab/Optimizely.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-api/src/main/java/com/optimizely/ab/Optimizely.java b/core-api/src/main/java/com/optimizely/ab/Optimizely.java index 090fa209a..cf8c23f41 100644 --- a/core-api/src/main/java/com/optimizely/ab/Optimizely.java +++ b/core-api/src/main/java/com/optimizely/ab/Optimizely.java @@ -562,7 +562,7 @@ else if (!variable.getType().equals(variableType)) { else { logger.info("User \"" + userId + "\" was not bucketed into any variation for feature flag \"" + featureKey + - "\". The default value is being returned." + "\". The default value for \"" + variableKey + "\" is being returned." ); } From 48df910b5807c8d47752b862a6ddc1d41bca5fc8 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Thu, 24 Aug 2017 11:24:02 -0700 Subject: [PATCH 03/20] change bucketing to expect an everyone else audience for all feature flags --- .../com/optimizely/ab/bucketing/DecisionService.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java index 167d59351..7478f1ba2 100644 --- a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java +++ b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java @@ -173,17 +173,13 @@ public DecisionService(@Nonnull Bucketer bucketer, for (Experiment experiment : rollout.getExperiments()) { Audience audience = projectConfig.getAudienceIdMapping().get(experiment.getAudienceIds().get(0)); - String audienceName = "Everyone Else"; - if (audience != null) { - audienceName = audience.getName(); - } if (!experiment.isActive()) { logger.info("Did not attempt to bucket user into rollout rule for audience \"" + - audienceName + "\" since the rule is not active."); + audience.getName() + "\" since the rule is not active."); } else if (ExperimentUtils.isUserInExperiment(projectConfig, experiment, filteredAttributes)) { logger.info("Attempting to bucket user \"" + userId + - "\" into rollout rule for audience \"" + audienceName + + "\" into rollout rule for audience \"" + audience.getName() + "\"."); Variation variation = bucketer.bucket(experiment, userId); if (variation == null) { @@ -194,7 +190,7 @@ else if (ExperimentUtils.isUserInExperiment(projectConfig, experiment, filteredA } else { logger.info("User \"" + userId + - "\" did not meet the conditions to be in rollout rule for audience \"" + audienceName + + "\" did not meet the conditions to be in rollout rule for audience \"" + audience.getName() + "\"."); } } From d439e4eb290650d6bef5485cf0d01264397dcd5f Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Tue, 29 Aug 2017 12:01:33 -0700 Subject: [PATCH 04/20] change and abstract rollout bucketing --- .../ab/bucketing/DecisionService.java | 39 +++++++++++++++---- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java index 7478f1ba2..3b5af645d 100644 --- a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java +++ b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java @@ -167,24 +167,44 @@ public DecisionService(@Nonnull Bucketer bucketer, logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in any experiments."); } + return getVariationForFeatureInRollout(featureFlag, userId, filteredAttributes); + } + + /** + * Try to bucket the user into a rollout rule. + * Evaluate the user for rules in priority order by seeing if the user satisfies the audience. + * Fall back onto the everyone else rule if the user is ever excluded from a rule due to traffic allocation. + * @param featureFlag The feature flag the user wants to access. + * @param userId User Identifier + * @param filteredAttributes A map of filtered attributes. + * @return null if the user is not bucketed into the rollout or if the feature flag was not attached to a rollout. + * {@link Variation} the user is bucketed into fi the user is successfully bucketed. + */ + @Nullable Variation getVariationForFeatureInRollout(@Nonnull FeatureFlag featureFlag, + @Nonnull String userId, + @Nonnull Map filteredAttributes) { // use rollout to get variation for feature if (!featureFlag.getRolloutId().isEmpty()) { Rollout rollout = projectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); - - for (Experiment experiment : rollout.getExperiments()) { - Audience audience = projectConfig.getAudienceIdMapping().get(experiment.getAudienceIds().get(0)); - if (!experiment.isActive()) { + int rolloutRulesLength = rollout.getExperiments().size(); + Variation variation; + // for all rules before the everyone else rule + for (int i = 0; i < rolloutRulesLength - 1; i++) { + Experiment rolloutRule= rollout.getExperiments().get(i); + Audience audience = projectConfig.getAudienceIdMapping().get(rolloutRule.getAudienceIds().get(0)); + if (!rolloutRule.isActive()) { logger.info("Did not attempt to bucket user into rollout rule for audience \"" + audience.getName() + "\" since the rule is not active."); } - else if (ExperimentUtils.isUserInExperiment(projectConfig, experiment, filteredAttributes)) { + else if (ExperimentUtils.isUserInExperiment(projectConfig, rolloutRule, filteredAttributes)) { logger.info("Attempting to bucket user \"" + userId + "\" into rollout rule for audience \"" + audience.getName() + "\"."); - Variation variation = bucketer.bucket(experiment, userId); + variation = bucketer.bucket(rolloutRule, userId); if (variation == null) { logger.info("User \"" + userId + "\" was excluded due to traffic allocation."); + break; } return variation; } @@ -194,11 +214,14 @@ else if (ExperimentUtils.isUserInExperiment(projectConfig, experiment, filteredA "\"."); } } + // get last rule which is the everyone else rule + Experiment everyoneElseRule = rollout.getExperiments().get(rolloutRulesLength - 1); + variation = bucketer.bucket(everyoneElseRule, userId); // ignore audience + return variation; } else { - logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in any experiments."); + logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in a rollout."); } - return null; } From 5da01a5e71e5e5caf89f8343e4ed0b9f3e474240 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Tue, 29 Aug 2017 14:09:31 -0700 Subject: [PATCH 05/20] address PR from Mike --- .../ab/bucketing/DecisionService.java | 73 ++++++++++--------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java index 3b5af645d..12cf3c74c 100644 --- a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java +++ b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java @@ -184,45 +184,48 @@ public DecisionService(@Nonnull Bucketer bucketer, @Nonnull String userId, @Nonnull Map filteredAttributes) { // use rollout to get variation for feature - if (!featureFlag.getRolloutId().isEmpty()) { - Rollout rollout = projectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); - int rolloutRulesLength = rollout.getExperiments().size(); - Variation variation; - // for all rules before the everyone else rule - for (int i = 0; i < rolloutRulesLength - 1; i++) { - Experiment rolloutRule= rollout.getExperiments().get(i); - Audience audience = projectConfig.getAudienceIdMapping().get(rolloutRule.getAudienceIds().get(0)); - if (!rolloutRule.isActive()) { - logger.info("Did not attempt to bucket user into rollout rule for audience \"" + - audience.getName() + "\" since the rule is not active."); - } - else if (ExperimentUtils.isUserInExperiment(projectConfig, rolloutRule, filteredAttributes)) { - logger.info("Attempting to bucket user \"" + userId + - "\" into rollout rule for audience \"" + audience.getName() + - "\"."); - variation = bucketer.bucket(rolloutRule, userId); - if (variation == null) { - logger.info("User \"" + userId + - "\" was excluded due to traffic allocation."); - break; - } - return variation; - } - else { - logger.info("User \"" + userId + - "\" did not meet the conditions to be in rollout rule for audience \"" + audience.getName() + - "\"."); + if (featureFlag.getRolloutId().isEmpty()) { + logger.debug("The feature flag \"" + featureFlag.getKey() + "\" is not used in a rollout."); + return null; + } + Rollout rollout = projectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); + int rolloutRulesLength = rollout.getExperiments().size(); + Variation variation; + // for all rules before the everyone else rule + for (int i = 0; i < rolloutRulesLength - 1; i++) { + Experiment rolloutRule= rollout.getExperiments().get(i); + Audience audience = projectConfig.getAudienceIdMapping().get(rolloutRule.getAudienceIds().get(0)); + if (!rolloutRule.isActive()) { + logger.debug("Did not attempt to bucket user into rollout rule for audience \"" + + audience.getName() + "\" since the rule is not active."); + } + else if (ExperimentUtils.isUserInExperiment(projectConfig, rolloutRule, filteredAttributes)) { + logger.debug("Attempting to bucket user \"" + userId + + "\" into rollout rule for audience \"" + audience.getName() + + "\"."); + variation = bucketer.bucket(rolloutRule, userId); + if (variation == null) { + logger.debug("User \"" + userId + + "\" was excluded due to traffic allocation."); + break; } + return variation; + } + else { + logger.debug("User \"" + userId + + "\" did not meet the conditions to be in rollout rule for audience \"" + audience.getName() + + "\"."); } - // get last rule which is the everyone else rule - Experiment everyoneElseRule = rollout.getExperiments().get(rolloutRulesLength - 1); - variation = bucketer.bucket(everyoneElseRule, userId); // ignore audience - return variation; } - else { - logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in a rollout."); + // get last rule which is the everyone else rule + Experiment everyoneElseRule = rollout.getExperiments().get(rolloutRulesLength - 1); + variation = bucketer.bucket(everyoneElseRule, userId); // ignore audience + if (variation == null) { + logger.debug("User \"" + userId + + "\" was excluded from the \"Everyone Else\" rule for feature flag \"" + featureFlag.getKey() + + "\"."); } - return null; + return variation; } /** From 6a8f8b819a9413ee68bae204e65abae5798f63de Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Tue, 29 Aug 2017 15:39:04 -0700 Subject: [PATCH 06/20] fix existing unit tests --- .../ab/bucketing/DecisionService.java | 2 +- .../com/optimizely/ab/OptimizelyTest.java | 40 +++++++-- .../ab/bucketing/DecisionServiceTest.java | 22 +++-- .../ab/config/ValidProjectConfigV4.java | 82 +++++++++---------- 4 files changed, 91 insertions(+), 55 deletions(-) diff --git a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java index 12cf3c74c..5739bdbb1 100644 --- a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java +++ b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java @@ -185,7 +185,7 @@ public DecisionService(@Nonnull Bucketer bucketer, @Nonnull Map filteredAttributes) { // use rollout to get variation for feature if (featureFlag.getRolloutId().isEmpty()) { - logger.debug("The feature flag \"" + featureFlag.getKey() + "\" is not used in a rollout."); + logger.info("The feature flag \"" + featureFlag.getKey() + "\" is not used in a rollout."); return null; } Rollout rollout = projectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); diff --git a/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java b/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java index 82fbbf8de..4caeef924 100644 --- a/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java +++ b/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java @@ -81,9 +81,12 @@ import static com.optimizely.ab.config.ValidProjectConfigV4.EXPERIMENT_PAUSED_EXPERIMENT_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_MULTI_VARIATE_FEATURE_KEY; +import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_SINGLE_VARIABLE_BOOLEAN_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_SINGLE_VARIABLE_STRING_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.MULTIVARIATE_EXPERIMENT_FORCED_VARIATION_USER_ID_GRED; import static com.optimizely.ab.config.ValidProjectConfigV4.PAUSED_EXPERIMENT_FORCED_VARIATION_USER_ID_CONTROL; +import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_BOOLEAN_VARIABLE_DEFAULT_VALUE; +import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_BOOLEAN_VARIABLE_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_FIRST_LETTER_DEFAULT_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_FIRST_LETTER_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_STRING_VARIABLE_DEFAULT_VALUE; @@ -2504,16 +2507,16 @@ public void getFeatureVariableValueReturnsNullWhenVariableTypeDoesNotMatch() thr /** * Verify {@link Optimizely#getFeatureVariableValueForType(String, String, String, Map, LiveVariable.VariableType)} * returns the String default value of a live variable - * when the feature is not attached to an experiment. + * when the feature is not attached to an experiment or a rollout. * @throws ConfigParseException */ @Test - public void getFeatureVariableValueForTypeReturnsDefaultValueWhenFeatureIsNotAttached() throws ConfigParseException { + public void getFeatureVariableValueForTypeReturnsDefaultValueWhenFeatureIsNotAttachedToExperimentOrRollout() throws ConfigParseException { assumeTrue(datafileVersion >= Integer.parseInt(ProjectConfig.Version.V4.toString())); - String validFeatureKey = FEATURE_SINGLE_VARIABLE_STRING_KEY; - String validVariableKey = VARIABLE_STRING_VARIABLE_KEY; - String defaultValue = VARIABLE_STRING_VARIABLE_DEFAULT_VALUE; + String validFeatureKey = FEATURE_SINGLE_VARIABLE_BOOLEAN_KEY; + String validVariableKey = VARIABLE_BOOLEAN_VARIABLE_KEY; + String defaultValue = VARIABLE_BOOLEAN_VARIABLE_DEFAULT_VALUE; Map attributes = Collections.singletonMap(ATTRIBUTE_HOUSE_KEY, AUDIENCE_GRYFFINDOR_VALUE); Optimizely optimizely = Optimizely.builder(validDatafile, mockEventHandler) @@ -2525,19 +2528,29 @@ public void getFeatureVariableValueForTypeReturnsDefaultValueWhenFeatureIsNotAtt validVariableKey, genericUserId, attributes, - LiveVariable.VariableType.STRING); + LiveVariable.VariableType.BOOLEAN); assertEquals(defaultValue, value); logbackVerifier.expectMessage( Level.INFO, "The feature flag \"" + validFeatureKey + "\" is not used in any experiments." ); + logbackVerifier.expectMessage( + Level.INFO, + "The feature flag \"" + validFeatureKey + "\" is not used in a rollout." + ); + logbackVerifier.expectMessage( + Level.INFO, + "User \"" + genericUserId + "\" was not bucketed into any variation for feature flag \"" + + validFeatureKey + "\". The default value for \"" + + validVariableKey + "\" is being returned." + ); } /** * Verify {@link Optimizely#getFeatureVariableValueForType(String, String, String, Map, LiveVariable.VariableType)} * returns the String default value for a live variable - * when the feature is attached to an experiment, but the user is excluded from the experiment. + * when the feature is attached to an experiment and no rollout, but the user is excluded from the experiment. * @throws ConfigParseException */ @Test @@ -2547,6 +2560,8 @@ public void getFeatureVariableValueReturnsDefaultValueWhenFeatureIsAttachedToOne String validFeatureKey = FEATURE_MULTI_VARIATE_FEATURE_KEY; String validVariableKey = VARIABLE_FIRST_LETTER_KEY; String expectedValue = VARIABLE_FIRST_LETTER_DEFAULT_VALUE; + FeatureFlag featureFlag = FEATURE_FLAG_MULTI_VARIATE_FEATURE; + Experiment experiment = validProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); Optimizely optimizely = Optimizely.builder(validDatafile, mockEventHandler) .withConfig(validProjectConfig) @@ -2561,11 +2576,20 @@ public void getFeatureVariableValueReturnsDefaultValueWhenFeatureIsAttachedToOne ); assertEquals(expectedValue, valueWithImproperAttributes); + logbackVerifier.expectMessage( + Level.INFO, + "User \"" + genericUserId + "\" does not meet conditions to be in experiment \"" + + experiment.getKey() + "\"." + ); + logbackVerifier.expectMessage( + Level.INFO, + "The feature flag \"" + validFeatureKey + "\" is not used in a rollout." + ); logbackVerifier.expectMessage( Level.INFO, "User \"" + genericUserId + "\" was not bucketed into any variation for feature flag \"" + validFeatureKey + - "\". The default value is being returned." + "\". The default value for \"" + validVariableKey + "\" is being returned." ); } diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index eef46cc5a..fe0ffc55e 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -283,7 +283,7 @@ public void getVariationOnNonRunningExperimentWithForcedVariation() { /** * Verify that {@link DecisionService#getVariationForFeature(FeatureFlag, String, Map)} - * returns null when the {@link FeatureFlag} is not used in an experiments. + * returns null when the {@link FeatureFlag} is not used in an experiments or rollouts. */ @Test @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") @@ -292,6 +292,7 @@ public void getVariationForFeatureReturnsNullWhenFeatureFlagExperimentIdsIsEmpty when(emptyFeatureFlag.getExperimentIds()).thenReturn(Collections.emptyList()); String featureKey = "testFeatureFlagKey"; when(emptyFeatureFlag.getKey()).thenReturn(featureKey); + when(emptyFeatureFlag.getRolloutId()).thenReturn(""); DecisionService decisionService = new DecisionService( mock(Bucketer.class), @@ -300,7 +301,9 @@ public void getVariationForFeatureReturnsNullWhenFeatureFlagExperimentIdsIsEmpty null); logbackVerifier.expectMessage(Level.INFO, - "The feature flag \"" + featureKey + "\" is not used in any experiments"); + "The feature flag \"" + featureKey + "\" is not used in any experiments."); + logbackVerifier.expectMessage(Level.INFO, + "The feature flag \"" + featureKey + "\" is not used in a rollout."); assertNull(decisionService.getVariationForFeature( emptyFeatureFlag, @@ -308,16 +311,17 @@ public void getVariationForFeatureReturnsNullWhenFeatureFlagExperimentIdsIsEmpty Collections.emptyMap())); verify(emptyFeatureFlag, times(1)).getExperimentIds(); - verify(emptyFeatureFlag, times(1)).getKey(); + verify(emptyFeatureFlag, times(1)).getRolloutId(); + verify(emptyFeatureFlag, times(2)).getKey(); } /** * Verify that {@link DecisionService#getVariationForFeature(FeatureFlag, String, Map)} - * returns null when the user is not bucketed into any experiments for the {@link FeatureFlag}. + * returns null when the user is not bucketed into any experiments or rollouts for the {@link FeatureFlag}. */ @Test @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") - public void getVariationForFeatureReturnsNullWhenItGetsNoVariationsForExperiments() { + public void getVariationForFeatureReturnsNullWhenItGetsNoVariationsForExperimentsAndRollouts() { FeatureFlag spyFeatureFlag = spy(ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE); DecisionService spyDecisionService = spy(new DecisionService( @@ -327,12 +331,20 @@ public void getVariationForFeatureReturnsNullWhenItGetsNoVariationsForExperiment null) ); + // do not bucket to any experiments doReturn(null).when(spyDecisionService).getVariation( any(Experiment.class), anyString(), anyMapOf(String.class, String.class) ); + // do not bucket to any rollouts + doReturn(null).when(spyDecisionService).getVariationForFeatureInRollout( + any(FeatureFlag.class), + anyString(), + anyMapOf(String.class, String.class) + ); + // try to get a variation back from the decision service for the feature flag assertNull(spyDecisionService.getVariationForFeature( spyFeatureFlag, genericUserId, diff --git a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java index b073b04d6..ccbb07196 100644 --- a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java +++ b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java @@ -108,10 +108,10 @@ public class ValidProjectConfigV4 { ) ); private static final String FEATURE_SINGLE_VARIABLE_BOOLEAN_ID = "2591051011"; - private static final String FEATURE_SINGLE_VARIABLE_BOOLEAN_KEY = "boolean_single_variable_feature"; + public static final String FEATURE_SINGLE_VARIABLE_BOOLEAN_KEY = "boolean_single_variable_feature"; private static final String VARIABLE_BOOLEAN_VARIABLE_ID = "3974680341"; - private static final String VARIABLE_BOOLEAN_VARIABLE_KEY = "boolean_variable"; - private static final String VARIABLE_BOOLEAN_VARIABLE_DEFAULT_VALUE = "true"; + public static final String VARIABLE_BOOLEAN_VARIABLE_KEY = "boolean_variable"; + public static final String VARIABLE_BOOLEAN_VARIABLE_DEFAULT_VALUE = "true"; private static final LiveVariable VARIABLE_BOOLEAN_VARIABLE = new LiveVariable( VARIABLE_BOOLEAN_VARIABLE_ID, VARIABLE_BOOLEAN_VARIABLE_KEY, @@ -140,10 +140,47 @@ public class ValidProjectConfigV4 { null, LiveVariable.VariableType.STRING ); + private static final String ROLLOUT_1_ID = "1058508303"; + private static final String ROLLOUT_1_EVERYONE_ELSE_EXPERIMENT_ID = "1785077004"; + private static final String ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID = "1566407342"; + private static final String ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_STRING_VALUE = "lumos"; + private static final Variation ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION = new Variation( + ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID, + ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID, + Collections.singletonList( + new LiveVariableUsageInstance( + VARIABLE_STRING_VARIABLE_ID, + ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_STRING_VALUE + ) + ) + ); + private static final Experiment ROLLOUT_1_EVERYONE_ELSE_RULE = new Experiment( + ROLLOUT_1_EVERYONE_ELSE_EXPERIMENT_ID, + ROLLOUT_1_EVERYONE_ELSE_EXPERIMENT_ID, + Experiment.ExperimentStatus.RUNNING.toString(), + ROLLOUT_1_ID, + Collections.emptyList(), + Collections.singletonList( + ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION + ), + Collections.emptyMap(), + Collections.singletonList( + new TrafficAllocation( + ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID, + 5000 + ) + ) + ); + private static final Rollout ROLLOUT_1 = new Rollout( + ROLLOUT_1_ID, + Collections.singletonList( + ROLLOUT_1_EVERYONE_ELSE_RULE + ) + ); private static final FeatureFlag FEATURE_FLAG_SINGLE_VARIABLE_STRING = new FeatureFlag( FEATURE_SINGLE_VARIABLE_STRING_ID, FEATURE_SINGLE_VARIABLE_STRING_KEY, - "1058508303", + ROLLOUT_1_ID, Collections.emptyList(), Collections.singletonList( VARIABLE_STRING_VARIABLE @@ -667,43 +704,6 @@ public class ValidProjectConfigV4 { ) ); - private static final String ROLLOUT_1_ID = "1058508303"; - private static final String ROLLOUT_1_EVERYONE_ELSE_EXPERIMENT_ID = "1785077004"; - private static final String ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID = "1566407342"; - private static final String ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_STRING_VALUE = "lumos"; - private static final Variation ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION = new Variation( - ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID, - ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID, - Collections.singletonList( - new LiveVariableUsageInstance( - VARIABLE_STRING_VARIABLE_ID, - ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_STRING_VALUE - ) - ) - ); - private static final Experiment ROLLOUT_1_EVERYONE_ELSE_RULE = new Experiment( - ROLLOUT_1_EVERYONE_ELSE_EXPERIMENT_ID, - ROLLOUT_1_EVERYONE_ELSE_EXPERIMENT_ID, - Experiment.ExperimentStatus.RUNNING.toString(), - ROLLOUT_1_ID, - Collections.emptyList(), - Collections.singletonList( - ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION - ), - Collections.emptyMap(), - Collections.singletonList( - new TrafficAllocation( - ROLLOUT_1_EVERYONE_ELSE_RULE_ENABLED_VARIATION_ID, - 5000 - ) - ) - ); - private static final Rollout ROLLOUT_1 = new Rollout( - ROLLOUT_1_ID, - Collections.singletonList( - ROLLOUT_1_EVERYONE_ELSE_RULE - ) - ); public static ProjectConfig generateValidProjectConfigV4() { From 8f4569a8bd0368a7b76f4ee6fd9dabb3c17be96f Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Tue, 29 Aug 2017 16:09:42 -0700 Subject: [PATCH 07/20] add test to make sure getVariationForFeatuerInRolloutR returns null when there is no rollout attached to the feature flag --- .../ab/bucketing/DecisionServiceTest.java | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index fe0ffc55e..fe04f0c4d 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -20,6 +20,7 @@ import com.optimizely.ab.config.Experiment; import com.optimizely.ab.config.FeatureFlag; import com.optimizely.ab.config.ProjectConfig; +import com.optimizely.ab.config.Rollout; import com.optimizely.ab.config.TrafficAllocation; import com.optimizely.ab.config.ValidProjectConfigV4; import com.optimizely.ab.config.Variation; @@ -394,6 +395,40 @@ public void getVariationForFeatureReturnsVariationReturnedFromGetVarition() { verify(spyFeatureFlag, never()).getKey(); } + //========== getVariationForFeatureInRollout tests ==========// + + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * returns null when trying to bucket a user into a {@link FeatureFlag} + * that does not have a {@link Rollout} attached. + */ + @Test + public void getVariationForFeatureInRolloutReturnsNullWhenFeatureIsNotAttachedToRollout() { + FeatureFlag mockFeatureFlag = mock(FeatureFlag.class); + when(mockFeatureFlag.getRolloutId()).thenReturn(""); + String featureKey = "featureKey"; + when(mockFeatureFlag.getKey()).thenReturn(featureKey); + + DecisionService decisionService = new DecisionService( + mock(Bucketer.class), + mockErrorHandler, + validProjectConfig, + null + ); + + assertNull(decisionService.getVariationForFeatureInRollout( + mockFeatureFlag, + genericUserId, + Collections.emptyMap() + )); + + logbackVerifier.expectMessage( + Level.INFO, + "The feature flag \"" + featureKey + "\" is not used in a rollout." + ); + } + + //========= white list tests ==========/ /** From 8b4bf73f77ebe42cd88cfb17e5ad198f7a1935d7 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Tue, 29 Aug 2017 16:35:23 -0700 Subject: [PATCH 08/20] add test to make sure getVariationForFeatureInRollout returns null when the user is excluded from all rollout rules due to traffic allocation --- .../ab/bucketing/DecisionService.java | 6 ++++ .../ab/bucketing/DecisionServiceTest.java | 36 +++++++++++++++++++ .../ab/config/ValidProjectConfigV4.java | 4 +-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java index 5739bdbb1..ef99337b9 100644 --- a/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java +++ b/core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java @@ -189,6 +189,12 @@ public DecisionService(@Nonnull Bucketer bucketer, return null; } Rollout rollout = projectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); + if (rollout == null) { + logger.error("The rollout with id \"" + featureFlag.getRolloutId() + + "\" was not able to be found in the datafile for feature flag \"" + featureFlag.getKey() + + "\"."); + return null; + } int rolloutRulesLength = rollout.getExperiments().size(); Variation variation; // for all rules before the everyone else rule diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index fe04f0c4d..8a4df6dc4 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -43,6 +43,10 @@ import static com.optimizely.ab.config.ProjectConfigTestUtils.validConfigJsonV2; import static com.optimizely.ab.config.ProjectConfigTestUtils.validProjectConfigV3; import static com.optimizely.ab.config.ProjectConfigTestUtils.validProjectConfigV4; +import static com.optimizely.ab.config.ValidProjectConfigV4.ATTRIBUTE_HOUSE_KEY; +import static com.optimizely.ab.config.ValidProjectConfigV4.AUDIENCE_GRYFFINDOR_VALUE; +import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_SINGLE_VARIABLE_STRING; +import static com.optimizely.ab.config.ValidProjectConfigV4.ROLLOUT_1; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -53,6 +57,7 @@ import static org.mockito.Matchers.anyMapOf; import static org.mockito.Matchers.anyString; import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.atMost; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; @@ -76,6 +81,7 @@ public class DecisionServiceTest { @Mock private ErrorHandler mockErrorHandler; private static ProjectConfig noAudienceProjectConfig; + private static ProjectConfig v4ProjectConfig; private static ProjectConfig validProjectConfig; private static Experiment whitelistedExperiment; private static Variation whitelistedVariation; @@ -83,6 +89,7 @@ public class DecisionServiceTest { @BeforeClass public static void setUp() throws Exception { validProjectConfig = validProjectConfigV3(); + v4ProjectConfig = validProjectConfigV4(); noAudienceProjectConfig = noAudienceProjectConfigV3(); whitelistedExperiment = validProjectConfig.getExperimentIdMapping().get("223"); whitelistedVariation = whitelistedExperiment.getVariationKeyToVariationMap().get("vtag1"); @@ -428,6 +435,35 @@ public void getVariationForFeatureInRolloutReturnsNullWhenFeatureIsNotAttachedTo ); } + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * return null when a user is excluded from every rule of a rollout due to traffic allocation. + */ + @Test + public void getVariationForFeatureInRolloutReturnsNullWhenUserIsExcludedFromAllTraffic() { + Bucketer mockBucketer = mock(Bucketer.class); + when(mockBucketer.bucket(any(Experiment.class), anyString())).thenReturn(null); + + DecisionService decisionService = new DecisionService( + mockBucketer, + mockErrorHandler, + v4ProjectConfig, + null + ); + + assertNull(decisionService.getVariationForFeatureInRollout( + FEATURE_FLAG_SINGLE_VARIABLE_STRING, + genericUserId, + Collections.singletonMap( + ATTRIBUTE_HOUSE_KEY, AUDIENCE_GRYFFINDOR_VALUE + ) + )); + + // with fall back bucketing, the user has at most 2 chances to get bucketed with traffic allocation + // one chance with the audience rollout rule + // one chance with the everyone else rule + verify(mockBucketer, atMost(2)).bucket(any(Experiment.class), anyString()); + } //========= white list tests ==========/ diff --git a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java index ccbb07196..d92b7da59 100644 --- a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java +++ b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java @@ -171,13 +171,13 @@ public class ValidProjectConfigV4 { ) ) ); - private static final Rollout ROLLOUT_1 = new Rollout( + public static final Rollout ROLLOUT_1 = new Rollout( ROLLOUT_1_ID, Collections.singletonList( ROLLOUT_1_EVERYONE_ELSE_RULE ) ); - private static final FeatureFlag FEATURE_FLAG_SINGLE_VARIABLE_STRING = new FeatureFlag( + public static final FeatureFlag FEATURE_FLAG_SINGLE_VARIABLE_STRING = new FeatureFlag( FEATURE_SINGLE_VARIABLE_STRING_ID, FEATURE_SINGLE_VARIABLE_STRING_KEY, ROLLOUT_1_ID, From 9c970f2c3f9d278e091998ee4c0121fa9b09787a Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Tue, 29 Aug 2017 17:57:41 -0700 Subject: [PATCH 09/20] add test for making sure getVariationForFeatureInRollout returns null when user fails all audiences and traffic in everyone else rule. Add new audience for slytherin. add new rollout 2. Add experiment on double feature to fix test in OptimizelyTest.java --- .../com/optimizely/ab/OptimizelyTest.java | 16 +- .../ab/bucketing/DecisionServiceTest.java | 27 +++ .../ab/config/ValidProjectConfigV4.java | 196 ++++++++++++++++-- .../config/valid-project-config-v4.json | 144 ++++++++++++- 4 files changed, 362 insertions(+), 21 deletions(-) diff --git a/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java b/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java index 4caeef924..9c8659969 100644 --- a/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java +++ b/core-api/src/test/java/com/optimizely/ab/OptimizelyTest.java @@ -80,13 +80,17 @@ import static com.optimizely.ab.config.ValidProjectConfigV4.EXPERIMENT_MULTIVARIATE_EXPERIMENT_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.EXPERIMENT_PAUSED_EXPERIMENT_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE; +import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_SINGLE_VARIABLE_DOUBLE; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_MULTI_VARIATE_FEATURE_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_SINGLE_VARIABLE_BOOLEAN_KEY; +import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_SINGLE_VARIABLE_DOUBLE_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_SINGLE_VARIABLE_STRING_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.MULTIVARIATE_EXPERIMENT_FORCED_VARIATION_USER_ID_GRED; import static com.optimizely.ab.config.ValidProjectConfigV4.PAUSED_EXPERIMENT_FORCED_VARIATION_USER_ID_CONTROL; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_BOOLEAN_VARIABLE_DEFAULT_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_BOOLEAN_VARIABLE_KEY; +import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_DOUBLE_DEFAULT_VALUE; +import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_DOUBLE_VARIABLE_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_FIRST_LETTER_DEFAULT_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_FIRST_LETTER_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.VARIABLE_STRING_VARIABLE_DEFAULT_VALUE; @@ -2557,10 +2561,10 @@ public void getFeatureVariableValueForTypeReturnsDefaultValueWhenFeatureIsNotAtt public void getFeatureVariableValueReturnsDefaultValueWhenFeatureIsAttachedToOneExperimentButFailsTargeting() throws ConfigParseException { assumeTrue(datafileVersion >= Integer.parseInt(ProjectConfig.Version.V4.toString())); - String validFeatureKey = FEATURE_MULTI_VARIATE_FEATURE_KEY; - String validVariableKey = VARIABLE_FIRST_LETTER_KEY; - String expectedValue = VARIABLE_FIRST_LETTER_DEFAULT_VALUE; - FeatureFlag featureFlag = FEATURE_FLAG_MULTI_VARIATE_FEATURE; + String validFeatureKey = FEATURE_SINGLE_VARIABLE_DOUBLE_KEY; + String validVariableKey = VARIABLE_DOUBLE_VARIABLE_KEY; + String expectedValue = VARIABLE_DOUBLE_DEFAULT_VALUE; + FeatureFlag featureFlag = FEATURE_FLAG_SINGLE_VARIABLE_DOUBLE; Experiment experiment = validProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); Optimizely optimizely = Optimizely.builder(validDatafile, mockEventHandler) @@ -2571,8 +2575,8 @@ public void getFeatureVariableValueReturnsDefaultValueWhenFeatureIsAttachedToOne validFeatureKey, validVariableKey, genericUserId, - Collections.singletonMap(ATTRIBUTE_HOUSE_KEY, "Slytherin"), - LiveVariable.VariableType.STRING + Collections.singletonMap(ATTRIBUTE_HOUSE_KEY, "Ravenclaw"), + LiveVariable.VariableType.DOUBLE ); assertEquals(expectedValue, valueWithImproperAttributes); diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 8a4df6dc4..3707e4379 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -465,6 +465,33 @@ public void getVariationForFeatureInRolloutReturnsNullWhenUserIsExcludedFromAllT verify(mockBucketer, atMost(2)).bucket(any(Experiment.class), anyString()); } + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * returns null when a user is excluded from every rule of a rollout due to targeting + * and also fails traffic allocation in the everyone else rollout. + */ + @Test + public void getVariationForFeatureInRolloutReturnsNullWhenUserFailsAllAudiencesAndTraffic() { + Bucketer mockBucketer = mock(Bucketer.class); + when(mockBucketer.bucket(any(Experiment.class), anyString())).thenReturn(null); + + DecisionService decisionService = new DecisionService( + mockBucketer, + mockErrorHandler, + v4ProjectConfig, + null + ); + + assertNull(decisionService.getVariationForFeatureInRollout( + FEATURE_FLAG_SINGLE_VARIABLE_STRING, + genericUserId, + Collections.emptyMap() + )); + + // user is only bucketed once for the everyone else rule + verify(mockBucketer, times(1)).bucket(any(Experiment.class), anyString()); + } + //========= white list tests ==========/ /** diff --git a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java index d92b7da59..aafbb6dfd 100644 --- a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java +++ b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java @@ -54,6 +54,18 @@ public class ValidProjectConfigV4 { CUSTOM_DIMENSION_TYPE, AUDIENCE_GRYFFINDOR_VALUE))))))) ); + private static final String AUDIENCE_SLYTHERIN_ID = "3988293898"; + private static final String AUDIENCE_SLYTHERIN_KEY = "Slytherins"; + public static final String AUDIENCE_SLYTHERIN_VALUE = "Slytherin"; + private static final Audience AUDIENCE_SLYTHERIN = new Audience( + AUDIENCE_SLYTHERIN_ID, + AUDIENCE_SLYTHERIN_KEY, + new AndCondition(Collections.singletonList( + new OrCondition(Collections.singletonList( + new OrCondition(Collections.singletonList((Condition) new UserAttribute(ATTRIBUTE_HOUSE_KEY, + CUSTOM_DIMENSION_TYPE, + AUDIENCE_SLYTHERIN_VALUE))))))) + ); // features private static final String FEATURE_BOOLEAN_FEATURE_ID = "4195505407"; @@ -66,10 +78,10 @@ public class ValidProjectConfigV4 { Collections.emptyList() ); private static final String FEATURE_SINGLE_VARIABLE_DOUBLE_ID = "3926744821"; - private static final String FEATURE_SINGLE_VARIABLE_DOUBLE_KEY = "double_single_variable_feature"; + public static final String FEATURE_SINGLE_VARIABLE_DOUBLE_KEY = "double_single_variable_feature"; private static final String VARIABLE_DOUBLE_VARIABLE_ID = "4111654444"; - private static final String VARIABLE_DOUBLE_VARIABLE_KEY = "double_variable"; - private static final String VARIABLE_DOUBLE_DEFAULT_VALUE = "14.99"; + public static final String VARIABLE_DOUBLE_VARIABLE_KEY = "double_variable"; + public static final String VARIABLE_DOUBLE_DEFAULT_VALUE = "14.99"; private static final LiveVariable VARIABLE_DOUBLE_VARIABLE = new LiveVariable( VARIABLE_DOUBLE_VARIABLE_ID, VARIABLE_DOUBLE_VARIABLE_KEY, @@ -77,15 +89,6 @@ public class ValidProjectConfigV4 { null, LiveVariable.VariableType.DOUBLE ); - private static final FeatureFlag FEATURE_FLAG_SINGLE_VARIABLE_DOUBLE = new FeatureFlag( - FEATURE_SINGLE_VARIABLE_DOUBLE_ID, - FEATURE_SINGLE_VARIABLE_DOUBLE_KEY, - "", - Collections.emptyList(), - Collections.singletonList( - VARIABLE_DOUBLE_VARIABLE - ) - ); private static final String FEATURE_SINGLE_VARIABLE_INTEGER_ID = "3281420120"; private static final String FEATURE_SINGLE_VARIABLE_INTEGER_KEY = "integer_single_variable_feature"; private static final String VARIABLE_INTEGER_VARIABLE_ID = "593964691"; @@ -494,6 +497,57 @@ public class ValidProjectConfigV4 { ) ) ); + + private static final String LAYER_DOUBLE_FEATURE_EXPERIMENT_ID = "1278722008"; + private static final String EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT_ID = "2201520193"; + public static final String EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT_KEY = "double_single_variable_feature_experiment"; + private static final String VARIATION_DOUBLE_FEATURE_PI_VARIATION_ID = "1505457580"; + private static final String VARIATION_DOUBLE_FEATURE_PI_VARIATION_KEY = "pi_variation"; + private static final Variation VARIATION_DOUBLE_FEATURE_PI_VARIATION = new Variation( + VARIATION_DOUBLE_FEATURE_PI_VARIATION_ID, + VARIATION_DOUBLE_FEATURE_PI_VARIATION_KEY, + Collections.singletonList( + new LiveVariableUsageInstance( + VARIABLE_DOUBLE_VARIABLE_ID, + "3.14" + ) + ) + ); + private static final String VARIATION_DOUBLE_FEATURE_EULER_VARIATION_ID = "119616179"; + private static final String VARIATION_DOUBLE_FEATURE_EULER_VARIATION_KEY = "euler_variation"; + private static final Variation VARIATION_DOUBLE_FEATURE_EULER_VARIATION = new Variation( + VARIATION_DOUBLE_FEATURE_EULER_VARIATION_ID, + VARIATION_DOUBLE_FEATURE_EULER_VARIATION_KEY, + Collections.singletonList( + new LiveVariableUsageInstance( + VARIABLE_DOUBLE_VARIABLE_ID, + "2.718" + ) + ) + ); + private static final Experiment EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT = new Experiment( + EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT_ID, + EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT_KEY, + Experiment.ExperimentStatus.RUNNING.toString(), + LAYER_DOUBLE_FEATURE_EXPERIMENT_ID, + Collections.singletonList(AUDIENCE_SLYTHERIN_ID), + ProjectConfigTestUtils.createListOfObjects( + VARIATION_DOUBLE_FEATURE_PI_VARIATION, + VARIATION_DOUBLE_FEATURE_EULER_VARIATION + ), + Collections.emptyMap(), + ProjectConfigTestUtils.createListOfObjects( + new TrafficAllocation( + VARIATION_DOUBLE_FEATURE_PI_VARIATION_ID, + 4000 + ), + new TrafficAllocation( + VARIATION_DOUBLE_FEATURE_EULER_VARIATION_ID, + 8000 + ) + ) + ); + private static final String LAYER_PAUSED_EXPERIMENT_ID = "3949273892"; private static final String EXPERIMENT_PAUSED_EXPERIMENT_ID = "2667098701"; public static final String EXPERIMENT_PAUSED_EXPERIMENT_KEY = "paused_experiment"; @@ -680,11 +734,113 @@ public class ValidProjectConfigV4 { ) ); + // rollouts + private static final String ROLLOUT_2_ID = "813411034"; + private static final Experiment ROLLOUT_2_RULE_1 = new Experiment( + "3421010877", + "3421010877", + Experiment.ExperimentStatus.RUNNING.toString(), + ROLLOUT_2_ID, + Collections.singletonList(AUDIENCE_GRYFFINDOR_ID), + Collections.singletonList( + new Variation( + "521740985", + "521740985", + ProjectConfigTestUtils.createListOfObjects( + new LiveVariableUsageInstance( + "675244127", + "G" + ), + new LiveVariableUsageInstance( + "4052219963", + "odric" + ) + ) + ) + ), + Collections.emptyMap(), + Collections.singletonList( + new TrafficAllocation( + "521740985", + 5000 + ) + ) + ); + private static final Experiment ROLLOUT_2_RULE_2 = new Experiment( + "600050626", + "600050626", + Experiment.ExperimentStatus.RUNNING.toString(), + ROLLOUT_2_ID, + Collections.singletonList(AUDIENCE_SLYTHERIN_ID), + Collections.singletonList( + new Variation( + "180042646", + "180042646", + ProjectConfigTestUtils.createListOfObjects( + new LiveVariableUsageInstance( + "675244127", + "S" + ), + new LiveVariableUsageInstance( + "4052219963", + "alazar" + ) + ) + ) + ), + Collections.emptyMap(), + Collections.singletonList( + new TrafficAllocation( + "180042646", + 5000 + ) + ) + + ); + private static final Experiment ROLLOUT_2_EVERYONE_ELSE_RULE = new Experiment( + "828245624", + "828245624", + Experiment.ExperimentStatus.RUNNING.toString(), + ROLLOUT_2_ID, + Collections.emptyList(), + Collections.singletonList( + new Variation( + "3137445031", + "3137445031", + ProjectConfigTestUtils.createListOfObjects( + new LiveVariableUsageInstance( + "675244127", + "M" + ), + new LiveVariableUsageInstance( + "4052219963", + "uggle" + ) + ) + ) + ), + Collections.emptyMap(), + Collections.singletonList( + new TrafficAllocation( + "3137445031", + 5000 + ) + ) + ); + public static final Rollout ROLLOUT_2 = new Rollout( + ROLLOUT_2_ID, + ProjectConfigTestUtils.createListOfObjects( + ROLLOUT_2_RULE_1, + ROLLOUT_2_RULE_2, + ROLLOUT_2_EVERYONE_ELSE_RULE + ) + ); + // finish features public static final FeatureFlag FEATURE_FLAG_MULTI_VARIATE_FEATURE = new FeatureFlag( FEATURE_MULTI_VARIATE_FEATURE_ID, FEATURE_MULTI_VARIATE_FEATURE_KEY, - "", + ROLLOUT_2_ID, Collections.singletonList(EXPERIMENT_MULTIVARIATE_EXPERIMENT_ID), ProjectConfigTestUtils.createListOfObjects( VARIABLE_FIRST_LETTER_VARIABLE, @@ -703,6 +859,17 @@ public class ValidProjectConfigV4 { VARIABLE_CORRELATING_VARIATION_NAME_VARIABLE ) ); + public static final FeatureFlag FEATURE_FLAG_SINGLE_VARIABLE_DOUBLE = new FeatureFlag( + FEATURE_SINGLE_VARIABLE_DOUBLE_ID, + FEATURE_SINGLE_VARIABLE_DOUBLE_KEY, + "", + Collections.singletonList( + EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT_ID + ), + Collections.singletonList( + VARIABLE_DOUBLE_VARIABLE + ) + ); public static ProjectConfig generateValidProjectConfigV4() { @@ -714,6 +881,7 @@ public static ProjectConfig generateValidProjectConfigV4() { // list audiences List audiences = new ArrayList(); audiences.add(AUDIENCE_GRYFFINDOR); + audiences.add(AUDIENCE_SLYTHERIN); // list events List events = new ArrayList(); @@ -725,6 +893,7 @@ public static ProjectConfig generateValidProjectConfigV4() { List experiments = new ArrayList(); experiments.add(EXPERIMENT_BASIC_EXPERIMENT); experiments.add(EXPERIMENT_MULTIVARIATE_EXPERIMENT); + experiments.add(EXPERIMENT_DOUBLE_FEATURE_EXPERIMENT); experiments.add(EXPERIMENT_PAUSED_EXPERIMENT); experiments.add(EXPERIMENT_LAUNCHED_EXPERIMENT); @@ -745,6 +914,7 @@ public static ProjectConfig generateValidProjectConfigV4() { // list rollouts List rollouts = new ArrayList(); rollouts.add(ROLLOUT_1); + rollouts.add(ROLLOUT_2); return new ProjectConfig( ACCOUNT_ID, diff --git a/core-api/src/test/resources/config/valid-project-config-v4.json b/core-api/src/test/resources/config/valid-project-config-v4.json index e56b804ed..acc41054b 100644 --- a/core-api/src/test/resources/config/valid-project-config-v4.json +++ b/core-api/src/test/resources/config/valid-project-config-v4.json @@ -9,6 +9,11 @@ "id": "3468206642", "name": "Gryffindors", "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_dimension\", \"value\":\"Gryffindor\"}]]]" + }, + { + "id": "3988293898", + "name": "Slytherins", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_dimension\", \"value\":\"Slytherin\"}]]]" } ], "attributes": [ @@ -169,6 +174,46 @@ "George": "George" } }, + { + "id": "2201520193", + "key": "double_single_variable_feature_experiment", + "layerId": "1278722008", + "status": "Running", + "variations": [ + { + "id": "1505457580", + "key": "pi_variation", + "variables": [ + { + "id": "4111654444", + "value": "3.14" + } + ] + }, + { + "id": "119616179", + "key": "euler_variation", + "variables": [ + { + "id": "4111654444", + "value": "2.718" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "1505457580", + "endOfRange": 4000 + }, + { + "entityId": "119616179", + "endOfRange": 8000 + } + ], + "audienceIds": ["3988293898"], + "forcedVariations": {} + }, { "id": "2667098701", "key": "paused_experiment", @@ -382,7 +427,7 @@ "id": "3926744821", "key": "double_single_variable_feature", "rolloutId": "", - "experimentIds": [], + "experimentIds": ["2201520193"], "variables": [ { "id": "4111654444", @@ -437,7 +482,7 @@ { "id": "3263342226", "key": "multi_variate_feature", - "rolloutId": "", + "rolloutId": "813411034", "experimentIds": ["3262035800"], "variables": [ { @@ -500,6 +545,101 @@ ] } ] + }, + { + "id": "813411034", + "experiments": [ + { + "id": "3421010877", + "key": "3421010877", + "status": "Running", + "layerId": "813411034", + "audienceIds": ["3468206642"], + "forcedVariations": {}, + "variations": [ + { + "id": "521740985", + "key": "521740985", + "variables": [ + { + "id": "675244127", + "value": "G" + }, + { + "id": "4052219963", + "value": "odric" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "521740985", + "endOfRange": 5000 + } + ] + }, + { + "id": "600050626", + "key": "600050626", + "status": "Running", + "layerId": "813411034", + "audienceIds": ["3988293898"], + "forcedVariations": {}, + "variations": [ + { + "id": "180042646", + "key": "180042646", + "variables": [ + { + "id": "675244127", + "value": "S" + }, + { + "id": "4052219963", + "value": "alazar" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "180042646", + "endOfRange": 5000 + } + ] + }, + { + "id": "828245624", + "key": "828245624", + "status": "Running", + "layerId": "813411034", + "audienceIds": [], + "forcedVariations": {}, + "variations": [ + { + "id": "3137445031", + "key": "3137445031", + "variables": [ + { + "id": "675244127", + "value": "M" + }, + { + "id": "4052219963", + "value": "uggle" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "3137445031", + "endOfRange": 5000 + } + ] + } + ] } ], "variables": [] From 0efaad7a5fcc3ba6d71b488c5912df99fa9a0fa1 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:08:49 -0700 Subject: [PATCH 10/20] add test to see that user gets bucketed into everyone else rule when failing targeting for all previous rules returns the everyone else variation --- .../ab/bucketing/DecisionServiceTest.java | 40 +++++++++++++++++-- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 3707e4379..110084dcc 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -45,8 +45,10 @@ import static com.optimizely.ab.config.ProjectConfigTestUtils.validProjectConfigV4; import static com.optimizely.ab.config.ValidProjectConfigV4.ATTRIBUTE_HOUSE_KEY; import static com.optimizely.ab.config.ValidProjectConfigV4.AUDIENCE_GRYFFINDOR_VALUE; +import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_SINGLE_VARIABLE_STRING; import static com.optimizely.ab.config.ValidProjectConfigV4.ROLLOUT_1; +import static com.optimizely.ab.config.ValidProjectConfigV4.ROLLOUT_2; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; @@ -330,7 +332,7 @@ public void getVariationForFeatureReturnsNullWhenFeatureFlagExperimentIdsIsEmpty @Test @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") public void getVariationForFeatureReturnsNullWhenItGetsNoVariationsForExperimentsAndRollouts() { - FeatureFlag spyFeatureFlag = spy(ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE); + FeatureFlag spyFeatureFlag = spy(FEATURE_FLAG_MULTI_VARIATE_FEATURE); DecisionService spyDecisionService = spy(new DecisionService( mock(Bucketer.class), @@ -452,7 +454,7 @@ public void getVariationForFeatureInRolloutReturnsNullWhenUserIsExcludedFromAllT ); assertNull(decisionService.getVariationForFeatureInRollout( - FEATURE_FLAG_SINGLE_VARIABLE_STRING, + FEATURE_FLAG_MULTI_VARIATE_FEATURE, genericUserId, Collections.singletonMap( ATTRIBUTE_HOUSE_KEY, AUDIENCE_GRYFFINDOR_VALUE @@ -483,7 +485,7 @@ public void getVariationForFeatureInRolloutReturnsNullWhenUserFailsAllAudiencesA ); assertNull(decisionService.getVariationForFeatureInRollout( - FEATURE_FLAG_SINGLE_VARIABLE_STRING, + FEATURE_FLAG_MULTI_VARIATE_FEATURE, genericUserId, Collections.emptyMap() )); @@ -492,6 +494,38 @@ public void getVariationForFeatureInRolloutReturnsNullWhenUserFailsAllAudiencesA verify(mockBucketer, times(1)).bucket(any(Experiment.class), anyString()); } + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * returns the variation of "Everyone Else" rule + * when the user fails targeting for all rules, but is bucketed into the "Everyone Else" rule. + */ + @Test + public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsAllAudienceButSatisfiesTraffic() { + Bucketer mockBucketer = mock(Bucketer.class); + Rollout rollout = ROLLOUT_2; + Experiment everyoneElseRule = rollout.getExperiments().get(rollout.getExperiments().size() - 1); + Variation expectedVariation = everyoneElseRule.getVariations().get(0); + when(mockBucketer.bucket(eq(everyoneElseRule), anyString())).thenReturn(expectedVariation); + + DecisionService decisionService = new DecisionService( + mockBucketer, + mockErrorHandler, + v4ProjectConfig, + null + ); + + assertEquals(expectedVariation, + decisionService.getVariationForFeatureInRollout( + FEATURE_FLAG_MULTI_VARIATE_FEATURE, + genericUserId, + Collections.emptyMap() + ) + ); + + // verify user is only bucketed once for everyone else rule + verify(mockBucketer, times(1)).bucket(any(Experiment.class), anyString()); + } + //========= white list tests ==========/ /** From e250de27d1049acc293c7b1ca696721c48428e22 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:15:06 -0700 Subject: [PATCH 11/20] add test to make sure user gets bucketed into everyone else rule when they pass audience targeting for another rule, but fail traffic allocation for that rule --- .../ab/bucketing/DecisionServiceTest.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 110084dcc..009c93e0a 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -526,6 +526,42 @@ public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsAllAudie verify(mockBucketer, times(1)).bucket(any(Experiment.class), anyString()); } + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * returns the variation of "Everyone Else" rule + * when the user passes targeting for a rule, but was failed the traffic allocation for that rule, + * and is bucketed successfully into the "Everyone Else" rule. + */ + @Test + public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsTrafficInRuleAndPassesInEveryoneElse() { + Bucketer mockBucketer = mock(Bucketer.class); + Rollout rollout = ROLLOUT_2; + Experiment everyoneElseRule = rollout.getExperiments().get(rollout.getExperiments().size() - 1); + Variation expectedVariation = everyoneElseRule.getVariations().get(0); + when(mockBucketer.bucket(any(Experiment.class), anyString())).thenReturn(null); + when(mockBucketer.bucket(eq(everyoneElseRule), anyString())).thenReturn(expectedVariation); + + DecisionService decisionService = new DecisionService( + mockBucketer, + mockErrorHandler, + v4ProjectConfig, + null + ); + + assertEquals(expectedVariation, + decisionService.getVariationForFeatureInRollout( + FEATURE_FLAG_MULTI_VARIATE_FEATURE, + genericUserId, + Collections.singletonMap( + ATTRIBUTE_HOUSE_KEY, AUDIENCE_GRYFFINDOR_VALUE + ) + ) + ); + + // verify user is only bucketed once for everyone else rule + verify(mockBucketer, times(2)).bucket(any(Experiment.class), anyString()); + } + //========= white list tests ==========/ /** From f6d3e4a5f1f62177329eead4cd160127a413ac10 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:21:31 -0700 Subject: [PATCH 12/20] add new attribute for nationality --- .../java/com/optimizely/ab/config/ValidProjectConfigV4.java | 5 +++++ .../src/test/resources/config/valid-project-config-v4.json | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java index aafbb6dfd..ecf6413a5 100644 --- a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java +++ b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java @@ -40,6 +40,10 @@ public class ValidProjectConfigV4 { public static final String ATTRIBUTE_HOUSE_KEY = "house"; private static final Attribute ATTRIBUTE_HOUSE = new Attribute(ATTRIBUTE_HOUSE_ID, ATTRIBUTE_HOUSE_KEY); + private static final String ATTRIBUTE_NATIONALITY_ID = "58339410"; + public static final String ATTRIBUTE_NATIONALITY_KEY = "nationality"; + private static final Attribute ATTRIBUTE_NATIONALITY = new Attribute(ATTRIBUTE_NATIONALITY_ID, ATTRIBUTE_NATIONALITY_KEY); + // audiences private static final String CUSTOM_DIMENSION_TYPE = "custom_dimension"; private static final String AUDIENCE_GRYFFINDOR_ID = "3468206642"; @@ -877,6 +881,7 @@ public static ProjectConfig generateValidProjectConfigV4() { // list attributes List attributes = new ArrayList(); attributes.add(ATTRIBUTE_HOUSE); + attributes.add(ATTRIBUTE_NATIONALITY); // list audiences List audiences = new ArrayList(); diff --git a/core-api/src/test/resources/config/valid-project-config-v4.json b/core-api/src/test/resources/config/valid-project-config-v4.json index acc41054b..1532a1c92 100644 --- a/core-api/src/test/resources/config/valid-project-config-v4.json +++ b/core-api/src/test/resources/config/valid-project-config-v4.json @@ -20,6 +20,10 @@ { "id": "553339214", "key": "house" + }, + { + "id": "58339410", + "key": "nationality" } ], "events": [ From 2e1166a08900bf1208fc651ec78f249b57ef27a0 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:26:43 -0700 Subject: [PATCH 13/20] add english citizens audience --- .../optimizely/ab/config/ValidProjectConfigV4.java | 14 ++++++++++++++ .../resources/config/valid-project-config-v4.json | 5 +++++ 2 files changed, 19 insertions(+) diff --git a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java index ecf6413a5..163dd28fb 100644 --- a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java +++ b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java @@ -71,6 +71,19 @@ public class ValidProjectConfigV4 { AUDIENCE_SLYTHERIN_VALUE))))))) ); + private static final String AUDIENCE_ENGLISH_CITIZENS_ID = "4194404272"; + private static final String AUDIENCE_ENGLISH_CITIZENS_KEY = "english_citizens"; + public static final String AUDIENCE_ENGLISH_CITIZENS_VALUE = "English"; + private static final Audience AUDIENCE_ENGLISH_CITIZENS = new Audience( + AUDIENCE_ENGLISH_CITIZENS_ID, + AUDIENCE_ENGLISH_CITIZENS_KEY, + new AndCondition(Collections.singletonList( + new OrCondition(Collections.singletonList( + new OrCondition(Collections.singletonList((Condition) new UserAttribute(ATTRIBUTE_NATIONALITY_KEY, + CUSTOM_DIMENSION_TYPE, + AUDIENCE_ENGLISH_CITIZENS_VALUE))))))) + ); + // features private static final String FEATURE_BOOLEAN_FEATURE_ID = "4195505407"; private static final String FEATURE_BOOLEAN_FEATURE_KEY = "boolean_feature"; @@ -887,6 +900,7 @@ public static ProjectConfig generateValidProjectConfigV4() { List audiences = new ArrayList(); audiences.add(AUDIENCE_GRYFFINDOR); audiences.add(AUDIENCE_SLYTHERIN); + audiences.add(AUDIENCE_ENGLISH_CITIZENS); // list events List events = new ArrayList(); diff --git a/core-api/src/test/resources/config/valid-project-config-v4.json b/core-api/src/test/resources/config/valid-project-config-v4.json index 1532a1c92..905f8b000 100644 --- a/core-api/src/test/resources/config/valid-project-config-v4.json +++ b/core-api/src/test/resources/config/valid-project-config-v4.json @@ -14,6 +14,11 @@ "id": "3988293898", "name": "Slytherins", "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"house\", \"type\": \"custom_dimension\", \"value\":\"Slytherin\"}]]]" + }, + { + "id": "4194404272", + "name": "english_citizens", + "conditions": "[\"and\", [\"or\", [\"or\", {\"name\": \"nationality\", \"type\": \"custom_dimension\", \"value\":\"English\"}]]]" } ], "attributes": [ From 0b10a334ed5bd89f3627d615e9c0f6d1fc67be37 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:34:05 -0700 Subject: [PATCH 14/20] add third rollout rule for english citizen audience that is not disjoint from previous rollout rules --- .../ab/config/ValidProjectConfigV4.java | 32 ++++++++++++++++++- .../config/valid-project-config-v4.json | 30 +++++++++++++++++ 2 files changed, 61 insertions(+), 1 deletion(-) diff --git a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java index 163dd28fb..bc0dab271 100644 --- a/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java +++ b/core-api/src/test/java/com/optimizely/ab/config/ValidProjectConfigV4.java @@ -812,7 +812,36 @@ public class ValidProjectConfigV4 { 5000 ) ) - + ); + private static final Experiment ROLLOUT_2_RULE_3 = new Experiment( + "2637642575", + "2637642575", + Experiment.ExperimentStatus.RUNNING.toString(), + ROLLOUT_2_ID, + Collections.singletonList(AUDIENCE_ENGLISH_CITIZENS_ID), + Collections.singletonList( + new Variation( + "2346257680", + "2346257680", + ProjectConfigTestUtils.createListOfObjects( + new LiveVariableUsageInstance( + "675244127", + "D" + ), + new LiveVariableUsageInstance( + "4052219963", + "udley" + ) + ) + ) + ), + Collections.emptyMap(), + Collections.singletonList( + new TrafficAllocation( + "2346257680", + 5000 + ) + ) ); private static final Experiment ROLLOUT_2_EVERYONE_ELSE_RULE = new Experiment( "828245624", @@ -849,6 +878,7 @@ public class ValidProjectConfigV4 { ProjectConfigTestUtils.createListOfObjects( ROLLOUT_2_RULE_1, ROLLOUT_2_RULE_2, + ROLLOUT_2_RULE_3, ROLLOUT_2_EVERYONE_ELSE_RULE ) ); diff --git a/core-api/src/test/resources/config/valid-project-config-v4.json b/core-api/src/test/resources/config/valid-project-config-v4.json index 905f8b000..9ef2e682f 100644 --- a/core-api/src/test/resources/config/valid-project-config-v4.json +++ b/core-api/src/test/resources/config/valid-project-config-v4.json @@ -618,6 +618,36 @@ } ] }, + { + "id": "2637642575", + "key": "2637642575", + "status": "Running", + "layerId": "813411034", + "audienceIds": ["4194404272"], + "forcedVariations": {}, + "variations": [ + { + "id": "2346257680", + "key": "2346257680", + "variables": [ + { + "id": "675244127", + "value": "D" + }, + { + "id": "4052219963", + "value": "udley" + } + ] + } + ], + "trafficAllocation": [ + { + "entityId": "2346257680", + "endOfRange": 5000 + } + ] + }, { "id": "828245624", "key": "828245624", From 8c5e506ccc2c7276e384b6d567ef1ade4db65af9 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:38:33 -0700 Subject: [PATCH 15/20] add test to ensure no other rules are bucketed when user fails first traffic allocation rule in fallback bucketing strategy --- .../ab/bucketing/DecisionServiceTest.java | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 009c93e0a..3521d423c 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -20,6 +20,7 @@ import com.optimizely.ab.config.Experiment; import com.optimizely.ab.config.FeatureFlag; import com.optimizely.ab.config.ProjectConfig; +import com.optimizely.ab.config.ProjectConfigTestUtils; import com.optimizely.ab.config.Rollout; import com.optimizely.ab.config.TrafficAllocation; import com.optimizely.ab.config.ValidProjectConfigV4; @@ -44,6 +45,8 @@ import static com.optimizely.ab.config.ProjectConfigTestUtils.validProjectConfigV3; import static com.optimizely.ab.config.ProjectConfigTestUtils.validProjectConfigV4; import static com.optimizely.ab.config.ValidProjectConfigV4.ATTRIBUTE_HOUSE_KEY; +import static com.optimizely.ab.config.ValidProjectConfigV4.ATTRIBUTE_NATIONALITY_KEY; +import static com.optimizely.ab.config.ValidProjectConfigV4.AUDIENCE_ENGLISH_CITIZENS_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.AUDIENCE_GRYFFINDOR_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_SINGLE_VARIABLE_STRING; @@ -562,7 +565,53 @@ public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsTrafficI verify(mockBucketer, times(2)).bucket(any(Experiment.class), anyString()); } - //========= white list tests ==========/ + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * returns the variation of "Everyone Else" rule + * when the user passes targeting for a rule, but was failed the traffic allocation for that rule, + * and is bucketed successfully into the "Everyone Else" rule. + * Fallback bucketing should not evaluate any other audiences. + * Even though the user would satisfy a later rollout rule, they are never evaluated for it or bucketed into it. + */ + @Test + public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsTrafficInRuleButWouldPassForAnotherRuleAndPassesInEveryoneElse() { + Bucketer mockBucketer = mock(Bucketer.class); + Rollout rollout = ROLLOUT_2; + Experiment englishCitizensRule = rollout.getExperiments().get(2); + Variation englishCitizenVariation = englishCitizensRule.getVariations().get(0); + Experiment everyoneElseRule = rollout.getExperiments().get(rollout.getExperiments().size() - 1); + Variation expectedVariation = everyoneElseRule.getVariations().get(0); + when(mockBucketer.bucket(any(Experiment.class), anyString())).thenReturn(null); + when(mockBucketer.bucket(eq(everyoneElseRule), anyString())).thenReturn(expectedVariation); + when(mockBucketer.bucket(eq(englishCitizensRule), anyString())).thenReturn(englishCitizenVariation); + + DecisionService decisionService = new DecisionService( + mockBucketer, + mockErrorHandler, + v4ProjectConfig, + null + ); + + assertEquals(expectedVariation, + decisionService.getVariationForFeatureInRollout( + FEATURE_FLAG_MULTI_VARIATE_FEATURE, + genericUserId, + ProjectConfigTestUtils.createMapOfObjects( + ProjectConfigTestUtils.createListOfObjects( + ATTRIBUTE_HOUSE_KEY, ATTRIBUTE_NATIONALITY_KEY + ), + ProjectConfigTestUtils.createListOfObjects( + AUDIENCE_GRYFFINDOR_VALUE, AUDIENCE_ENGLISH_CITIZENS_VALUE + ) + ) + ) + ); + + // verify user is only bucketed once for everyone else rule + verify(mockBucketer, times(2)).bucket(any(Experiment.class), anyString()); + } + + //========= white list tests ==========/ /** * Test {@link DecisionService#getWhitelistedVariation(Experiment, String)} correctly returns a whitelisted variation. From 4f60179cbd195daf9a34b0f49b97ba5d725ce650 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Wed, 30 Aug 2017 15:42:49 -0700 Subject: [PATCH 16/20] add test to make sure we continue evaluating more rules when we fail audience targeting in fallback strategy --- .../ab/bucketing/DecisionServiceTest.java | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 3521d423c..550e1dc7c 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -611,6 +611,44 @@ public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsTrafficI verify(mockBucketer, times(2)).bucket(any(Experiment.class), anyString()); } + /** + * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} + * returns the variation of "English Citizens" rule + * when the user fails targeting for previous rules, but passes targeting and traffic for Rule 3. + */ + @Test + public void getVariationForFeatureInRolloutReturnsVariationWhenUserFailsTargetingInPreviousRulesButPassesRule3() { + Bucketer mockBucketer = mock(Bucketer.class); + Rollout rollout = ROLLOUT_2; + Experiment englishCitizensRule = rollout.getExperiments().get(2); + Variation englishCitizenVariation = englishCitizensRule.getVariations().get(0); + Experiment everyoneElseRule = rollout.getExperiments().get(rollout.getExperiments().size() - 1); + Variation everyoneElseVariation = everyoneElseRule.getVariations().get(0); + when(mockBucketer.bucket(any(Experiment.class), anyString())).thenReturn(null); + when(mockBucketer.bucket(eq(everyoneElseRule), anyString())).thenReturn(everyoneElseVariation); + when(mockBucketer.bucket(eq(englishCitizensRule), anyString())).thenReturn(englishCitizenVariation); + + DecisionService decisionService = new DecisionService( + mockBucketer, + mockErrorHandler, + v4ProjectConfig, + null + ); + + assertEquals(englishCitizenVariation, + decisionService.getVariationForFeatureInRollout( + FEATURE_FLAG_MULTI_VARIATE_FEATURE, + genericUserId, + Collections.singletonMap( + ATTRIBUTE_NATIONALITY_KEY, AUDIENCE_ENGLISH_CITIZENS_VALUE + ) + ) + ); + + // verify user is only bucketed once for everyone else rule + verify(mockBucketer, times(1)).bucket(any(Experiment.class), anyString()); + } + //========= white list tests ==========/ /** From bc2c95eafcbe97db32b9094f6c9b52cf62992e89 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Thu, 31 Aug 2017 16:53:44 -0700 Subject: [PATCH 17/20] add test to make sure feature variable bucketing checks experiment first --- .../ab/bucketing/DecisionServiceTest.java | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 550e1dc7c..c712e0d7d 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -49,12 +49,11 @@ import static com.optimizely.ab.config.ValidProjectConfigV4.AUDIENCE_ENGLISH_CITIZENS_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.AUDIENCE_GRYFFINDOR_VALUE; import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_MULTI_VARIATE_FEATURE; -import static com.optimizely.ab.config.ValidProjectConfigV4.FEATURE_FLAG_SINGLE_VARIABLE_STRING; -import static com.optimizely.ab.config.ValidProjectConfigV4.ROLLOUT_1; import static com.optimizely.ab.config.ValidProjectConfigV4.ROLLOUT_2; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertFalse; @@ -374,7 +373,7 @@ public void getVariationForFeatureReturnsNullWhenItGetsNoVariationsForExperiment */ @Test @SuppressFBWarnings("RV_RETURN_VALUE_IGNORED_NO_SIDE_EFFECT") - public void getVariationForFeatureReturnsVariationReturnedFromGetVarition() { + public void getVariationForFeatureReturnsVariationReturnedFromGetVariation() { FeatureFlag spyFeatureFlag = spy(ValidProjectConfigV4.FEATURE_FLAG_MUTEX_GROUP_FEATURE); DecisionService spyDecisionService = spy(new DecisionService( @@ -407,6 +406,69 @@ public void getVariationForFeatureReturnsVariationReturnedFromGetVarition() { verify(spyFeatureFlag, never()).getKey(); } + /** + * Verify that when getting a {@link Variation} for a {@link FeatureFlag} in + * {@link DecisionService#getVariationForFeature(FeatureFlag, String, Map)}, + * check first if the user is bucketed to an {@link Experiment} + * then check if the user is not bucketed to an experiment, + * check for a {@link Rollout}. + */ + @Test + public void getVariationForFeatureReturnsVariationFromExperimentBeforeRollout() { + FeatureFlag featureFlag = FEATURE_FLAG_MULTI_VARIATE_FEATURE; + Experiment featureExperiment = v4ProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); + assertNotNull(featureExperiment); + Rollout featureRollout = v4ProjectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); + Variation experimentVariation = featureExperiment.getVariations().get(0); + Variation rolloutVariation = featureRollout.getExperiments().get(0).getVariations().get(0); + + DecisionService decisionService = spy(new DecisionService( + mock(Bucketer.class), + mockErrorHandler, + v4ProjectConfig, + null + ) + ); + + // return variation for experiment + doReturn(experimentVariation) + .when(decisionService).getVariation( + eq(featureExperiment), + anyString(), + anyMapOf(String.class, String.class) + ); + + // return variation for rollout + doReturn(rolloutVariation) + .when(decisionService).getVariationForFeatureInRollout( + eq(featureFlag), + anyString(), + anyMapOf(String.class, String.class) + ); + + // make sure we get the right variation back + assertEquals(experimentVariation, + decisionService.getVariationForFeature(featureFlag, + genericUserId, + Collections.emptyMap() + ) + ); + + // make sure we do not even check for rollout bucketing + verify(decisionService, never()).getVariationForFeatureInRollout( + any(FeatureFlag.class), + anyString(), + anyMapOf(String.class, String.class) + ); + + // make sure we ask for experiment bucketing once + verify(decisionService, times(1)).getVariation( + any(Experiment.class), + anyString(), + anyMapOf(String.class, String.class) + ); + } + //========== getVariationForFeatureInRollout tests ==========// /** From 0fe34d71a7aad15d443417b41a506f92d4162c26 Mon Sep 17 00:00:00 2001 From: wangjoshuah Date: Thu, 31 Aug 2017 17:01:09 -0700 Subject: [PATCH 18/20] add test to make sure we return rollout variation when we do not get a varaition for the experiment --- .../ab/bucketing/DecisionServiceTest.java | 64 ++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index c712e0d7d..83c5421e0 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -469,7 +469,69 @@ public void getVariationForFeatureReturnsVariationFromExperimentBeforeRollout() ); } - //========== getVariationForFeatureInRollout tests ==========// + /** + * Verify that when getting a {@link Variation} for a {@link FeatureFlag} in + * {@link DecisionService#getVariationForFeature(FeatureFlag, String, Map)}, + * check first if the user is bucketed to an {@link Rollout} + * if the user is not bucketed to an experiment. + */ + @Test + public void getVariationForFeatureReturnsVariationFromRolloutWhenExperimentFails() { + FeatureFlag featureFlag = FEATURE_FLAG_MULTI_VARIATE_FEATURE; + Experiment featureExperiment = v4ProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); + assertNotNull(featureExperiment); + Rollout featureRollout = v4ProjectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); + Variation experimentVariation = featureExperiment.getVariations().get(0); + Variation rolloutVariation = featureRollout.getExperiments().get(0).getVariations().get(0); + + DecisionService decisionService = spy(new DecisionService( + mock(Bucketer.class), + mockErrorHandler, + v4ProjectConfig, + null + ) + ); + + // return variation for experiment + doReturn(null) + .when(decisionService).getVariation( + eq(featureExperiment), + anyString(), + anyMapOf(String.class, String.class) + ); + + // return variation for rollout + doReturn(rolloutVariation) + .when(decisionService).getVariationForFeatureInRollout( + eq(featureFlag), + anyString(), + anyMapOf(String.class, String.class) + ); + + // make sure we get the right variation back + assertEquals(rolloutVariation, + decisionService.getVariationForFeature(featureFlag, + genericUserId, + Collections.emptyMap() + ) + ); + + // make sure we do not even check for rollout bucketing + verify(decisionService,times(1)).getVariationForFeatureInRollout( + any(FeatureFlag.class), + anyString(), + anyMapOf(String.class, String.class) + ); + + // make sure we ask for experiment bucketing once + verify(decisionService, times(1)).getVariation( + any(Experiment.class), + anyString(), + anyMapOf(String.class, String.class) + ); + } + + //========== getVariationForFeatureInRollout tests ==========// /** * Verify that {@link DecisionService#getVariationForFeatureInRollout(FeatureFlag, String, Map)} From a8f748ea46c0d0ef97cec0bbfd68bc26a7e8d76d Mon Sep 17 00:00:00 2001 From: "Joshua H. Wang" Date: Thu, 31 Aug 2017 21:00:59 -0700 Subject: [PATCH 19/20] fix dead store jmh bug --- .../java/com/optimizely/ab/bucketing/DecisionServiceTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index 83c5421e0..d69224118 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -419,7 +419,6 @@ public void getVariationForFeatureReturnsVariationFromExperimentBeforeRollout() Experiment featureExperiment = v4ProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); assertNotNull(featureExperiment); Rollout featureRollout = v4ProjectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); - Variation experimentVariation = featureExperiment.getVariations().get(0); Variation rolloutVariation = featureRollout.getExperiments().get(0).getVariations().get(0); DecisionService decisionService = spy(new DecisionService( From 2e2aa5b2a251548c8411fc1c093816566597a811 Mon Sep 17 00:00:00 2001 From: "Joshua H. Wang" Date: Thu, 31 Aug 2017 21:43:18 -0700 Subject: [PATCH 20/20] fix rebase issue --- .../java/com/optimizely/ab/bucketing/DecisionServiceTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java index d69224118..190a64cfd 100644 --- a/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java +++ b/core-api/src/test/java/com/optimizely/ab/bucketing/DecisionServiceTest.java @@ -419,6 +419,7 @@ public void getVariationForFeatureReturnsVariationFromExperimentBeforeRollout() Experiment featureExperiment = v4ProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); assertNotNull(featureExperiment); Rollout featureRollout = v4ProjectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); + Variation experimentVariation = featureExperiment.getVariations().get(0); Variation rolloutVariation = featureRollout.getExperiments().get(0).getVariations().get(0); DecisionService decisionService = spy(new DecisionService( @@ -480,7 +481,6 @@ public void getVariationForFeatureReturnsVariationFromRolloutWhenExperimentFails Experiment featureExperiment = v4ProjectConfig.getExperimentIdMapping().get(featureFlag.getExperimentIds().get(0)); assertNotNull(featureExperiment); Rollout featureRollout = v4ProjectConfig.getRolloutIdMapping().get(featureFlag.getRolloutId()); - Variation experimentVariation = featureExperiment.getVariations().get(0); Variation rolloutVariation = featureRollout.getExperiments().get(0).getVariations().get(0); DecisionService decisionService = spy(new DecisionService(