[AI-FSSDK] [FSSDK-12958] Continue rule evaluation after local holdout hit (#640)

Author: alexjoeyyong

core-api/src/main/java/com/optimizely/ab/Optimizely.java

@@ -1,5 +1,5 @@
 /****************************************************************************
- * Copyright 2016-2024, Optimizely, Inc. and contributors                   *
+ * Copyright 2016-2024, 2026, Optimizely, Inc. and contributors                *
  *                                                                          *
  * Licensed under the Apache License, Version 2.0 (the "License");          *
  * you may not use this file except in compliance with the License.         *
@@ -1356,17 +1356,19 @@ private OptimizelyDecision createOptimizelyDecision(
                 cmabUuid);
         }
 
-        if (flagDecision.holdoutDecision != null && !allOptions.contains(OptimizelyDecideOption.DISABLE_DECISION_EVENT)) {
-            decisionEventDispatched = sendImpression(
-                projectConfig,
-                flagDecision.holdoutDecision.experiment,
-                userId,
-                copiedAttributes,
-                flagDecision.holdoutDecision.variation,
-                flagKey,
-                flagDecision.holdoutDecision.decisionSource != null ? flagDecision.holdoutDecision.decisionSource.toString() : FeatureDecision.DecisionSource.HOLDOUT.toString(),
-                flagDecision.holdoutDecision.variation != null && flagDecision.holdoutDecision.variation.getFeatureEnabled(),
-                null) || decisionEventDispatched;
+        if (!allOptions.contains(OptimizelyDecideOption.DISABLE_DECISION_EVENT)) {
+            for (FeatureDecision holdoutDecision : flagDecision.holdoutDecisions) {
+                decisionEventDispatched = sendImpression(
+                    projectConfig,
+                    holdoutDecision.experiment,
+                    userId,
+                    copiedAttributes,
+                    holdoutDecision.variation,
+                    flagKey,
+                    holdoutDecision.decisionSource != null ? holdoutDecision.decisionSource.toString() : FeatureDecision.DecisionSource.HOLDOUT.toString(),
+                    holdoutDecision.variation != null && holdoutDecision.variation.getFeatureEnabled(),
+                    null) || decisionEventDispatched;
+            }
         }
 
         DecisionNotification decisionNotification = DecisionNotification.newFlagDecisionNotificationBuilder()

core-api/src/main/java/com/optimizely/ab/bucketing/DecisionService.java

@@ -351,38 +351,44 @@ public  List<DecisionResponse<FeatureDecision>> getVariationsForFeatureList(@Non
                 reasons.addInfo("Holdout \"%s\" has excludeTargetedDeliveries enabled, continuing to rollout evaluation.", globalHoldoutDecision.experiment.getKey());
             }
 
-            if (globalHoldoutDecision == null) {
+            List<FeatureDecision> holdoutDecisions = new ArrayList<>();
+            if (globalHoldoutDecision != null) {
+                holdoutDecisions.add(globalHoldoutDecision);
+            } else {
                 DecisionResponse<FeatureDecision> decisionVariationResponse = getVariationFromExperiment(projectConfig, featureFlag, user, options, userProfileTracker, decisionPath);
                 reasons.merge(decisionVariationResponse.getReasons());
 
                 FeatureDecision decision = decisionVariationResponse.getResult();
                 boolean error = decisionVariationResponse.isError();
 
-                if (decision != null) {
+                if (decision != null && (decision.variation != null || error)) {
+                    syncDeprecatedHoldoutDecision(decision);
                     decisions.add(new DecisionResponse(decision, reasons, error, decision.cmabUuid));
                     continue flagLoop;
                 }
+                if (decision != null) {
+                    holdoutDecisions.addAll(decision.holdoutDecisions);
+                }
             }
 
             DecisionResponse<FeatureDecision> decisionFeatureResponse = getVariationForFeatureInRollout(featureFlag, user, projectConfig);
             reasons.merge(decisionFeatureResponse.getReasons());
             FeatureDecision decision = decisionFeatureResponse.getResult();
 
-            if (decision != null && decision.variation != null) {
-                if (globalHoldoutDecision != null) {
-                    decision.setHoldoutDecision(globalHoldoutDecision);
+            if (!holdoutDecisions.isEmpty()) {
+                if (decision == null) {
+                    decision = new FeatureDecision(null, null, null);
                 }
+                decision.holdoutDecisions.addAll(0, holdoutDecisions);
+            }
+            syncDeprecatedHoldoutDecision(decision);
+
+            if (decision != null && decision.variation != null) {
                 String message = reasons.addInfo("The user \"%s\" was bucketed into a rollout for feature flag \"%s\".",
                     user.getUserId(), featureFlag.getKey());
                 logger.info(message);
                 decisions.add(new DecisionResponse(decision, reasons));
             } else {
-                if (globalHoldoutDecision != null) {
-                    if (decision == null) {
-                        decision = new FeatureDecision(null, null, null);
-                    }
-                    decision.setHoldoutDecision(globalHoldoutDecision);
-                }
                 String message = reasons.addInfo("The user \"%s\" was not bucketed into a rollout for feature flag \"%s\".",
                     user.getUserId(), featureFlag.getKey());
                 logger.info(message);
@@ -397,6 +403,13 @@ public  List<DecisionResponse<FeatureDecision>> getVariationsForFeatureList(@Non
         return decisions;
     }
 
+    @SuppressWarnings("deprecation")
+    private static void syncDeprecatedHoldoutDecision(@Nullable FeatureDecision decision) {
+        if (decision != null && !decision.holdoutDecisions.isEmpty()) {
+            decision.setHoldoutDecision(decision.holdoutDecisions.get(0));
+        }
+    }
+
     @Nonnull
     public DecisionResponse<FeatureDecision> getVariationForFeature(@Nonnull FeatureFlag featureFlag,
                                                                     @Nonnull OptimizelyUserContext user,
@@ -422,6 +435,7 @@ DecisionResponse<FeatureDecision> getVariationFromExperiment(@Nonnull ProjectCon
         DecisionReasons reasons = DefaultDecisionReasons.newInstance();
         // Cache flagKey once to avoid multiple getKey() calls (important for mock-based tests)
         String flagKey = featureFlag.getKey();
+        List<FeatureDecision> holdoutDecisions = new ArrayList<>();
         if (!featureFlag.getExperimentIds().isEmpty()) {
             for (String experimentId : featureFlag.getExperimentIds()) {
                 Experiment experiment = projectConfig.getExperimentIdMapping().get(experimentId);
@@ -430,7 +444,12 @@ DecisionResponse<FeatureDecision> getVariationFromExperiment(@Nonnull ProjectCon
                     getVariationFromExperimentRule(projectConfig, flagKey, experiment, user, options, userProfileTracker, decisionPath);
                 reasons.merge(decisionVariation.getReasons());
                 FeatureDecision featureDecision = decisionVariation.getResult();
+                if (featureDecision != null && featureDecision.decisionSource == FeatureDecision.DecisionSource.HOLDOUT) {
+                    holdoutDecisions.add(featureDecision);
+                    continue;
+                }
                 if (decisionVariation.isError() || (featureDecision != null && featureDecision.variation != null)) {
+                    featureDecision.holdoutDecisions.addAll(holdoutDecisions);
                     return new DecisionResponse(featureDecision, reasons, decisionVariation.isError(), decisionVariation.getCmabUuid());
                 }
             }
@@ -439,6 +458,11 @@ DecisionResponse<FeatureDecision> getVariationFromExperiment(@Nonnull ProjectCon
             logger.info(message);
         }
 
+        if (!holdoutDecisions.isEmpty()) {
+            FeatureDecision emptyDecision = new FeatureDecision(null, null, null);
+            emptyDecision.holdoutDecisions.addAll(holdoutDecisions);
+            return new DecisionResponse(emptyDecision, reasons);
+        }
         return new DecisionResponse(null, reasons);
 
     }
@@ -480,6 +504,7 @@ DecisionResponse<FeatureDecision> getVariationForFeatureInRollout(@Nonnull Featu
         }
 
 
+        List<FeatureDecision> holdoutDecisions = new ArrayList<>();
         int index = 0;
         while (index < rolloutRulesLength) {
             Experiment rolloutRule = rollout.getExperiments().get(index);
@@ -496,15 +521,23 @@ DecisionResponse<FeatureDecision> getVariationForFeatureInRollout(@Nonnull Featu
             AbstractMap.SimpleEntry<FeatureDecision, Boolean> response = decisionVariationResponse.getResult();
             FeatureDecision featureDecision = response.getKey();
             Boolean skipToEveryoneElse = response.getValue();
+            if (featureDecision != null && featureDecision.decisionSource == FeatureDecision.DecisionSource.HOLDOUT) {
+                holdoutDecisions.add(featureDecision);
+                index++;
+                continue;
+            }
             if (featureDecision != null) {
+                featureDecision.holdoutDecisions.addAll(holdoutDecisions);
                 return new DecisionResponse(featureDecision, reasons);
             }
 
             // The last rule is special for "Everyone Else"
             index = skipToEveryoneElse ? (rolloutRulesLength - 1) : (index + 1);
         }
 
-        return new DecisionResponse(new FeatureDecision(null, null, null), reasons);
+        FeatureDecision emptyDecision = new FeatureDecision(null, null, null);
+        emptyDecision.holdoutDecisions.addAll(holdoutDecisions);
+        return new DecisionResponse(emptyDecision, reasons);
     }
 
     /**

core-api/src/main/java/com/optimizely/ab/bucketing/FeatureDecision.java

@@ -1,5 +1,5 @@
 /****************************************************************************
- * Copyright 2017, 2019, Optimizely, Inc. and contributors                        *
+ * Copyright 2017, 2019, 2026, Optimizely, Inc. and contributors                  *
  *                                                                          *
  * Licensed under the Apache License, Version 2.0 (the "License");          *
  * you may not use this file except in compliance with the License.         *
@@ -15,8 +15,12 @@
  ***************************************************************************/
 package com.optimizely.ab.bucketing;
 
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.optimizely.ab.config.ExperimentCore;
 import com.optimizely.ab.config.Variation;
 
@@ -45,9 +49,20 @@ public class FeatureDecision {
     @Nullable
     public String cmabUuid;
 
+    /**
+     * @deprecated use {@link #holdoutDecisions}
+     */
+    @Deprecated
     @Nullable
     public FeatureDecision holdoutDecision;
 
+    @Nonnull
+    public List<FeatureDecision> holdoutDecisions = new ArrayList<>();
+
+    /**
+     * @deprecated use {@link #holdoutDecisions}
+     */
+    @Deprecated
     public void setHoldoutDecision(@Nullable FeatureDecision holdoutDecision) {
         this.holdoutDecision = holdoutDecision;
     }