Skip to content

Commit b07d7b4

Browse files
alexjoeyyongalexjoeyyong
committed
[AI-FSSDK] [FSSDK-12958] Continue rule evaluation after local holdout hit
1 parent f308ecb commit b07d7b4

4 files changed

Lines changed: 402 additions & 82 deletions

File tree

‎lib/core/decision_service/index.spec.ts‎

Lines changed: 256 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/**
2-
* Copyright 2025, Optimizely
2+
* Copyright 2025-2026, Optimizely
33
*
44
* Licensed under the Apache License, Version 2.0 (the "License");
55
* you may not use this file except in compliance with the License.
@@ -2212,6 +2212,11 @@ describe('DecisionService', () => {
22122212
return datafile;
22132213
};
22142214

2215+
const globalHit = (config: ProjectConfig) => ({
2216+
experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
2217+
variation: config.variationIdMap['holdout_variation_running_id'],
2218+
});
2219+
22152220
it('should return holdout variation immediately when excludeTargetedDeliveries is false', async () => {
22162221
const { decisionService } = getDecisionService();
22172222
const config = createProjectConfig(JSON.stringify(getExcludeTDDatafile(false)));
@@ -2281,10 +2286,8 @@ describe('DecisionService', () => {
22812286
experiment: config.experimentKeyMap['delivery_1'],
22822287
variation: config.variationIdMap['5004'],
22832288
decisionSource: DECISION_SOURCES.ROLLOUT,
2284-
holdout: {
2285-
experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
2286-
variation: config.variationIdMap['holdout_variation_running_id'],
2287-
},
2289+
holdout: globalHit(config),
2290+
holdouts: [globalHit(config)],
22882291
});
22892292

22902293
const reasonStrings = variation.reasons.map((r: any) => typeof r === 'string' ? r : r[0]);
@@ -2317,10 +2320,8 @@ describe('DecisionService', () => {
23172320
experiment: null,
23182321
variation: null,
23192322
decisionSource: DECISION_SOURCES.ROLLOUT,
2320-
holdout: {
2321-
experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
2322-
variation: config.variationIdMap['holdout_variation_running_id'],
2323-
},
2323+
holdout: globalHit(config),
2324+
holdouts: [globalHit(config)],
23242325
});
23252326
});
23262327

@@ -2353,10 +2354,8 @@ describe('DecisionService', () => {
23532354
experiment: null,
23542355
variation: null,
23552356
decisionSource: DECISION_SOURCES.ROLLOUT,
2356-
holdout: {
2357-
experiment: config.holdoutIdMap && config.holdoutIdMap['holdout_running_id'],
2358-
variation: config.variationIdMap['holdout_variation_running_id'],
2359-
},
2357+
holdout: globalHit(config),
2358+
holdouts: [globalHit(config)],
23602359
});
23612360
});
23622361
});
@@ -3076,6 +3075,11 @@ describe('DecisionService', () => {
30763075
return datafile;
30773076
};
30783077

3078+
const localHit = (config: ProjectConfig) => ({
3079+
experiment: config.holdoutIdMap!['local_holdout_id'],
3080+
variation: config.variationIdMap['local_holdout_variation_id'],
3081+
});
3082+
30793083
beforeEach(() => {
30803084
mockBucket.mockReset();
30813085
});
@@ -3118,12 +3122,10 @@ describe('DecisionService', () => {
31183122
expect(value[0].result.experiment?.id).toBe('global_holdout_id');
31193123
});
31203124

3121-
it('local holdout hit branch: user bucketed into local holdout for experiment rule returns holdout variation; audience and traffic not evaluated for that rule', async () => {
3122-
// exp_1 has id '2001'
3125+
it('local holdout hit on an experiment rule excludes the user from that rule only; with no other rule serving the decision is null with the hit recorded', async () => {
31233126
const config = createProjectConfig(JSON.stringify(makeLocalHoldoutDatafile('2001')));
31243127
const { decisionService } = getDecisionService();
31253128

3126-
// bucket returns holdout variation when evaluating the local holdout
31273129
mockBucket.mockImplementation((params: BucketerParams) => {
31283130
if (params.experimentId === 'local_holdout_id') {
31293131
return { result: 'local_holdout_variation_id', reasons: [] };
@@ -3135,10 +3137,70 @@ describe('DecisionService', () => {
31353137
const feature = config.featureKeyMap['flag_1'];
31363138
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
31373139

3138-
// Should return holdout decision for the local holdout
3139-
expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.HOLDOUT);
3140-
expect(value[0].result.experiment?.id).toBe('local_holdout_id');
3141-
expect(value[0].result.variation?.id).toBe('local_holdout_variation_id');
3140+
const hit = localHit(config);
3141+
expect(value[0].result).toEqual({
3142+
experiment: null,
3143+
variation: null,
3144+
decisionSource: DECISION_SOURCES.ROLLOUT,
3145+
holdout: hit,
3146+
holdouts: [hit],
3147+
});
3148+
expect(mockBucket.mock.calls.some(([p]) => p.experimentId === '2001')).toBe(false);
3149+
});
3150+
3151+
it('local holdout hit on an experiment rule continues to the next experiment rule, which serves', async () => {
3152+
const config = createProjectConfig(JSON.stringify(makeLocalHoldoutDatafile('2001')));
3153+
const { decisionService } = getDecisionService();
3154+
3155+
mockBucket.mockImplementation((params: BucketerParams) => {
3156+
if (params.experimentId === 'local_holdout_id') {
3157+
return { result: 'local_holdout_variation_id', reasons: [] };
3158+
}
3159+
if (params.experimentId === '2001') {
3160+
return { result: '5001', reasons: [] };
3161+
}
3162+
if (params.experimentId === '2002') {
3163+
return { result: '5002', reasons: [] };
3164+
}
3165+
return { result: null, reasons: [] };
3166+
});
3167+
3168+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
3169+
const feature = config.featureKeyMap['flag_1'];
3170+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3171+
3172+
const hit = localHit(config);
3173+
expect(value[0].result).toEqual({
3174+
experiment: config.experimentIdMap['2002'],
3175+
variation: config.variationIdMap['5002'],
3176+
decisionSource: DECISION_SOURCES.FEATURE_TEST,
3177+
holdout: hit,
3178+
holdouts: [hit],
3179+
});
3180+
});
3181+
3182+
it('local holdout hit on an experiment rule continues to a delivery rule when no later experiment serves', async () => {
3183+
const config = createProjectConfig(JSON.stringify(makeLocalHoldoutDatafile('2001')));
3184+
const { decisionService } = getDecisionService();
3185+
3186+
mockBucket.mockImplementation((params: BucketerParams) => {
3187+
if (params.experimentId === 'local_holdout_id') {
3188+
return { result: 'local_holdout_variation_id', reasons: [] };
3189+
}
3190+
if (params.experimentId === '3001') {
3191+
return { result: '5004', reasons: [] };
3192+
}
3193+
return { result: null, reasons: [] };
3194+
});
3195+
3196+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
3197+
const feature = config.featureKeyMap['flag_1'];
3198+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3199+
3200+
expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.ROLLOUT);
3201+
expect(value[0].result.experiment?.id).toBe('3001');
3202+
expect(value[0].result.variation?.id).toBe('5004');
3203+
expect(value[0].result.holdouts).toEqual([localHit(config)]);
31423204
});
31433205

31443206
it('local holdout miss branch: user not bucketed into local holdout falls through to regular rule evaluation', async () => {
@@ -3202,31 +3264,191 @@ describe('DecisionService', () => {
32023264
expect(value[0].result.experiment?.id).toBe('2001');
32033265
});
32043266

3205-
it('local holdout applies to delivery rules (rollouts) as well as experiment rules', async () => {
3206-
// delivery_1 has id '3001'
3267+
it('local holdout hit on a delivery rule continues to the next delivery rule, not straight to everyone else', async () => {
32073268
const config = createProjectConfig(JSON.stringify(makeLocalHoldoutDatafile('3001')));
32083269
const { decisionService } = getDecisionService();
32093270

3210-
// bucket returns null for all experiments and the local holdout variation for delivery rule
32113271
mockBucket.mockImplementation((params: BucketerParams) => {
32123272
if (params.experimentId === 'local_holdout_id') {
32133273
return { result: 'local_holdout_variation_id', reasons: [] };
32143274
}
3275+
if (params.experimentId === '3002') {
3276+
return { result: '5005', reasons: [] };
3277+
}
3278+
if (params.experimentId === 'default-rollout-id') {
3279+
return { result: '5007', reasons: [] };
3280+
}
32153281
return { result: null, reasons: [] };
32163282
});
32173283

3218-
// No audience attributes → experiments won't match, falls through to rollout
3219-
const user = new OptimizelyUserContext({
3220-
optimizely: {} as any,
3221-
userId: 'user1',
3222-
attributes: { age: 15 }, // satisfies 4001 used by delivery_1
3284+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
3285+
const feature = config.featureKeyMap['flag_1'];
3286+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3287+
3288+
const hit = localHit(config);
3289+
expect(value[0].result).toEqual({
3290+
experiment: config.experimentIdMap['3002'],
3291+
variation: config.variationIdMap['5005'],
3292+
decisionSource: DECISION_SOURCES.ROLLOUT,
3293+
holdout: hit,
3294+
holdouts: [hit],
32233295
});
3296+
});
3297+
3298+
it('one local holdout targeting two rules records a hit for each rule', async () => {
3299+
const config = createProjectConfig(JSON.stringify(makeLocalHoldoutDatafile('2001', ['2001', '3001'])));
3300+
const { decisionService } = getDecisionService();
3301+
3302+
mockBucket.mockImplementation((params: BucketerParams) => {
3303+
if (params.experimentId === 'local_holdout_id') {
3304+
return { result: 'local_holdout_variation_id', reasons: [] };
3305+
}
3306+
if (params.experimentId === '3002') {
3307+
return { result: '5005', reasons: [] };
3308+
}
3309+
return { result: null, reasons: [] };
3310+
});
3311+
3312+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
32243313
const feature = config.featureKeyMap['flag_1'];
32253314
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
32263315

3227-
// Should be a holdout decision from the local holdout targeting the delivery rule
3228-
expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.HOLDOUT);
3229-
expect(value[0].result.experiment?.id).toBe('local_holdout_id');
3316+
expect(value[0].result.experiment?.id).toBe('3002');
3317+
expect(value[0].result.holdouts).toEqual([localHit(config), localHit(config)]);
3318+
});
3319+
3320+
it('holdouts list is ordered by evaluation: experiment-rule hits before delivery-rule hits, and holdout is the first entry', async () => {
3321+
const datafile = makeLocalHoldoutDatafile('2002');
3322+
(datafile as any).localHoldouts.push({
3323+
...(datafile as any).localHoldouts[0],
3324+
id: 'local_holdout_2_id',
3325+
key: 'local_holdout_2',
3326+
includedRules: ['3001'],
3327+
variations: [{ id: 'local_holdout_2_variation_id', key: 'local_holdout_2_variation', variables: [] }],
3328+
trafficAllocation: [{ entityId: 'local_holdout_2_variation_id', endOfRange: 10000 }],
3329+
});
3330+
const config = createProjectConfig(JSON.stringify(datafile));
3331+
const { decisionService } = getDecisionService();
3332+
3333+
mockBucket.mockImplementation((params: BucketerParams) => {
3334+
if (params.experimentId === 'local_holdout_id') {
3335+
return { result: 'local_holdout_variation_id', reasons: [] };
3336+
}
3337+
if (params.experimentId === 'local_holdout_2_id') {
3338+
return { result: 'local_holdout_2_variation_id', reasons: [] };
3339+
}
3340+
if (params.experimentId === '3002') {
3341+
return { result: '5005', reasons: [] };
3342+
}
3343+
return { result: null, reasons: [] };
3344+
});
3345+
3346+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
3347+
const feature = config.featureKeyMap['flag_1'];
3348+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3349+
3350+
const expHit = localHit(config);
3351+
const deliveryHit = {
3352+
experiment: config.holdoutIdMap!['local_holdout_2_id'],
3353+
variation: config.variationIdMap['local_holdout_2_variation_id'],
3354+
};
3355+
expect(value[0].result.experiment?.id).toBe('3002');
3356+
expect(value[0].result.holdouts).toEqual([expHit, deliveryHit]);
3357+
expect(value[0].result.holdout).toEqual(expHit);
3358+
});
3359+
3360+
it('global excludeTargetedDeliveries holdout hit is listed before local delivery-rule hits', async () => {
3361+
const datafile = makeLocalHoldoutDatafile('3001');
3362+
(datafile as any).holdouts = [{
3363+
id: 'global_holdout_id',
3364+
key: 'global_holdout',
3365+
status: 'Running',
3366+
includedFlags: [],
3367+
excludedFlags: [],
3368+
excludeTargetedDeliveries: true,
3369+
audienceIds: [],
3370+
audienceConditions: [],
3371+
variations: [{ id: 'global_holdout_var_id', key: 'global_holdout_var', variables: [] }],
3372+
trafficAllocation: [{ entityId: 'global_holdout_var_id', endOfRange: 10000 }],
3373+
}];
3374+
const config = createProjectConfig(JSON.stringify(datafile));
3375+
const { decisionService } = getDecisionService();
3376+
3377+
mockBucket.mockImplementation((params: BucketerParams) => {
3378+
if (params.experimentId === 'global_holdout_id') {
3379+
return { result: 'global_holdout_var_id', reasons: [] };
3380+
}
3381+
if (params.experimentId === 'local_holdout_id') {
3382+
return { result: 'local_holdout_variation_id', reasons: [] };
3383+
}
3384+
if (params.experimentId === '3002') {
3385+
return { result: '5005', reasons: [] };
3386+
}
3387+
return { result: null, reasons: [] };
3388+
});
3389+
3390+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
3391+
const feature = config.featureKeyMap['flag_1'];
3392+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3393+
3394+
const globalHit = {
3395+
experiment: config.holdoutIdMap!['global_holdout_id'],
3396+
variation: config.variationIdMap['global_holdout_var_id'],
3397+
};
3398+
expect(value[0].result.experiment?.id).toBe('3002');
3399+
expect(value[0].result.holdouts).toEqual([globalHit, localHit(config)]);
3400+
expect(value[0].result.holdout).toEqual(globalHit);
3401+
});
3402+
3403+
it('when several local holdouts match one rule only the first is recorded', async () => {
3404+
const datafile = makeLocalHoldoutDatafile('2001');
3405+
(datafile as any).localHoldouts.push({
3406+
...(datafile as any).localHoldouts[0],
3407+
id: 'local_holdout_2_id',
3408+
key: 'local_holdout_2',
3409+
variations: [{ id: 'local_holdout_2_variation_id', key: 'local_holdout_2_variation', variables: [] }],
3410+
trafficAllocation: [{ entityId: 'local_holdout_2_variation_id', endOfRange: 10000 }],
3411+
});
3412+
const config = createProjectConfig(JSON.stringify(datafile));
3413+
const { decisionService } = getDecisionService();
3414+
3415+
mockBucket.mockImplementation((params: BucketerParams) => {
3416+
if (params.experimentId === 'local_holdout_id') {
3417+
return { result: 'local_holdout_variation_id', reasons: [] };
3418+
}
3419+
if (params.experimentId === 'local_holdout_2_id') {
3420+
return { result: 'local_holdout_2_variation_id', reasons: [] };
3421+
}
3422+
return { result: null, reasons: [] };
3423+
});
3424+
3425+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1' });
3426+
const feature = config.featureKeyMap['flag_1'];
3427+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3428+
3429+
expect(value[0].result.holdouts).toEqual([localHit(config)]);
3430+
});
3431+
3432+
it('user with no local holdout hit gets no holdout fields on the decision', async () => {
3433+
const config = createProjectConfig(JSON.stringify(makeLocalHoldoutDatafile('2001')));
3434+
const { decisionService } = getDecisionService();
3435+
3436+
mockBucket.mockImplementation((params: BucketerParams) => {
3437+
if (params.experimentId === '2001') {
3438+
return { result: '5001', reasons: [] };
3439+
}
3440+
return { result: null, reasons: [] };
3441+
});
3442+
3443+
const user = new OptimizelyUserContext({ optimizely: {} as any, userId: 'user1', attributes: { age: 15 } });
3444+
const feature = config.featureKeyMap['flag_1'];
3445+
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
3446+
3447+
expect(value[0].result).toEqual({
3448+
experiment: config.experimentIdMap['2001'],
3449+
variation: config.variationIdMap['5001'],
3450+
decisionSource: DECISION_SOURCES.FEATURE_TEST,
3451+
});
32303452
});
32313453

32323454
it('forced decision beats a 100% traffic local holdout: forced decision takes precedence over local holdout', async () => {
@@ -3288,10 +3510,9 @@ describe('DecisionService', () => {
32883510
const feature = config.featureKeyMap['flag_1'];
32893511
const value = await decisionService.resolveVariationsForFeatureList('async', config, [feature], user, {}).get();
32903512

3291-
// Local holdout applies normally even with excludeTargetedDeliveries set
3292-
expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.HOLDOUT);
3293-
expect(value[0].result.experiment?.id).toBe('local_holdout_id');
3294-
expect(value[0].result.variation?.id).toBe('local_holdout_variation_id');
3513+
expect(value[0].result.decisionSource).toBe(DECISION_SOURCES.ROLLOUT);
3514+
expect(value[0].result.variation).toBeNull();
3515+
expect(value[0].result.holdouts).toEqual([localHit(config)]);
32953516
});
32963517
});
32973518
});

0 commit comments

Comments
 (0)