Skip to content

Commit 222b812

Browse files
committed
[AI-FSSDK] [FSSDK-12958] Add test for holdout hit then CMAB decision error
1 parent 0a7dc91 commit 222b812

1 file changed

Lines changed: 45 additions & 0 deletions

File tree

‎tests/test_decision_service_holdout.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1899,6 +1899,51 @@ def test_local_holdout_hit_on_first_experiment_then_second_experiment_serves(sel
18991899
self.assertEqual(mock_send.call_count, 2)
19001900
self.assertEqual(mock_send.call_args_list[1].args[5], str(enums.DecisionSources.FEATURE_TEST))
19011901

1902+
def test_local_holdout_hit_then_later_rule_error_sends_no_impressions(self):
1903+
"""Held out of the first rule; a CMAB error on the later rule yields an error decision and no events."""
1904+
cfg = copy.deepcopy(self.config_dict_with_features)
1905+
cfg['experiments'].append({
1906+
'key': 'cmab_experiment',
1907+
'status': 'Running',
1908+
'forcedVariations': {},
1909+
'layerId': '111183',
1910+
'audienceIds': [],
1911+
'trafficAllocation': [{'entityId': '111141', 'endOfRange': 10000}],
1912+
'id': '111140',
1913+
'variations': [{'key': 'cmab_variation', 'id': '111141', 'featureEnabled': True, 'variables': []}],
1914+
'cmab': {'attributeIds': [], 'trafficAllocation': 10000},
1915+
})
1916+
for flag in cfg['featureFlags']:
1917+
if flag['key'] == 'test_feature_in_experiment':
1918+
flag['experimentIds'] = ['111127', '111140']
1919+
cfg['holdouts'] = []
1920+
cfg['localHoldouts'] = [_holdout('lh_exp', 'for_exp', included_rules=['111127'])]
1921+
self.opt_obj = opt = optimizely_module.Optimizely(json.dumps(cfg))
1922+
config = opt.config_manager.get_config()
1923+
holdout_hit = ("The user 'test_user' is bucketed into local holdout 'for_exp' "
1924+
"for experiment rule 'test_experiment'.")
1925+
cmab_error = enums.Errors.CMAB_FETCH_FAILED_DETAILED.format('cmab_experiment')
1926+
1927+
with mock.patch.object(opt.decision_service.cmab_service, 'get_decision', side_effect=Exception('boom')):
1928+
result = opt.decision_service.get_decision_for_flag(
1929+
config.get_feature_from_key('test_feature_in_experiment'),
1930+
opt.create_user_context('test_user', {}), config
1931+
)
1932+
self.assertTrue(result['error'])
1933+
self.assertEqual(result['decision'].experiment.id, '111140')
1934+
self.assertNotIn('holdout_decisions', result)
1935+
self.assertTrue(any(r.startswith(holdout_hit) for r in result['reasons']))
1936+
self.assertIn(cmab_error, result['reasons'])
1937+
1938+
with mock.patch.object(opt, '_send_impression_event', wraps=opt._send_impression_event) as mock_send:
1939+
decision = opt.create_user_context('test_user', {}).decide('test_feature_in_experiment')
1940+
1941+
self.assertIsNone(decision.variation_key)
1942+
self.assertIsNone(decision.rule_key)
1943+
self.assertFalse(decision.enabled)
1944+
self.assertIn(cmab_error, decision.reasons)
1945+
mock_send.assert_not_called()
1946+
19021947

19031948
# ---------------------------------------------------------------------------
19041949
# Exclude Targeted Deliveries Tests

0 commit comments

Comments
 (0)