|
1 | 1 | # frozen_string_literal: true |
2 | 2 |
|
3 | 3 | # |
4 | | -# Copyright 2017-2020, 2023, Optimizely and contributors |
| 4 | +# Copyright 2017-2020, 2023, 2026, Optimizely and contributors |
5 | 5 | # |
6 | 6 | # Licensed under the Apache License, Version 2.0 (the "License"); |
7 | 7 | # you may not use this file except in compliance with the License. |
|
1386 | 1386 | end |
1387 | 1387 | end |
1388 | 1388 | end |
| 1389 | + |
| 1390 | + describe 'local holdout hit continues rule evaluation' do |
| 1391 | + let(:spy_event_processor) { spy('event_processor') } |
| 1392 | + let(:holdout_source) { Optimizely::DecisionService::DECISION_SOURCES['HOLDOUT'] } |
| 1393 | + let(:chrome_user) { {'browser_type' => 'chrome'} } |
| 1394 | + |
| 1395 | + def local_holdout(id, rule_ids) |
| 1396 | + { |
| 1397 | + 'id' => id, |
| 1398 | + 'key' => id, |
| 1399 | + 'status' => 'Running', |
| 1400 | + 'audiences' => [], |
| 1401 | + 'includedRules' => rule_ids, |
| 1402 | + 'variations' => [{'id' => "#{id}_var", 'key' => 'holdout', 'featureEnabled' => false}], |
| 1403 | + 'trafficAllocation' => [{'entityId' => "#{id}_var", 'endOfRange' => 10_000}] |
| 1404 | + } |
| 1405 | + end |
| 1406 | + |
| 1407 | + # Global holdouts in the shared fixture bucket every user, so they are removed here. |
| 1408 | + def datafile(local_holdouts: nil, send_flag_decisions: true) |
| 1409 | + body = OptimizelySpec.deep_clone(OptimizelySpec::CONFIG_BODY_WITH_HOLDOUTS) |
| 1410 | + body['holdouts'] = [] |
| 1411 | + body['localHoldouts'] = local_holdouts if local_holdouts |
| 1412 | + body['sendFlagDecisions'] = send_flag_decisions |
| 1413 | + body |
| 1414 | + end |
| 1415 | + |
| 1416 | + def build_project(body, user_profile_service: nil, cmab_service: nil) |
| 1417 | + project = Optimizely::Project.new( |
| 1418 | + datafile: JSON.dump(body), |
| 1419 | + logger: spy_logger, |
| 1420 | + error_handler: error_handler, |
| 1421 | + event_processor: spy_event_processor, |
| 1422 | + user_profile_service: user_profile_service, |
| 1423 | + cmab_service: cmab_service |
| 1424 | + ) |
| 1425 | + allow(project).to receive(:send_impression).and_wrap_original do |original, *args| |
| 1426 | + @impressions << args |
| 1427 | + original.call(*args) |
| 1428 | + end |
| 1429 | + @projects << project |
| 1430 | + project |
| 1431 | + end |
| 1432 | + |
| 1433 | + def holdout_impressions |
| 1434 | + @impressions.select { |args| args[6] == holdout_source }.map { |args| args[1]['id'] } |
| 1435 | + end |
| 1436 | + |
| 1437 | + before do |
| 1438 | + @projects = [] |
| 1439 | + @impressions = [] |
| 1440 | + end |
| 1441 | + after { @projects.each(&:close) } |
| 1442 | + |
| 1443 | + it 'serves a later delivery rule after a holdout hit without jumping to Everyone Else' do |
| 1444 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_a', ['177770'])])) |
| 1445 | + decision = project.create_user_context('user', chrome_user).decide('boolean_single_variable_feature') |
| 1446 | + |
| 1447 | + expect(decision.rule_key).to eq('177772') |
| 1448 | + expect(decision.variation_key).to eq('177773') |
| 1449 | + expect(holdout_impressions).to eq(['lh_a']) |
| 1450 | + end |
| 1451 | + |
| 1452 | + it 'sends one holdout event per held-out rule' do |
| 1453 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_a', ['177770']), local_holdout('lh_b', ['177772'])])) |
| 1454 | + decision = project.create_user_context('user', chrome_user).decide('boolean_single_variable_feature') |
| 1455 | + |
| 1456 | + expect(decision.rule_key).to eq('177776') |
| 1457 | + expect(holdout_impressions).to eq(%w[lh_a lh_b]) |
| 1458 | + expect(project).to have_received(:send_impression) |
| 1459 | + .with(anything, anything, '177778', 'boolean_single_variable_feature', '177776', true, 'rollout', 'user', anything, anything) |
| 1460 | + end |
| 1461 | + |
| 1462 | + it 'sends holdout events when no rule serves' do |
| 1463 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_all', %w[177770 177772 177776])])) |
| 1464 | + decision = project.create_user_context('user', chrome_user).decide('boolean_single_variable_feature') |
| 1465 | + |
| 1466 | + expect(decision.variation_key).to be_nil |
| 1467 | + expect(decision.enabled).to be false |
| 1468 | + expect(holdout_impressions).to eq(%w[lh_all lh_all lh_all]) |
| 1469 | + end |
| 1470 | + |
| 1471 | + it 'sends two holdout events when one local holdout targets two hit rules' do |
| 1472 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_two', %w[177770 177772])])) |
| 1473 | + decision = project.create_user_context('user', chrome_user).decide('boolean_single_variable_feature') |
| 1474 | + |
| 1475 | + expect(decision.rule_key).to eq('177776') |
| 1476 | + expect(holdout_impressions).to eq(%w[lh_two lh_two]) |
| 1477 | + end |
| 1478 | + |
| 1479 | + it 'sends the holdout event for a delivery rule even when sendFlagDecisions is off' do |
| 1480 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_a', ['177770'])], send_flag_decisions: false)) |
| 1481 | + decision = project.create_user_context('user', chrome_user).decide('boolean_single_variable_feature') |
| 1482 | + |
| 1483 | + expect(decision.rule_key).to eq('177772') |
| 1484 | + expect(@impressions.length).to eq(1) |
| 1485 | + expect(holdout_impressions).to eq(['lh_a']) |
| 1486 | + end |
| 1487 | + |
| 1488 | + it 'suppresses local holdout events with DISABLE_DECISION_EVENT' do |
| 1489 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_a', ['177770']), local_holdout('lh_b', ['177772'])])) |
| 1490 | + decision = project.create_user_context('user', chrome_user).decide( |
| 1491 | + 'boolean_single_variable_feature', |
| 1492 | + [Optimizely::Decide::OptimizelyDecideOption::DISABLE_DECISION_EVENT] |
| 1493 | + ) |
| 1494 | + |
| 1495 | + expect(decision.rule_key).to eq('177776') |
| 1496 | + expect(@impressions).to be_empty |
| 1497 | + end |
| 1498 | + |
| 1499 | + it 'serves the second experiment rule after being held out of the first' do |
| 1500 | + body = datafile |
| 1501 | + body['featureFlags'].find { |f| f['key'] == 'boolean_feature' }['experimentIds'] = %w[122227 111127] |
| 1502 | + project = build_project(body) |
| 1503 | + |
| 1504 | + decision = project.create_user_context('user', {}).decide('boolean_feature') |
| 1505 | + |
| 1506 | + expect(decision.rule_key).to eq('test_experiment') |
| 1507 | + expect(holdout_impressions).to eq(['holdout_local_1']) |
| 1508 | + expect(project).to have_received(:send_impression) |
| 1509 | + .with(anything, anything, decision.variation_key, 'boolean_feature', 'test_experiment', true, 'feature-test', 'user', anything, anything) |
| 1510 | + |
| 1511 | + config = project.config_manager.config |
| 1512 | + result = Optimizely::DecisionService.new(spy_logger, spy_cmab_service).get_decision_for_flag( |
| 1513 | + config.feature_flag_key_map['boolean_feature'], |
| 1514 | + project.create_user_context('user', {}), |
| 1515 | + config |
| 1516 | + ) |
| 1517 | + expect(result.decision.experiment['id']).to eq('111127') |
| 1518 | + expect(result.holdout_decisions.map { |d| d.experiment['id'] }).to eq(['holdout_local_1']) |
| 1519 | + expect(result.holdout_decision).to eq(result.holdout_decisions.first) |
| 1520 | + end |
| 1521 | + |
| 1522 | + it 'discards earlier local holdout hits when a later rule ends in a decision error' do |
| 1523 | + body = datafile |
| 1524 | + body['featureFlags'].find { |f| f['key'] == 'boolean_feature' }['experimentIds'] = %w[122227 111127] |
| 1525 | + body['experiments'].find { |e| e['id'] == '111127' }['cmab'] = {'attributeIds' => [], 'trafficAllocation' => 10_000} |
| 1526 | + failing_cmab = double('cmab_service') |
| 1527 | + allow(failing_cmab).to receive(:get_decision).and_raise(StandardError, 'CMAB request failed') |
| 1528 | + project = build_project(body, cmab_service: failing_cmab) |
| 1529 | + |
| 1530 | + decisions = project.create_user_context('user', chrome_user).decide_for_keys(%w[boolean_feature boolean_single_variable_feature]) |
| 1531 | + |
| 1532 | + config = project.config_manager.config |
| 1533 | + result = Optimizely::DecisionService.new(spy_logger, failing_cmab).get_decision_for_flag( |
| 1534 | + config.feature_flag_key_map['boolean_feature'], |
| 1535 | + project.create_user_context('user', chrome_user), |
| 1536 | + config |
| 1537 | + ) |
| 1538 | + expect(result.error).to be true |
| 1539 | + expect(result.holdout_decisions.map { |d| d.experiment['id'] }).to eq(['holdout_local_1']) |
| 1540 | + |
| 1541 | + expect(decisions['boolean_feature'].variation_key).to be_nil |
| 1542 | + expect(decisions['boolean_feature'].reasons).to include('Failed to fetch CMAB data for experiment test_experiment.') |
| 1543 | + expect(@impressions.map { |args| args[3] }).not_to include('boolean_feature') |
| 1544 | + expect(holdout_impressions).to be_empty |
| 1545 | + expect(decisions['boolean_single_variable_feature'].variation_key).not_to be_nil |
| 1546 | + expect(@impressions.map { |args| args[3] }).to include('boolean_single_variable_feature') |
| 1547 | + end |
| 1548 | + |
| 1549 | + it 'applies the local holdout over a stored sticky variation' do |
| 1550 | + ups = double('user_profile_service', save: nil) |
| 1551 | + allow(ups).to receive(:lookup).with('user').and_return( |
| 1552 | + user_id: 'user', |
| 1553 | + experiment_bucket_map: {'122227' => {variation_id: '122228'}} |
| 1554 | + ) |
| 1555 | + project = build_project(datafile, user_profile_service: ups) |
| 1556 | + |
| 1557 | + decision = project.create_user_context('user', {}).decide('boolean_feature') |
| 1558 | + |
| 1559 | + expect(decision.variation_key).not_to eq('control_with_audience') |
| 1560 | + expect(decision.rule_key).not_to eq('test_experiment_with_audience') |
| 1561 | + expect(decision.variation_key).to be_nil |
| 1562 | + expect(decision.rule_key).to be_nil |
| 1563 | + expect(decision.enabled).to be false |
| 1564 | + expect(holdout_impressions).to eq(['holdout_local_1']) |
| 1565 | + end |
| 1566 | + |
| 1567 | + it 'skips a held-out rule without a holdout event in legacy APIs' do |
| 1568 | + project = build_project(datafile(local_holdouts: [local_holdout('lh_a', %w[177770 177772])])) |
| 1569 | + |
| 1570 | + expect(project.is_feature_enabled('boolean_single_variable_feature', 'user', chrome_user)).to be true |
| 1571 | + expect(holdout_impressions).to eq([]) |
| 1572 | + end |
| 1573 | + end |
1389 | 1574 | end |
0 commit comments