From 006e947fce0a033e7a8321859411377f4a036164 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Thu, 5 Nov 2026 14:40:38 +0530 Subject: [PATCH 1/3] feat: Record exposures under local evaluation Carry each feature state's experiment metadata from the environment document into the engine context, and set it and the variant on flags built from evaluation results, so getExperimentFlag records exposures under local evaluation as it does under remote evaluation. Environment flags carry no experiment, matching the API, which only sends experiment metadata for identity flags. --- .../evaluation/evaluationContext/mappers.ts | 15 +-- flagsmith-engine/evaluation/models.ts | 4 + flagsmith-engine/features/models.ts | 10 ++ flagsmith-engine/features/util.ts | 2 + sdk/index.ts | 7 +- sdk/models.ts | 10 +- .../sdk/data/environment-with-experiment.json | 97 +++++++++++++++++++ tests/sdk/flagsmith-experiments.test.ts | 87 +++++++++++++++-- 8 files changed, 209 insertions(+), 23 deletions(-) create mode 100644 tests/sdk/data/environment-with-experiment.json diff --git a/flagsmith-engine/evaluation/evaluationContext/mappers.ts b/flagsmith-engine/evaluation/evaluationContext/mappers.ts index 6fff9db..2a0a0c2 100644 --- a/flagsmith-engine/evaluation/evaluationContext/mappers.ts +++ b/flagsmith-engine/evaluation/evaluationContext/mappers.ts @@ -82,9 +82,7 @@ function mapEnvironmentModelToEvaluationContext( value: fs.getValue(), variants: mapFeatureStateVariants(fs), priority: fs.featureSegment?.priority, - metadata: { - id: fs.feature.id - } + metadata: mapFeatureStateMetadata(fs) }; } @@ -104,9 +102,7 @@ function mapEnvironmentModelToEvaluationContext( value: fs.getValue(), variants: mapFeatureStateVariants(fs), priority: fs.featureSegment?.priority, - metadata: { - id: fs.feature.id - } + metadata: mapFeatureStateMetadata(fs) })) : [], metadata: { @@ -154,6 +150,13 @@ function mapIdentityModelToIdentityContext( return identityContext; } +function mapFeatureStateMetadata(fs: FeatureStateModel): SDKFeatureMetadata { + return { + id: fs.feature.id, + ...(fs.experiment && { experiment: fs.experiment }) + }; +} + function mapFeatureStateVariants(fs: FeatureStateModel) { return fs.multivariateFeatureStateValues?.length > 0 ? fs.multivariateFeatureStateValues.map(mv => ({ diff --git a/flagsmith-engine/evaluation/models.ts b/flagsmith-engine/evaluation/models.ts index 72f98dd..5fc5892 100644 --- a/flagsmith-engine/evaluation/models.ts +++ b/flagsmith-engine/evaluation/models.ts @@ -9,6 +9,8 @@ import type { SegmentMetadata } from './evaluationResult/evaluationResult.types.js'; +import type { FeatureStateExperiment } from '../features/models.js'; + import type { FeatureContext, EnvironmentContext, @@ -26,6 +28,8 @@ export enum SegmentSource { // Feature types export interface SDKFeatureMetadata extends FeatureMetadata { id: number; + /** The running experiment the feature state belongs to, if any. */ + experiment?: FeatureStateExperiment; } export interface FeatureContextWithMetadata diff --git a/flagsmith-engine/features/models.ts b/flagsmith-engine/features/models.ts index 36fbb10..26cc771 100644 --- a/flagsmith-engine/features/models.ts +++ b/flagsmith-engine/features/models.ts @@ -48,12 +48,22 @@ export class MultivariateFeatureStateValueModel { } } +/** + * The running experiment a feature state belongs to, as the environment document reports it. + */ +export interface FeatureStateExperiment { + id: number; + name: string; + in_experiment: boolean; +} + export class FeatureStateModel { feature: FeatureModel; enabled: boolean; djangoID: number; featurestateUUID: string = uuidv4(); featureSegment?: FeatureSegment; + experiment?: FeatureStateExperiment; private value: any; multivariateFeatureStateValues: MultivariateFeatureStateValueModel[] = []; diff --git a/flagsmith-engine/features/util.ts b/flagsmith-engine/features/util.ts index ac90bf1..f23fb0c 100644 --- a/flagsmith-engine/features/util.ts +++ b/flagsmith-engine/features/util.ts @@ -23,6 +23,8 @@ export function buildFeatureStateModel(featuresStateModelJSON: any): FeatureStat ? buildFeatureSegment(featuresStateModelJSON.feature_segment) : undefined; + featureStateModel.experiment = featuresStateModelJSON.metadata?.experiment ?? undefined; + const multivariateFeatureStateValues = featuresStateModelJSON.multivariate_feature_state_values ? featuresStateModelJSON.multivariate_feature_state_values.map((fsv: any) => { const featureOption = new MultivariateFeatureOptionModel( diff --git a/sdk/index.ts b/sdk/index.ts index e262278..7836d3e 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -289,9 +289,6 @@ export class Flagsmith { * Flags are fetched exactly as {@link getIdentityFlags} fetches them, so traits are upserted and * the identity cache is used when one is configured. * - * Experiment metadata is only produced by remote identity evaluation. Local evaluation and - * offline mode never carry it, so no exposure is ever recorded for them. - * * @param featureName the name of the feature to evaluate. * @param identifier a unique identifier for the identity in the current environment. * @param traits? a dictionary of traits to add / update on the identity in Flagsmith. @@ -638,6 +635,10 @@ export class Flagsmith { } const evaluationResult = getEvaluationResult(context as EvaluationContextWithMetadata); const flags = Flags.fromEvaluationResult(evaluationResult); + // Only an identity is enrolled in an experiment, so environment flags carry none. + for (const flag of Object.values(flags.flags)) { + flag.experiment = undefined; + } if (!!this.cache) { await this.cache.set('flags', flags); diff --git a/sdk/models.ts b/sdk/models.ts index c4bb51b..c8c32ea 100644 --- a/sdk/models.ts +++ b/sdk/models.ts @@ -43,7 +43,7 @@ export class DefaultFlag extends BaseFlag { } /** - * A running experiment on a feature, as reported by a remote identity evaluation. + * A running experiment on a feature, as reported by an identity evaluation. */ export class ExperimentMetadata { /** @@ -110,11 +110,11 @@ export class Flag extends BaseFlag { */ reason?: string; /** - * The variant key this identity was bucketed into. Set by remote evaluation only. + * The variant key this identity was bucketed into. Set by identity evaluation only. */ variant?: string; /** - * The experiment running on this feature, if any. Set by remote identity evaluation only. + * The experiment running on this feature, if any. Set by identity evaluation only. */ experiment?: ExperimentMetadata; @@ -196,7 +196,9 @@ export class Flags { value: flag.value ?? null, featureId: flagMetadataId, featureName: flag.name, - reason: flag.reason + reason: flag.reason, + variant: flag.variant ?? undefined, + experiment: ExperimentMetadata.fromAPIMetadata(flag.metadata) }); } return new Flags({ diff --git a/tests/sdk/data/environment-with-experiment.json b/tests/sdk/data/environment-with-experiment.json new file mode 100644 index 0000000..580260c --- /dev/null +++ b/tests/sdk/data/environment-with-experiment.json @@ -0,0 +1,97 @@ +{ + "api_key": "B62qaMZNwfiqT76p38ggrQ", + "name": "Test Environment", + "project": { + "name": "Test project", + "organisation": { + "feature_analytics": false, + "name": "Test Org", + "id": 1, + "persist_trait_data": true, + "stop_serving_flags": false + }, + "id": 1, + "hide_disabled_flags": false, + "segments": [ + { + "id": 7, + "name": "Checkout experiment rollout", + "feature_states": [ + { + "multivariate_feature_state_values": [ + { + "id": 3, + "multivariate_feature_option": { + "id": 3, + "value": "new checkout", + "key": "treatment" + }, + "percentage_allocation": 100, + "mv_fs_value_uuid": "3e3e3e3e-3e3e-3e3e-3e3e-3e3e3e3e3e3e" + } + ], + "feature_state_value": "control", + "django_id": 2, + "featurestate_uuid": "c2c2c2c2-c2c2-c2c2-c2c2-c2c2c2c2c2c2", + "feature": { "name": "checkout", "type": "MULTIVARIATE", "id": 10 }, + "feature_segment": { "priority": 0 }, + "enabled": true, + "metadata": { + "experiment": { + "id": 42, + "name": "Checkout experiment", + "in_experiment": true + } + } + } + ], + "rules": [ + { + "type": "ALL", + "conditions": [{ "operator": "EQUAL", "property_": "plan", "value": "premium" }], + "rules": [] + } + ] + } + ] + }, + "segment_overrides": [], + "id": 1, + "feature_states": [ + { + "multivariate_feature_state_values": [], + "feature_state_value": "control", + "django_id": 1, + "featurestate_uuid": "c1c1c1c1-c1c1-c1c1-c1c1-c1c1c1c1c1c1", + "feature": { "name": "checkout", "type": "MULTIVARIATE", "id": 10 }, + "feature_segment": null, + "enabled": true, + "metadata": { + "experiment": { "id": 42, "name": "Checkout experiment", "in_experiment": false } + } + }, + { + "multivariate_feature_state_values": [], + "feature_state_value": "hello", + "django_id": 3, + "featurestate_uuid": "c3c3c3c3-c3c3-c3c3-c3c3-c3c3c3c3c3c3", + "feature": { "name": "plain", "type": "STANDARD", "id": 11 }, + "feature_segment": null, + "enabled": true + } + ], + "identity_overrides": [ + { + "identifier": "user_overridden", + "identity_features": [ + { + "multivariate_feature_state_values": [], + "feature_state_value": "overridden checkout", + "featurestate_uuid": "c4c4c4c4-c4c4-c4c4-c4c4-c4c4c4c4c4c4", + "feature": { "name": "checkout", "type": "MULTIVARIATE", "id": 10 }, + "enabled": true + } + ] + } + ] +} diff --git a/tests/sdk/flagsmith-experiments.test.ts b/tests/sdk/flagsmith-experiments.test.ts index 5af75f4..d7ec2e1 100644 --- a/tests/sdk/flagsmith-experiments.test.ts +++ b/tests/sdk/flagsmith-experiments.test.ts @@ -4,8 +4,14 @@ import { DefaultFlag, Flag } from '../../sdk/models.js'; import { FlagsmithConfig } from '../../sdk/types.js'; import { getUserAgent } from '../../sdk/utils.js'; import { SDK_VERSION } from '../../sdk/version.js'; -import { fetch } from './fetchMock.js'; +import { fetch, fetchImpl } from './fetchMock.js'; import { flagsmith, postedEvents, TestCache } from './utils.js'; +import { readFileSync } from 'fs'; + +const environmentWithExperimentJSON = readFileSync( + __dirname + '/data/environment-with-experiment.json', + 'utf-8' +); vi.mock('../../sdk/polling_manager'); @@ -201,19 +207,80 @@ test('getExperimentFlag records no exposure for a feature served by the default expect(postedEvents()).toHaveLength(0); }); -test('getExperimentFlag records no exposure when evaluating locally', async () => { - const flg = experimentsFlagsmith({ - environmentKey: 'ser.key', - enableLocalEvaluation: true - }); +/** + * A locally evaluating client whose environment document runs an experiment on `checkout`. Its + * rollout segment is identities on the premium plan, who are served the `treatment` variant. + * `user_overridden` has an identity override for `checkout`, and `plain` has no experiment. + */ +function localExperimentsFlagsmith() { + fetch.mockImplementation((url, options) => + String(url).includes('/environment-document') + ? Promise.resolve(new Response(environmentWithExperimentJSON, { status: 200 })) + : fetchImpl(url, options) + ); + return experimentsFlagsmith({ environmentKey: 'ser.key', enableLocalEvaluation: true }); +} - const flag = (await flg.getExperimentFlag('some_feature', 'user-123')) as Flag; +test('getExperimentFlag records an exposure for an enrolled identity when evaluating locally', async () => { + const flg = localExperimentsFlagsmith(); + + const flag = (await flg.getExperimentFlag('checkout', 'user-123', { plan: 'premium' })) as Flag; await flg.flushEvents(); - expect(flag.enabled).toBe(true); - expect(flag.variant).toBeUndefined(); + expect(flag.value).toBe('new checkout'); + expect(flag.variant).toBe('treatment'); + expect(flag.experiment).toEqual({ id: 42, name: 'Checkout experiment', inExperiment: true }); + expect(postedEvents()).toEqual([ + expect.objectContaining({ + event: FLAG_EXPOSURE_EVENT, + feature_name: 'checkout', + identifier: 'user-123', + value: 'treatment', + traits: { plan: 'premium' }, + metadata: { experiment_id: 42, sdk_version: SDK_VERSION } + }) + ]); +}); + +test.each([ + [ + 'the identity is outside the rollout', + 'checkout', + 'user-123', + 'free', + 'control', + { id: 42, name: 'Checkout experiment', inExperiment: false } + ], + [ + 'the identity has an identity override', + 'checkout', + 'user_overridden', + 'premium', + 'overridden checkout', + undefined + ], + ['the feature has no experiment', 'plain', 'user-123', 'premium', 'hello', undefined] +])( + 'getExperimentFlag records no exposure when evaluating locally and %s', + async (_name, featureName, identifier, plan, expectedValue, expectedExperiment) => { + const flg = localExperimentsFlagsmith(); + + const flag = (await flg.getExperimentFlag(featureName, identifier, { plan })) as Flag; + await flg.flushEvents(); + + expect(flag.value).toBe(expectedValue); + expect(flag.experiment).toEqual(expectedExperiment); + expect(postedEvents()).toHaveLength(0); + } +); + +test('environment flags carry no experiment when evaluating locally', async () => { + const flg = localExperimentsFlagsmith(); + + const flag = (await flg.getEnvironmentFlags()).getFlag('checkout') as Flag; + + expect(flag.value).toBe('control'); expect(flag.experiment).toBeUndefined(); - expect(postedEvents()).toHaveLength(0); }); test('getExperimentFlag records one exposure per identity', async () => { From 7249580cbe5069cb70053eed7f534ee864fa5001 Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Fri, 6 Nov 2026 10:12:07 +0530 Subject: [PATCH 2/3] refactor: leave the experiment out when mapping for environment evaluation --- .../evaluation/evaluationContext/mappers.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/flagsmith-engine/evaluation/evaluationContext/mappers.ts b/flagsmith-engine/evaluation/evaluationContext/mappers.ts index 2a0a0c2..3ef8b1d 100644 --- a/flagsmith-engine/evaluation/evaluationContext/mappers.ts +++ b/flagsmith-engine/evaluation/evaluationContext/mappers.ts @@ -82,7 +82,7 @@ function mapEnvironmentModelToEvaluationContext( value: fs.getValue(), variants: mapFeatureStateVariants(fs), priority: fs.featureSegment?.priority, - metadata: mapFeatureStateMetadata(fs) + metadata: mapFeatureStateMetadata(fs, isEnvironmentEvaluation) }; } @@ -102,7 +102,7 @@ function mapEnvironmentModelToEvaluationContext( value: fs.getValue(), variants: mapFeatureStateVariants(fs), priority: fs.featureSegment?.priority, - metadata: mapFeatureStateMetadata(fs) + metadata: mapFeatureStateMetadata(fs, isEnvironmentEvaluation) })) : [], metadata: { @@ -150,10 +150,16 @@ function mapIdentityModelToIdentityContext( return identityContext; } -function mapFeatureStateMetadata(fs: FeatureStateModel): SDKFeatureMetadata { +/** + * Only an identity is enrolled in an experiment, so environment evaluation leaves it out. + */ +function mapFeatureStateMetadata( + fs: FeatureStateModel, + isEnvironmentEvaluation: boolean +): SDKFeatureMetadata { return { id: fs.feature.id, - ...(fs.experiment && { experiment: fs.experiment }) + ...(!isEnvironmentEvaluation && fs.experiment && { experiment: fs.experiment }) }; } From 4a08a961cfbea1ee676ecbbf7c18eae18532621d Mon Sep 17 00:00:00 2001 From: Gagan Trivedi Date: Fri, 6 Nov 2026 10:12:07 +0530 Subject: [PATCH 3/3] fix: record no exposure for a flag without a variant --- sdk/index.ts | 10 ++++++---- tests/sdk/flagsmith-experiments.test.ts | 20 +++++++++++++++++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/sdk/index.ts b/sdk/index.ts index 7836d3e..870fc65 100644 --- a/sdk/index.ts +++ b/sdk/index.ts @@ -327,6 +327,12 @@ export class Flagsmith { ); return flag; } + if (flag.variant === undefined) { + this.logger.debug( + `Not recording an exposure for "${featureName}": the flag has no variant.` + ); + return flag; + } this.trackExposureEvent(featureName, { identifier: identifier, @@ -635,10 +641,6 @@ export class Flagsmith { } const evaluationResult = getEvaluationResult(context as EvaluationContextWithMetadata); const flags = Flags.fromEvaluationResult(evaluationResult); - // Only an identity is enrolled in an experiment, so environment flags carry none. - for (const flag of Object.values(flags.flags)) { - flag.experiment = undefined; - } if (!!this.cache) { await this.cache.set('flags', flags); diff --git a/tests/sdk/flagsmith-experiments.test.ts b/tests/sdk/flagsmith-experiments.test.ts index d7ec2e1..c55157a 100644 --- a/tests/sdk/flagsmith-experiments.test.ts +++ b/tests/sdk/flagsmith-experiments.test.ts @@ -4,7 +4,7 @@ import { DefaultFlag, Flag } from '../../sdk/models.js'; import { FlagsmithConfig } from '../../sdk/types.js'; import { getUserAgent } from '../../sdk/utils.js'; import { SDK_VERSION } from '../../sdk/version.js'; -import { fetch, fetchImpl } from './fetchMock.js'; +import { fetch, fetchImpl, identitiesJSON } from './fetchMock.js'; import { flagsmith, postedEvents, TestCache } from './utils.js'; import { readFileSync } from 'fs'; @@ -185,6 +185,24 @@ test.each([ expect(postedEvents()).toHaveLength(0); }); +test('getExperimentFlag records no exposure when the flag has no variant', async () => { + const identities = JSON.parse(identitiesJSON); + delete identities.flags.find((f: any) => f.feature.name === 'some_feature').variant; + fetch.mockImplementation((url, options) => + String(url).includes('/identities') + ? Promise.resolve(new Response(JSON.stringify(identities), { status: 200 })) + : fetchImpl(url, options) + ); + const flg = experimentsFlagsmith(); + + const flag = (await flg.getExperimentFlag('some_feature', 'user-123')) as Flag; + await flg.flushEvents(); + + expect(flag.experiment?.inExperiment).toBe(true); + expect(flag.variant).toBeUndefined(); + expect(postedEvents()).toHaveLength(0); +}); + test('getExperimentFlag still returns the flag when the identity is outside the rollout', async () => { const flg = experimentsFlagsmith();