feat: Record exposures under local evaluation (#318)

Since Flagsmith/flagsmith#8532, the environment document carries
`metadata.experiment` on feature states. The SDK dropped it when
building engine models, so under local evaluation `flag.experiment` was
always undefined and `getExperimentFlag` never recorded an exposure.

### Changes
- `FeatureStateModel` keeps `metadata.experiment`; the mapper adds it to
the engine context's feature metadata for environment and segment
feature states. Identity overrides carry none, as on the API.
- `Flags.fromEvaluationResult` sets `experiment`, and also `variant`,
which local evaluation dropped until now, so local exposures carry the
variant key.
- Locally evaluated environment flags carry no experiment, matching the
API, which only sends it for identity flags.

### Tests
`getExperimentFlag` under local evaluation: an enrolled identity records
one exposure with its variant and experiment ID; an identity outside the
rollout, one with an identity override, and a feature with no experiment
record none. Environment flags carry no experiment.

Also checked against a local Flagsmith with the events pipeline: local
and remote evaluation give the same value, variant and experiment, and
one exposure reached the warehouse.

### Not included
Bumping `engine-test-data` to v3.13.0, which adds the experiment cases:
it also adds two numeric trait cases (Flagsmith/engine-test-data#63) the
engine already fails, to be fixed separately.

Author: gagantrivedi

flagsmith-engine/evaluation/evaluationContext/mappers.ts

@@ -82,9 +82,7 @@ function mapEnvironmentModelToEvaluationContext(
             value: fs.getValue(),
             variants: mapFeatureStateVariants(fs),
             priority: fs.featureSegment?.priority,
-            metadata: {
-                id: fs.feature.id
-            }
+            metadata: mapFeatureStateMetadata(fs, isEnvironmentEvaluation)
         };
     }
 
@@ -104,9 +102,7 @@ function mapEnvironmentModelToEvaluationContext(
                           value: fs.getValue(),
                           variants: mapFeatureStateVariants(fs),
                           priority: fs.featureSegment?.priority,
-                          metadata: {
-                              id: fs.feature.id
-                          }
+                          metadata: mapFeatureStateMetadata(fs, isEnvironmentEvaluation)
                       }))
                     : [],
             metadata: {
@@ -154,6 +150,19 @@ function mapIdentityModelToIdentityContext(
     return identityContext;
 }
 
+/**
+ * Only an identity is enrolled in an experiment, so environment evaluation leaves it out.
+ */
+function mapFeatureStateMetadata(
+    fs: FeatureStateModel,
+    isEnvironmentEvaluation: boolean
+): SDKFeatureMetadata {
+    return {
+        id: fs.feature.id,
+        ...(!isEnvironmentEvaluation && fs.experiment && { experiment: fs.experiment })
+    };
+}
+
 function mapFeatureStateVariants(fs: FeatureStateModel) {
     return fs.multivariateFeatureStateValues?.length > 0
         ? fs.multivariateFeatureStateValues.map(mv => ({

flagsmith-engine/evaluation/models.ts

@@ -9,6 +9,8 @@ import type {
     SegmentMetadata
 } from './evaluationResult/evaluationResult.types.js';
 
+import type { FeatureStateExperiment } from '../features/models.js';
+
 import type {
     FeatureContext,
     EnvironmentContext,
@@ -26,6 +28,8 @@ export enum SegmentSource {
 // Feature types
 export interface SDKFeatureMetadata extends FeatureMetadata {
     id: number;
+    /** The running experiment the feature state belongs to, if any. */
+    experiment?: FeatureStateExperiment;
 }
 
 export interface FeatureContextWithMetadata<T extends FeatureMetadata = FeatureMetadata>

flagsmith-engine/features/models.ts

@@ -48,12 +48,22 @@ export class MultivariateFeatureStateValueModel {
     }
 }
 
+/**
+ * The running experiment a feature state belongs to, as the environment document reports it.
+ */
+export interface FeatureStateExperiment {
+    id: number;
+    name: string;
+    in_experiment: boolean;
+}
+
 export class FeatureStateModel {
     feature: FeatureModel;
     enabled: boolean;
     djangoID: number;
     featurestateUUID: string = uuidv4();
     featureSegment?: FeatureSegment;
+    experiment?: FeatureStateExperiment;
     private value: any;
     multivariateFeatureStateValues: MultivariateFeatureStateValueModel[] = [];
 

flagsmith-engine/features/util.ts

@@ -23,6 +23,8 @@ export function buildFeatureStateModel(featuresStateModelJSON: any): FeatureStat
         ? buildFeatureSegment(featuresStateModelJSON.feature_segment)
         : undefined;
 
+    featureStateModel.experiment = featuresStateModelJSON.metadata?.experiment ?? undefined;
+
     const multivariateFeatureStateValues = featuresStateModelJSON.multivariate_feature_state_values
         ? featuresStateModelJSON.multivariate_feature_state_values.map((fsv: any) => {
               const featureOption = new MultivariateFeatureOptionModel(

sdk/index.ts

@@ -289,9 +289,6 @@ export class Flagsmith {
      * Flags are fetched exactly as {@link getIdentityFlags} fetches them, so traits are upserted and
      * the identity cache is used when one is configured.
      *
-     * Experiment metadata is only produced by remote identity evaluation. Local evaluation and
-     * offline mode never carry it, so no exposure is ever recorded for them.
-     *
      * @param featureName the name of the feature to evaluate.
      * @param identifier a unique identifier for the identity in the current environment.
      * @param traits? a dictionary of traits to add / update on the identity in Flagsmith.
@@ -330,6 +327,12 @@ export class Flagsmith {
             );
             return flag;
         }
+        if (flag.variant === undefined) {
+            this.logger.debug(
+                `Not recording an exposure for "${featureName}": the flag has no variant.`
+            );
+            return flag;
+        }
 
         this.trackExposureEvent(featureName, {
             identifier: identifier,

sdk/models.ts

@@ -43,7 +43,7 @@ export class DefaultFlag extends BaseFlag {
 }
 
 /**
- * A running experiment on a feature, as reported by a remote identity evaluation.
+ * A running experiment on a feature, as reported by an identity evaluation.
  */
 export class ExperimentMetadata {
     /**
@@ -110,11 +110,11 @@ export class Flag extends BaseFlag {
      */
     reason?: string;
     /**
-     * The variant key this identity was bucketed into. Set by remote evaluation only.
+     * The variant key this identity was bucketed into. Set by identity evaluation only.
      */
     variant?: string;
     /**
-     * The experiment running on this feature, if any. Set by remote identity evaluation only.
+     * The experiment running on this feature, if any. Set by identity evaluation only.
      */
     experiment?: ExperimentMetadata;
 
@@ -196,7 +196,9 @@ export class Flags {
                 value: flag.value ?? null,
                 featureId: flagMetadataId,
                 featureName: flag.name,
-                reason: flag.reason
+                reason: flag.reason,
+                variant: flag.variant ?? undefined,
+                experiment: ExperimentMetadata.fromAPIMetadata(flag.metadata)
             });
         }
         return new Flags({

tests/sdk/data/environment-with-experiment.json

@@ -0,0 +1,97 @@
+{
+    "api_key": "B62qaMZNwfiqT76p38ggrQ",
+    "name": "Test Environment",
+    "project": {
+        "name": "Test project",
+        "organisation": {
+            "feature_analytics": false,
+            "name": "Test Org",
+            "id": 1,
+            "persist_trait_data": true,
+            "stop_serving_flags": false
+        },
+        "id": 1,
+        "hide_disabled_flags": false,
+        "segments": [
+            {
+                "id": 7,
+                "name": "Checkout experiment rollout",
+                "feature_states": [
+                    {
+                        "multivariate_feature_state_values": [
+                            {
+                                "id": 3,
+                                "multivariate_feature_option": {
+                                    "id": 3,
+                                    "value": "new checkout",
+                                    "key": "treatment"
+                                },
+                                "percentage_allocation": 100,
+                                "mv_fs_value_uuid": "3e3e3e3e-3e3e-3e3e-3e3e-3e3e3e3e3e3e"
+                            }
+                        ],
+                        "feature_state_value": "control",
+                        "django_id": 2,
+                        "featurestate_uuid": "c2c2c2c2-c2c2-c2c2-c2c2-c2c2c2c2c2c2",
+                        "feature": { "name": "checkout", "type": "MULTIVARIATE", "id": 10 },
+                        "feature_segment": { "priority": 0 },
+                        "enabled": true,
+                        "metadata": {
+                            "experiment": {
+                                "id": 42,
+                                "name": "Checkout experiment",
+                                "in_experiment": true
+                            }
+                        }
+                    }
+                ],
+                "rules": [
+                    {
+                        "type": "ALL",
+                        "conditions": [{ "operator": "EQUAL", "property_": "plan", "value": "premium" }],
+                        "rules": []
+                    }
+                ]
+            }
+        ]
+    },
+    "segment_overrides": [],
+    "id": 1,
+    "feature_states": [
+        {
+            "multivariate_feature_state_values": [],
+            "feature_state_value": "control",
+            "django_id": 1,
+            "featurestate_uuid": "c1c1c1c1-c1c1-c1c1-c1c1-c1c1c1c1c1c1",
+            "feature": { "name": "checkout", "type": "MULTIVARIATE", "id": 10 },
+            "feature_segment": null,
+            "enabled": true,
+            "metadata": {
+                "experiment": { "id": 42, "name": "Checkout experiment", "in_experiment": false }
+            }
+        },
+        {
+            "multivariate_feature_state_values": [],
+            "feature_state_value": "hello",
+            "django_id": 3,
+            "featurestate_uuid": "c3c3c3c3-c3c3-c3c3-c3c3-c3c3c3c3c3c3",
+            "feature": { "name": "plain", "type": "STANDARD", "id": 11 },
+            "feature_segment": null,
+            "enabled": true
+        }
+    ],
+    "identity_overrides": [
+        {
+            "identifier": "user_overridden",
+            "identity_features": [
+                {
+                    "multivariate_feature_state_values": [],
+                    "feature_state_value": "overridden checkout",
+                    "featurestate_uuid": "c4c4c4c4-c4c4-c4c4-c4c4-c4c4c4c4c4c4",
+                    "feature": { "name": "checkout", "type": "MULTIVARIATE", "id": 10 },
+                    "enabled": true
+                }
+            ]
+        }
+    ]
+}

tests/sdk/flagsmith-experiments.test.ts

@@ -4,8 +4,14 @@ import { DefaultFlag, Flag } from '../../sdk/models.js';
 import { FlagsmithConfig } from '../../sdk/types.js';
 import { getUserAgent } from '../../sdk/utils.js';
 import { SDK_VERSION } from '../../sdk/version.js';
-import { fetch } from './fetchMock.js';
+import { fetch, fetchImpl, identitiesJSON } from './fetchMock.js';
 import { flagsmith, postedEvents, TestCache } from './utils.js';
+import { readFileSync } from 'fs';
+
+const environmentWithExperimentJSON = readFileSync(
+    __dirname + '/data/environment-with-experiment.json',
+    'utf-8'
+);
 
 vi.mock('../../sdk/polling_manager');
 
@@ -179,6 +185,24 @@ test.each([
     expect(postedEvents()).toHaveLength(0);
 });
 
+test('getExperimentFlag records no exposure when the flag has no variant', async () => {
+    const identities = JSON.parse(identitiesJSON);
+    delete identities.flags.find((f: any) => f.feature.name === 'some_feature').variant;
+    fetch.mockImplementation((url, options) =>
+        String(url).includes('/identities')
+            ? Promise.resolve(new Response(JSON.stringify(identities), { status: 200 }))
+            : fetchImpl(url, options)
+    );
+    const flg = experimentsFlagsmith();
+
+    const flag = (await flg.getExperimentFlag('some_feature', 'user-123')) as Flag;
+    await flg.flushEvents();
+
+    expect(flag.experiment?.inExperiment).toBe(true);
+    expect(flag.variant).toBeUndefined();
+    expect(postedEvents()).toHaveLength(0);
+});
+
 test('getExperimentFlag still returns the flag when the identity is outside the rollout', async () => {
     const flg = experimentsFlagsmith();
 
@@ -201,19 +225,80 @@ test('getExperimentFlag records no exposure for a feature served by the default
     expect(postedEvents()).toHaveLength(0);
 });
 
-test('getExperimentFlag records no exposure when evaluating locally', async () => {
-    const flg = experimentsFlagsmith({
-        environmentKey: 'ser.key',
-        enableLocalEvaluation: true
-    });
+/**
+ * A locally evaluating client whose environment document runs an experiment on `checkout`. Its
+ * rollout segment is identities on the premium plan, who are served the `treatment` variant.
+ * `user_overridden` has an identity override for `checkout`, and `plain` has no experiment.
+ */
+function localExperimentsFlagsmith() {
+    fetch.mockImplementation((url, options) =>
+        String(url).includes('/environment-document')
+            ? Promise.resolve(new Response(environmentWithExperimentJSON, { status: 200 }))
+            : fetchImpl(url, options)
+    );
+    return experimentsFlagsmith({ environmentKey: 'ser.key', enableLocalEvaluation: true });
+}
 
-    const flag = (await flg.getExperimentFlag('some_feature', 'user-123')) as Flag;
+test('getExperimentFlag records an exposure for an enrolled identity when evaluating locally', async () => {
+    const flg = localExperimentsFlagsmith();
+
+    const flag = (await flg.getExperimentFlag('checkout', 'user-123', { plan: 'premium' })) as Flag;
     await flg.flushEvents();
 
-    expect(flag.enabled).toBe(true);
-    expect(flag.variant).toBeUndefined();
+    expect(flag.value).toBe('new checkout');
+    expect(flag.variant).toBe('treatment');
+    expect(flag.experiment).toEqual({ id: 42, name: 'Checkout experiment', inExperiment: true });
+    expect(postedEvents()).toEqual([
+        expect.objectContaining({
+            event: FLAG_EXPOSURE_EVENT,
+            feature_name: 'checkout',
+            identifier: 'user-123',
+            value: 'treatment',
+            traits: { plan: 'premium' },
+            metadata: { experiment_id: 42, sdk_version: SDK_VERSION }
+        })
+    ]);
+});
+
+test.each([
+    [
+        'the identity is outside the rollout',
+        'checkout',
+        'user-123',
+        'free',
+        'control',
+        { id: 42, name: 'Checkout experiment', inExperiment: false }
+    ],
+    [
+        'the identity has an identity override',
+        'checkout',
+        'user_overridden',
+        'premium',
+        'overridden checkout',
+        undefined
+    ],
+    ['the feature has no experiment', 'plain', 'user-123', 'premium', 'hello', undefined]
+])(
+    'getExperimentFlag records no exposure when evaluating locally and %s',
+    async (_name, featureName, identifier, plan, expectedValue, expectedExperiment) => {
+        const flg = localExperimentsFlagsmith();
+
+        const flag = (await flg.getExperimentFlag(featureName, identifier, { plan })) as Flag;
+        await flg.flushEvents();
+
+        expect(flag.value).toBe(expectedValue);
+        expect(flag.experiment).toEqual(expectedExperiment);
+        expect(postedEvents()).toHaveLength(0);
+    }
+);
+
+test('environment flags carry no experiment when evaluating locally', async () => {
+    const flg = localExperimentsFlagsmith();
+
+    const flag = (await flg.getEnvironmentFlags()).getFlag('checkout') as Flag;
+
+    expect(flag.value).toBe('control');
     expect(flag.experiment).toBeUndefined();
-    expect(postedEvents()).toHaveLength(0);
 });
 
 test('getExperimentFlag records one exposure per identity', async () => {