Skip to content

Commit a0d835d

Browse files
emyllerclaude
emyller
and
claude
authored
feat(CI): Set up CodSpeed (#151)
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent 351aeae commit a0d835d

8 files changed

Lines changed: 230 additions & 0 deletions

File tree

‎.github/workflows/codspeed.yml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
name: CodSpeed
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
pull_request:
8+
workflow_dispatch:
9+
10+
permissions:
11+
contents: read # Check out the repository and its submodules
12+
id-token: write # Authenticate result uploads to CodSpeed via OIDC
13+
14+
jobs:
15+
benchmarks:
16+
name: Run benchmarks
17+
runs-on: codspeed-macro
18+
19+
steps:
20+
- name: Checkout repository
21+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
22+
with:
23+
submodules: recursive
24+
persist-credentials: false
25+
26+
- name: Set up PHP
27+
uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0
28+
with:
29+
php-version: "8.5"
30+
extensions: "bcmath, gmp"
31+
32+
- name: Install CodSpeed CLI
33+
env:
34+
CODSPEED_VERSION: 5.6.0
35+
CODSPEED_INSTALLER_SHA256: bf1bf63661e203bcabc0c84fdef0337f8658fc8340c467a6a3bca81040d99f40
36+
run: |
37+
installer=$(mktemp)
38+
curl -fsSL -o "$installer" "https://codspeed.io/v$CODSPEED_VERSION/install.sh"
39+
echo "$CODSPEED_INSTALLER_SHA256 $installer" | sha256sum --check --strict
40+
bash "$installer" --quiet
41+
42+
- name: Run benchmarks
43+
run: make benchmark

‎.gitmodules‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@
22
path = tests/Engine/EngineTests/EngineTestData
33
url = git@github.com:Flagsmith/engine-test-data.git
44
branch = v3.7.0
5+
[submodule "tests/Engine/EngineTests/SdkBenchmarks"]
6+
path = tests/Engine/EngineTests/SdkBenchmarks
7+
url = https://github.com/Flagsmith/sdk-benchmarks.git
8+
branch = 0.0.2

‎Makefile‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
SDK_BENCHMARKS := tests/Engine/EngineTests/SdkBenchmarks
2+
3+
$(SDK_BENCHMARKS)/environments:
4+
git submodule update --init -- $(SDK_BENCHMARKS)
5+
6+
vendor/autoload.php: composer.json
7+
composer install --prefer-dist --no-interaction
8+
touch $@
9+
10+
.PHONY: benchmark
11+
benchmark: vendor/autoload.php $(SDK_BENCHMARKS)/environments
12+
SDK_BENCHMARKS_ENVIRONMENTS_DIR=$(abspath $(SDK_BENCHMARKS)/environments) codspeed run --mode walltime

‎benchmarks/SdkBenchmark.php‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<?php
2+
3+
namespace FlagsmithBenchmark;
4+
5+
use Flagsmith\Flagsmith;
6+
use Flagsmith\Models\Segment;
7+
use GuzzleHttp\Client;
8+
use GuzzleHttp\Promise\Create;
9+
use GuzzleHttp\Promise\PromiseInterface;
10+
use GuzzleHttp\Psr7\Response;
11+
use Psr\Http\Message\RequestInterface;
12+
13+
/**
14+
* Benchmarks a locally evaluating SDK client against environment documents tiered by size.
15+
*/
16+
class SdkBenchmark
17+
{
18+
public const ENVIRONMENTS = ['small', 'medium', 'large'];
19+
public const BENCHMARKS = [
20+
'getEnvironmentFlags',
21+
'getIdentityFlagsWithIdentityOverrides',
22+
'getIdentityFlagsNoTraits',
23+
'getIdentityFlagsWithTraits',
24+
'getIdentitySegmentsWithTraits',
25+
];
26+
27+
private const IDENTITY_OVERRIDES_IDENTIFIER = 'identity-override-1';
28+
private const IDENTIFIER = 'user-1';
29+
private const TRAITS = [
30+
'plan' => 'plan-7',
31+
'age' => 30,
32+
'country' => 'GB',
33+
'version' => '2.5.1',
34+
'email' => 'jane@example.com',
35+
'user_id' => 12,
36+
'beta' => true,
37+
'score' => 95,
38+
'tier' => 'pro',
39+
'device' => 'iPhone 15',
40+
];
41+
42+
private Flagsmith $client;
43+
44+
/**
45+
* Build a locally evaluating client serving the environment document from memory.
46+
*/
47+
public function __construct(string $environmentsDir, string $environment)
48+
{
49+
$environmentDocument = file_get_contents("{$environmentsDir}/{$environment}.json");
50+
51+
$this->client = (new Flagsmith(
52+
apiKey: 'ser.benchmark',
53+
host: 'http://flagsmith.benchmark/api/v1/',
54+
environmentTtl: 86400, // Refreshed by benchmarks only
55+
))->withClient(new Client([
56+
'handler' => static fn (RequestInterface $request): PromiseInterface => Create::promiseFor(
57+
new Response(200, ['Content-Type' => 'application/json'], $environmentDocument),
58+
),
59+
]));
60+
$this->client->updateEnvironment();
61+
}
62+
63+
/**
64+
* Get a feature value from the environment flags.
65+
*/
66+
public function getEnvironmentFlags(): mixed
67+
{
68+
return $this->client->getEnvironmentFlags()->getFeatureValue('feature_1');
69+
}
70+
71+
/**
72+
* Check a feature for an identity with identity overrides.
73+
*/
74+
public function getIdentityFlagsWithIdentityOverrides(): bool
75+
{
76+
return $this->client->getIdentityFlags(self::IDENTITY_OVERRIDES_IDENTIFIER)->isFeatureEnabled('feature_8');
77+
}
78+
79+
/**
80+
* Get a feature value for an identity without traits.
81+
*/
82+
public function getIdentityFlagsNoTraits(): mixed
83+
{
84+
return $this->client->getIdentityFlags(self::IDENTIFIER)->getFeatureValue('feature_1');
85+
}
86+
87+
/**
88+
* Get a feature value for an identity with traits.
89+
*/
90+
public function getIdentityFlagsWithTraits(): mixed
91+
{
92+
return $this->client->getIdentityFlags(self::IDENTIFIER, (object) self::TRAITS)->getFeatureValue('feature_1');
93+
}
94+
95+
/**
96+
* List the segments of an identity with traits.
97+
* @return array<Segment>
98+
*/
99+
public function getIdentitySegmentsWithTraits(): array
100+
{
101+
return $this->client->getIdentitySegments(self::IDENTIFIER, (object) self::TRAITS);
102+
}
103+
}

‎benchmarks/run.php‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
<?php
2+
3+
use FlagsmithBenchmark\SdkBenchmark;
4+
5+
require __DIR__ . '/../vendor/autoload.php';
6+
7+
const REVOLUTIONS = 1000;
8+
9+
[, $benchmark, $environment] = $argv + [null, null, null];
10+
$environmentsDir = getenv('SDK_BENCHMARKS_ENVIRONMENTS_DIR');
11+
12+
if (
13+
$environmentsDir === false
14+
|| !in_array($benchmark, SdkBenchmark::BENCHMARKS, true)
15+
|| !in_array($environment, SdkBenchmark::ENVIRONMENTS, true)
16+
) {
17+
fwrite(STDERR, sprintf(
18+
"Usage: SDK_BENCHMARKS_ENVIRONMENTS_DIR=<dir> php %s <%s> <%s>\n",
19+
$argv[0],
20+
implode('|', SdkBenchmark::BENCHMARKS),
21+
implode('|', SdkBenchmark::ENVIRONMENTS),
22+
));
23+
exit(2);
24+
}
25+
26+
$sdkBenchmark = new SdkBenchmark($environmentsDir, $environment);
27+
for ($revolution = 0; $revolution < REVOLUTIONS; $revolution++) {
28+
$sdkBenchmark->$benchmark();
29+
}

‎codspeed.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
$schema: https://raw.githubusercontent.com/CodSpeedHQ/codspeed/refs/heads/main/schemas/codspeed.schema.json
2+
3+
benchmarks:
4+
- name: getEnvironmentFlags[small]
5+
exec: php benchmarks/run.php getEnvironmentFlags small
6+
- name: getEnvironmentFlags[medium]
7+
exec: php benchmarks/run.php getEnvironmentFlags medium
8+
- name: getEnvironmentFlags[large]
9+
exec: php benchmarks/run.php getEnvironmentFlags large
10+
- name: getIdentityFlagsWithIdentityOverrides[small]
11+
exec: php benchmarks/run.php getIdentityFlagsWithIdentityOverrides small
12+
- name: getIdentityFlagsWithIdentityOverrides[medium]
13+
exec: php benchmarks/run.php getIdentityFlagsWithIdentityOverrides medium
14+
- name: getIdentityFlagsWithIdentityOverrides[large]
15+
exec: php benchmarks/run.php getIdentityFlagsWithIdentityOverrides large
16+
- name: getIdentityFlagsNoTraits[small]
17+
exec: php benchmarks/run.php getIdentityFlagsNoTraits small
18+
- name: getIdentityFlagsNoTraits[medium]
19+
exec: php benchmarks/run.php getIdentityFlagsNoTraits medium
20+
- name: getIdentityFlagsNoTraits[large]
21+
exec: php benchmarks/run.php getIdentityFlagsNoTraits large
22+
- name: getIdentityFlagsWithTraits[small]
23+
exec: php benchmarks/run.php getIdentityFlagsWithTraits small
24+
- name: getIdentityFlagsWithTraits[medium]
25+
exec: php benchmarks/run.php getIdentityFlagsWithTraits medium
26+
- name: getIdentityFlagsWithTraits[large]
27+
exec: php benchmarks/run.php getIdentityFlagsWithTraits large
28+
- name: getIdentitySegmentsWithTraits[small]
29+
exec: php benchmarks/run.php getIdentitySegmentsWithTraits small
30+
- name: getIdentitySegmentsWithTraits[medium]
31+
exec: php benchmarks/run.php getIdentitySegmentsWithTraits medium
32+
- name: getIdentitySegmentsWithTraits[large]
33+
exec: php benchmarks/run.php getIdentitySegmentsWithTraits large

‎composer.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@
3535
"FlagsmithTest\\": "tests"
3636
}
3737
},
38+
"autoload-dev": {
39+
"psr-4": {
40+
"FlagsmithBenchmark\\": "benchmarks"
41+
}
42+
},
3843
"config": {
3944
"allow-plugins": {
4045
"php-http/discovery": true

‎tests/Engine/EngineTests/SdkBenchmarks‎

Submodule SdkBenchmarks added at 7da2be4

0 commit comments

Comments
 (0)