Merge pull request #4 from runloopai/wall/remove-benchmark-sdk

feat(sdk)!: remove hosted scenario and scorer wrappers

Author: tode-rl

README.md

@@ -10,7 +10,6 @@ Why teams pick Runloop:
 - **A real machine, not just a code runner.** Full filesystem, shell, background processes, and public URLs for ports.
 - **Fast repeat starts.** Blueprints and disk snapshots boot pre-built environments instead of reinstalling dependencies.
 - **Built for production.** SOC 2 compliant, with bring-your-own-cloud deployment on AWS, GCP, and Azure.
-- **Evaluation built in.** Scenarios, scorers, and benchmarks let you measure agent performance on the same platform you run it on.
 
 ## Installation
 
@@ -103,8 +102,6 @@ The SDK provides object-oriented interfaces for all major Runloop resources:
 - **[`runloop.storageObject`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.StorageObjectOps.html)** - Storage object management (upload, download, list objects)
 - **[`runloop.agent`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.AgentOps.html)** - Agent management (create, list agents from npm/pip/git)
 - **[`runloop.axon`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.AxonOps.html)** - [Beta] Axon management (create, publish events, subscribe via SSE)
-- **[`runloop.scenario`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.ScenarioOps.html)** - Scenario management (list scenarios, start runs)
-- **[`runloop.scorer`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.ScorerOps.html)** - Scorer management (create, list, update)
 - **[`runloop.networkPolicy`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.NetworkPolicyOps.html)** - Network policy management (create, list, update egress rules)
 - **[`runloop.gatewayConfig`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.GatewayConfigOps.html)** - Gateway config management (create, list API proxy configurations)
 - **[`runloop.mcpConfig`](https://runloopai.github.io/api-client-ts/stable/classes/sdk.McpConfigOps.html)** - MCP config management (create, list MCP server configurations)
@@ -152,54 +149,6 @@ console.log(logs.logs);
 const devbox = await blueprint.createDevbox({ name: 'my-devbox' });
 ```
 
-### Scorers
-
-Scorers are custom scoring functions used to evaluate scenario outputs. Create scorers via `runloop.scorer.create()`, then update them with the returned `Scorer` instance:
-
-```typescript
-import { RunloopSDK } from '@runloop/api-client';
-
-const runloop = new RunloopSDK();
-
-const scorer = await runloop.scorer.create({
-  type: 'my_scorer',
-  bash_script: 'echo "1.0"',
-});
-
-await scorer.update({ bash_script: 'echo "0.5"' });
-```
-
-### Scenarios
-
-Scenarios define tasks with a well defined starting environment, task evaluation scorer and an optional reference solution. Use `runloop.scenario.fromId()` to get a scenario, then `scenario.run()` to start a run with your agent mounted:
-
-```typescript
-const scenario = runloop.scenario.fromId('scn_123');
-const run = await scenario.run({
-  run_name: 'my-run',
-  runProfile: {
-    mounts: [
-      {
-        type: 'agent_mount',
-        agent_id: 'agt_123',
-        agent_path: '/home/user/agent',
-      },
-    ],
-  },
-});
-await run.devbox.cmd.exec('python /home/user/agent/main.py');
-await run.scoreAndComplete();
-```
-
-### Benchmarks
-
-Benchmarks are collections of scenarios for evaluating AI agents. Access via `runloop.api.benchmarks`:
-
-```typescript
-const benchmarks = await runloop.api.benchmarks.listPublic();
-const definitions = await runloop.api.benchmarks.definitions('benchmark_id');
-```
-
 ## Migration from API Client
 
 If you're currently using the legacy API, migration is straightforward:

llms.txt

@@ -24,7 +24,7 @@
 
 - Always use `RunloopSDK` as the entry point (not legacy `Runloop` default export)
 - Prefer SDK methods (`sdk.devbox`, `sdk.blueprint`, etc.) for object-oriented patterns and convenience
-- For resources without SDK coverage (e.g., benchmarks), use `sdk.api.*` as a fallback
+- For resources without SDK coverage, use `sdk.api.*` as a fallback
 - Use `sdk.devbox.create()` to create devboxes; they auto-await readiness
 - Use `devbox.suspend()` plus `devbox.awaitSuspended()` when you want to park a devbox and preserve its disk state, then use `devbox.resume()` to bring the same devbox back to `running`
 - Use `devbox.snapshotDisk()` to capture a reusable disk baseline and `snapshot.createDevbox()` or `sdk.devbox.createFromSnapshot(snapshot.id, ...)` to branch multiple devboxes from that same state

packages/mcp-server/src/code-tool-paths.cts

@@ -1,3 +1 @@
-// File generated from our OpenAPI spec by Stainless. See CONTRIBUTING.md for details.
-
 export const workerPath = require.resolve('./code-tool-worker.mjs');

src/resources/scenarios/runs.ts

@@ -6,8 +6,6 @@ import * as Core from '../../core';
 import * as ScenariosAPI from './scenarios';
 import { ScenarioRunViewsBenchmarkRunsCursorIDPage } from './scenarios';
 import { type BenchmarkRunsCursorIDPageParams } from '../../pagination';
-import { LongPollRequestOptions, poll, resolveLongPollTimeoutMs } from '@runloop/api-client/lib/polling';
-import { RunloopError } from '../..';
 import { type Response } from '../../_shims/index';
 
 export class Runs extends APIResource {
@@ -75,65 +73,6 @@ export class Runs extends APIResource {
   score(id: string, options?: Core.RequestOptions): Core.APIPromise<ScenariosAPI.ScenarioRunView> {
     return this._client.post(`/v1/scenarios/runs/${id}/score`, options);
   }
-
-  /**
-   * Wait for a scenario run to finish scoring.
-   * Polls the scenario run status until it reaches a terminal state.
-   */
-  async awaitScored(
-    id: string,
-    options?: LongPollRequestOptions<ScenariosAPI.ScenarioRunView>,
-  ): Promise<ScenariosAPI.ScenarioRunView> {
-    const pollTimeoutMs = resolveLongPollTimeoutMs(options);
-    const { longPoll: _lp, polling, signal, ...requestOptions } = options ?? {};
-    const finalResult = await poll(
-      () => this.retrieve(id, requestOptions),
-      () => this.retrieve(id, requestOptions),
-      {
-        ...polling,
-        signal,
-        ...(pollTimeoutMs !== undefined ? { timeoutMs: pollTimeoutMs } : {}),
-        shouldStop: (result) => {
-          return result.state !== 'scoring';
-        },
-      },
-    );
-
-    // Check if the run was scored successfully, otherwise throw an error
-    if (finalResult.state !== 'scored') {
-      throw new RunloopError(
-        `Scenario run ${id} failed to transition to scored state. Final state: ${finalResult.state}`,
-      );
-    }
-
-    return finalResult;
-  }
-
-  /**
-   * Score a scenario run and wait for it to finish scoring.
-   * This is a convenience method that combines score() and awaitScoring().
-   */
-  async scoreAndAwait(
-    id: string,
-    options?: LongPollRequestOptions<ScenariosAPI.ScenarioRunView>,
-  ): Promise<ScenariosAPI.ScenarioRunView> {
-    const { longPoll, polling, signal, ...requestOptions } = options ?? {};
-    const run = await this.score(id, requestOptions);
-    return this.awaitScored(run.id, { ...requestOptions, longPoll, polling, signal });
-  }
-
-  /**
-   * Score a scenario run, wait for scoring to complete, and then complete the run.
-   * This is a convenience method that combines scoreAndAwait() and complete().
-   */
-  async scoreAndComplete(
-    id: string,
-    options?: LongPollRequestOptions<ScenariosAPI.ScenarioRunView>,
-  ): Promise<ScenariosAPI.ScenarioRunView> {
-    const { longPoll, polling, signal, ...requestOptions } = options ?? {};
-    const scoredRun = await this.scoreAndAwait(id, { ...requestOptions, longPoll, polling, signal });
-    return this.complete(scoredRun.id, requestOptions);
-  }
 }
 
 export interface RunListParams extends BenchmarkRunsCursorIDPageParams {

src/resources/scenarios/scenarios.ts

@@ -23,8 +23,6 @@ import {
   ScenariosCursorIDPage,
   type ScenariosCursorIDPageParams,
 } from '../../pagination';
-import { LongPollRequestOptions } from '@runloop/api-client/lib/polling';
-import { DevboxView } from '../devboxes';
 
 export class Scenarios extends APIResource {
   runs: RunsAPI.Runs = new RunsAPI.Runs(this._client);
@@ -125,19 +123,6 @@ export class Scenarios extends APIResource {
   startRun(body: ScenarioStartRunParams, options?: Core.RequestOptions): Core.APIPromise<ScenarioRunView> {
     return this._client.post('/v1/scenarios/start_run', { body, ...options });
   }
-
-  /**
-   * Start a new ScenarioRun and wait for its environment to be ready.
-   * This is a convenience method that combines startRun() and awaitReady() on the devbox.
-   */
-  async startRunAndAwaitEnvReady(
-    body: ScenarioStartRunParams,
-    options?: LongPollRequestOptions<DevboxView>,
-  ): Promise<ScenarioRunView> {
-    const run = await this.startRun(body, options);
-    await this._client.devboxes.awaitRunning(run.devbox_id, options);
-    return run;
-  }
 }
 
 export class ScenarioViewsScenariosCursorIDPage extends ScenariosCursorIDPage<ScenarioView> {}

src/resources/shared.ts

@@ -522,11 +522,6 @@ export interface ResumeTriggers {
   http?: boolean | null;
 }
 
-/**
- * Configuration profile for scenario/benchmark runs.
- *
- * @category Shared Types
- */
 export interface RunProfile {
   /**
    * Mapping of Environment Variable to Value. May be shown in devbox logging.