Skip to content

Commit 45d369e

Browse files
committed
test: add mode-contract tests for tool separation verification
1 parent 6f573f7 commit 45d369e

1 file changed

Lines changed: 205 additions & 0 deletions

File tree

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/**
2+
* Contract tests for tool-mode separation.
3+
*
4+
* These tests verify the invariants that must hold across modes:
5+
* - Each mode produces the expected set of tools per category
6+
* - Mode-variant tools share identical inputSchema (same args accepted)
7+
* - Tool definitions are frozen (immutable)
8+
* - _meta stripping works for non-openai modes
9+
*/
10+
import { describe, expect, it } from 'vitest';
11+
12+
import { HelperTools } from '../../src/const.js';
13+
import { buildCategories, CATEGORY_NAMES } from '../../src/tools/index.js';
14+
import type { ToolEntry, UiMode } from '../../src/types.js';
15+
import { getToolPublicFieldOnly } from '../../src/utils/tools.js';
16+
17+
/** Helper to extract tool names from a category. */
18+
function toolNames(tools: ToolEntry[]): string[] {
19+
return tools.map((t) => t.name);
20+
}
21+
22+
describe('buildCategories mode contract', () => {
23+
const defaultCategories = buildCategories();
24+
const openaiCategories = buildCategories('openai');
25+
26+
describe('per-mode tool lists', () => {
27+
it('should have correct tools in experimental category (both modes)', () => {
28+
expect(toolNames(defaultCategories.experimental)).toEqual([HelperTools.ACTOR_ADD]);
29+
expect(toolNames(openaiCategories.experimental)).toEqual([HelperTools.ACTOR_ADD]);
30+
});
31+
32+
it('should have correct tools in actors category (both modes)', () => {
33+
const expected = [HelperTools.STORE_SEARCH, HelperTools.ACTOR_GET_DETAILS, HelperTools.ACTOR_CALL];
34+
expect(toolNames(defaultCategories.actors)).toEqual(expected);
35+
expect(toolNames(openaiCategories.actors)).toEqual(expected);
36+
});
37+
38+
it('should have empty ui category in default mode', () => {
39+
expect(toolNames(defaultCategories.ui)).toEqual([]);
40+
});
41+
42+
it('should have internal tools in ui category in openai mode', () => {
43+
expect(toolNames(openaiCategories.ui)).toEqual([
44+
HelperTools.STORE_SEARCH_INTERNAL,
45+
HelperTools.ACTOR_GET_DETAILS_INTERNAL,
46+
]);
47+
});
48+
49+
it('should have correct tools in docs category (both modes)', () => {
50+
const expected = [HelperTools.DOCS_SEARCH, HelperTools.DOCS_FETCH];
51+
expect(toolNames(defaultCategories.docs)).toEqual(expected);
52+
expect(toolNames(openaiCategories.docs)).toEqual(expected);
53+
});
54+
55+
it('should have correct tools in runs category (both modes)', () => {
56+
const expected = [
57+
HelperTools.ACTOR_RUNS_GET,
58+
HelperTools.ACTOR_RUN_LIST_GET,
59+
HelperTools.ACTOR_RUNS_LOG,
60+
HelperTools.ACTOR_RUNS_ABORT,
61+
];
62+
expect(toolNames(defaultCategories.runs)).toEqual(expected);
63+
expect(toolNames(openaiCategories.runs)).toEqual(expected);
64+
});
65+
66+
it('should have correct tools in storage category (both modes)', () => {
67+
const expected = [
68+
HelperTools.DATASET_GET,
69+
HelperTools.DATASET_GET_ITEMS,
70+
HelperTools.DATASET_SCHEMA_GET,
71+
HelperTools.ACTOR_OUTPUT_GET,
72+
HelperTools.KEY_VALUE_STORE_GET,
73+
HelperTools.KEY_VALUE_STORE_KEYS_GET,
74+
HelperTools.KEY_VALUE_STORE_RECORD_GET,
75+
HelperTools.DATASET_LIST_GET,
76+
HelperTools.KEY_VALUE_STORE_LIST_GET,
77+
];
78+
expect(toolNames(defaultCategories.storage)).toEqual(expected);
79+
expect(toolNames(openaiCategories.storage)).toEqual(expected);
80+
});
81+
82+
it('should have correct tools in dev category (both modes)', () => {
83+
expect(toolNames(defaultCategories.dev)).toEqual([HelperTools.GET_HTML_SKELETON]);
84+
expect(toolNames(openaiCategories.dev)).toEqual([HelperTools.GET_HTML_SKELETON]);
85+
});
86+
});
87+
88+
describe('mode-variant tool name parity', () => {
89+
it('should have identical tool names in actors category across modes', () => {
90+
expect(toolNames(defaultCategories.actors)).toEqual(toolNames(openaiCategories.actors));
91+
});
92+
93+
it('should have identical tool names in runs category across modes', () => {
94+
expect(toolNames(defaultCategories.runs)).toEqual(toolNames(openaiCategories.runs));
95+
});
96+
});
97+
98+
describe('inputSchema parity for mode-variant tools', () => {
99+
const modeVariantToolNames = [
100+
HelperTools.STORE_SEARCH,
101+
HelperTools.ACTOR_GET_DETAILS,
102+
HelperTools.ACTOR_CALL,
103+
HelperTools.ACTOR_RUNS_GET,
104+
];
105+
106+
for (const name of modeVariantToolNames) {
107+
it(`should have identical inputSchema for ${name} across modes`, () => {
108+
const defaultTool = [...defaultCategories.actors, ...defaultCategories.runs]
109+
.find((t) => t.name === name);
110+
const openaiTool = [...openaiCategories.actors, ...openaiCategories.runs]
111+
.find((t) => t.name === name);
112+
113+
expect(defaultTool).toBeDefined();
114+
expect(openaiTool).toBeDefined();
115+
expect(defaultTool!.inputSchema).toEqual(openaiTool!.inputSchema);
116+
});
117+
}
118+
});
119+
120+
describe('tool definitions are frozen', () => {
121+
for (const mode of [undefined, 'openai' as UiMode]) {
122+
const label = mode ?? 'default';
123+
const categories = buildCategories(mode);
124+
125+
for (const categoryName of CATEGORY_NAMES) {
126+
for (const tool of categories[categoryName]) {
127+
it(`${tool.name} (${label} mode) should be frozen`, () => {
128+
expect(Object.isFrozen(tool)).toBe(true);
129+
});
130+
}
131+
}
132+
}
133+
});
134+
135+
describe('all tool names match HelperTools enum values', () => {
136+
const allHelperToolNames = new Set(Object.values(HelperTools));
137+
138+
for (const mode of [undefined, 'openai' as UiMode]) {
139+
const label = mode ?? 'default';
140+
const categories = buildCategories(mode);
141+
142+
for (const categoryName of CATEGORY_NAMES) {
143+
for (const tool of categories[categoryName]) {
144+
it(`${tool.name} (${label} mode) should be a known HelperTools value`, () => {
145+
expect(allHelperToolNames.has(tool.name as HelperTools)).toBe(true);
146+
});
147+
}
148+
}
149+
}
150+
});
151+
});
152+
153+
describe('getToolPublicFieldOnly _meta filtering', () => {
154+
const toolWithOpenAiMeta = {
155+
name: 'test-tool',
156+
description: 'Test',
157+
inputSchema: { type: 'object' as const, properties: {} },
158+
ajvValidate: (() => true) as never,
159+
_meta: {
160+
'openai/widget': { type: 'test' },
161+
'openai/config': { key: 'value' },
162+
'regular-key': { data: 123 },
163+
},
164+
};
165+
166+
it('should strip openai/ _meta keys when filterOpenAiMeta is true and not in openai mode', () => {
167+
const result = getToolPublicFieldOnly(toolWithOpenAiMeta, {
168+
filterOpenAiMeta: true,
169+
uiMode: undefined,
170+
});
171+
expect(result._meta).toBeDefined();
172+
expect(result._meta).toEqual({ 'regular-key': { data: 123 } });
173+
expect(result._meta).not.toHaveProperty('openai/widget');
174+
expect(result._meta).not.toHaveProperty('openai/config');
175+
});
176+
177+
it('should preserve all _meta keys in openai mode', () => {
178+
const result = getToolPublicFieldOnly(toolWithOpenAiMeta, {
179+
filterOpenAiMeta: true,
180+
uiMode: 'openai',
181+
});
182+
expect(result._meta).toEqual(toolWithOpenAiMeta._meta);
183+
});
184+
185+
it('should preserve all _meta keys when filterOpenAiMeta is false', () => {
186+
const result = getToolPublicFieldOnly(toolWithOpenAiMeta, {
187+
filterOpenAiMeta: false,
188+
});
189+
expect(result._meta).toEqual(toolWithOpenAiMeta._meta);
190+
});
191+
192+
it('should return undefined _meta when all keys are openai/ and mode is not openai', () => {
193+
const toolWithOnlyOpenAiMeta = {
194+
...toolWithOpenAiMeta,
195+
_meta: {
196+
'openai/widget': { type: 'test' },
197+
},
198+
};
199+
const result = getToolPublicFieldOnly(toolWithOnlyOpenAiMeta, {
200+
filterOpenAiMeta: true,
201+
uiMode: undefined,
202+
});
203+
expect(result._meta).toBeUndefined();
204+
});
205+
});

0 commit comments

Comments
 (0)