Skip to content

Commit 3e8a11d

Browse files
committed
bit parity with training harness
1 parent 394b5e5 commit 3e8a11d

3 files changed

Lines changed: 61 additions & 4 deletions

File tree

‎rlm/clients/base_lm.py‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,19 @@ class BaseLM(ABC):
1313
does so in a model-agnostic way, so this class provides a base interface for all language models.
1414
"""
1515

16-
def __init__(self, model_name: str, timeout: float = DEFAULT_TIMEOUT, **kwargs):
16+
def __init__(
17+
self,
18+
model_name: str,
19+
timeout: float = DEFAULT_TIMEOUT,
20+
sampling_args: dict[str, Any] | None = None,
21+
**kwargs,
22+
):
1723
self.model_name = model_name
1824
self.timeout = timeout
25+
# Sampling args forwarded to the underlying completion API
26+
# (e.g. temperature, top_p, max_tokens, seed). Forwarded by
27+
# subclasses as **self.sampling_args.
28+
self.sampling_args: dict[str, Any] = dict(sampling_args or {})
1929
self.kwargs = kwargs
2030

2131
@abstractmethod

‎rlm/clients/openai.py‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,19 @@
1818
DEFAULT_PRIME_INTELLECT_BASE_URL = "https://api.pinference.ai/api/v1/"
1919

2020

21+
def _normalize_sampling_args(sampling_args: dict[str, Any]) -> dict[str, Any]:
22+
"""Match the rename done by verifiers' OpenAIChatCompletionsClient so the
23+
same sampling_args dict produces byte-equivalent chat.completions.create
24+
calls in both harnesses. ``max_tokens`` is the legacy OpenAI param;
25+
vLLM treats ``max_completion_tokens`` differently in its pre-reservation
26+
check (legacy is reserved up-front against max_model_len; new is not).
27+
"""
28+
args = dict(sampling_args or {})
29+
if "max_tokens" in args:
30+
args["max_completion_tokens"] = args.pop("max_tokens")
31+
return {k: v for k, v in args.items() if v is not None}
32+
33+
2134
class OpenAIClient(BaseLM):
2235
"""
2336
LM Client for running models with the OpenAI API. Works with vLLM as well.
@@ -32,9 +45,10 @@ def __init__(
3245
api_key: str | None = None,
3346
model_name: str | None = None,
3447
base_url: str | None = None,
48+
sampling_args: dict[str, Any] | None = None,
3549
**kwargs,
3650
):
37-
super().__init__(model_name=model_name, **kwargs)
51+
super().__init__(model_name=model_name, sampling_args=sampling_args, **kwargs)
3852

3953
if api_key is None:
4054
if base_url == "https://api.openai.com/v1" or base_url is None:
@@ -83,7 +97,8 @@ def completion(self, prompt: str | list[dict[str, Any]], model: str | None = Non
8397
extra_body["usage"] = {"include": True}
8498

8599
response = self.client.chat.completions.create(
86-
model=model, messages=messages, extra_body=extra_body
100+
model=model, messages=messages, extra_body=extra_body,
101+
**_normalize_sampling_args(self.sampling_args),
87102
)
88103
self._track_cost(response, model)
89104
return response.choices[0].message.content
@@ -107,7 +122,8 @@ async def acompletion(
107122
extra_body["usage"] = {"include": True}
108123

109124
response = await self.async_client.chat.completions.create(
110-
model=model, messages=messages, extra_body=extra_body
125+
model=model, messages=messages, extra_body=extra_body,
126+
**_normalize_sampling_args(self.sampling_args),
111127
)
112128
self._track_cost(response, model)
113129
return response.choices[0].message.content

‎rlm/core/rlm.py‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,9 @@ def __init__(
7474
on_subcall_complete: Callable[[int, str, float, str | None], None] | None = None,
7575
on_iteration_start: Callable[[int, int], None] | None = None,
7676
on_iteration_complete: Callable[[int, int, float], None] | None = None,
77+
sampling_args: dict[str, Any] | None = None,
78+
sub_sampling_args: dict[str, Any] | None = None,
79+
orchestrator: bool = True,
7780
):
7881
"""
7982
Args:
@@ -109,6 +112,32 @@ def __init__(
109112
on_iteration_start: Callback fired when an iteration starts. Args: (depth, iteration_num).
110113
on_iteration_complete: Callback fired when an iteration completes. Args: (depth, iteration_num, duration).
111114
"""
115+
# Sampling args plumbed into backend_kwargs / other_backend_kwargs
116+
# before the clients are constructed, so they reach the chat-completions
117+
# call (e.g. temperature, top_p, max_tokens, seed). ``sampling_args``
118+
# applies to the root model (depth=0); ``sub_sampling_args`` to
119+
# depth=1 sub-LLM calls. If ``sub_sampling_args`` is set without an
120+
# ``other_backends``, we mirror the root backend so depth=1 routes
121+
# through a separate client with its own sampling args.
122+
if sampling_args is not None:
123+
backend_kwargs = dict(backend_kwargs or {})
124+
existing = dict(backend_kwargs.get("sampling_args") or {})
125+
existing.update(sampling_args)
126+
backend_kwargs["sampling_args"] = existing
127+
if sub_sampling_args is not None:
128+
if other_backends is None:
129+
other_backends = [backend]
130+
other_backend_kwargs = [dict(backend_kwargs or {})]
131+
else:
132+
other_backend_kwargs = [
133+
dict(kw or {}) for kw in (other_backend_kwargs or [{}])
134+
]
135+
first = dict(other_backend_kwargs[0])
136+
existing = dict(first.get("sampling_args") or {})
137+
existing.update(sub_sampling_args)
138+
first["sampling_args"] = existing
139+
other_backend_kwargs[0] = first
140+
112141
# Store config for spawning per-completion
113142
self.backend = backend
114143
self.backend_kwargs = backend_kwargs
@@ -144,6 +173,7 @@ def __init__(
144173
self.max_tokens = max_tokens
145174
self.max_errors = max_errors
146175
self.system_prompt = custom_system_prompt if custom_system_prompt else RLM_SYSTEM_PROMPT
176+
self.orchestrator = orchestrator
147177
self.logger = logger
148178
self.verbose = VerbosePrinter(enabled=verbose)
149179

@@ -276,6 +306,7 @@ def _setup_prompt(
276306
query_metadata=metadata,
277307
custom_tools=self.custom_tools,
278308
root_prompt=root_prompt,
309+
orchestrator=self.orchestrator,
279310
)
280311
if self.compaction:
281312
message_history[0]["content"] += (

0 commit comments

Comments
 (0)