Skip to content

Commit 2bbf11a

Browse files
Merge pull request #198 from caogaoshuai/feat/pg-opd
feat: support PG-OPD unified training flow
1 parent 0810585 commit 2bbf11a

19 files changed

Lines changed: 698 additions & 68 deletions

‎dashscope/cli/agentic_rl.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -460,6 +460,14 @@ def run(
460460
"--config",
461461
help="Path to YAML configuration file",
462462
),
463+
teacher_model: Optional[str] = typer.Option(
464+
None,
465+
"--teacher-model",
466+
help=(
467+
"Enable OPD with this teacher model (overrides YAML); "
468+
"Rollout and Reward are optional"
469+
),
470+
),
463471
job_name: Optional[str] = typer.Option(
464472
None,
465473
help="Custom name for the tuning job",
@@ -491,10 +499,8 @@ def run(
491499
1. Configuration-driven: Use -c/--config to specify a YAML file
492500
2. Direct parameter: Provide all required arguments via CLI options
493501
494-
Required parameters:
495-
- rollout_classpath
496-
- reward_classpaths (at least one)
497-
- training_files (at least one)
502+
Rollout and Reward are required for regular reinforcement learning, but
503+
optional for OPD.
498504
"""
499505
output_format = validate_output_format(output_format)
500506
_apply_verbose(verbose)
@@ -505,6 +511,7 @@ def run(
505511

506512
# Prepare workflow parameters
507513
run_kwargs = {
514+
"teacher_model": teacher_model,
508515
"job_name": job_name,
509516
}
510517

‎dashscope/finetune/agentic_rl.py‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
from dashscope.finetune.reinforcement import (
2929
FunctionType,
3030
DatasetsType,
31+
TrainingType,
3132
)
3233
from dashscope.finetune.reinforcement import (
3334
RewardInput,
@@ -171,11 +172,14 @@ def submit_job(
171172
hyper_parameters: Optional[Dict[str, str]] = None,
172173
resources: Optional[Dict[str, str]] = None,
173174
job_name: Optional[str] = None,
175+
teacher_model: Optional[str] = None,
174176
**kwargs,
175177
) -> FineTune:
176178
"""
177179
Submit RL tuning job to the platform.
178180
"""
181+
teacher_model = teacher_model or self.tuning.teacher_model
182+
179183
# Resolve job name (fallback to class default)
180184
if job_name:
181185
self.tuning.name = job_name
@@ -230,6 +234,15 @@ def submit_job(
230234
if resources:
231235
self.tuning.training.resources = resources
232236

237+
training_type = self.tuning.training.type
238+
if teacher_model:
239+
training_type = TrainingType.PG_OPD
240+
elif training_type == TrainingType.PG_OPD:
241+
raise ValueErrorWithCode(
242+
f"teacher_model is required when training.type is {training_type}",
243+
error_code=3006,
244+
)
245+
233246
request = {
234247
"model": self.tuning.model.name,
235248
"training_datasets": [ds.model_dump() for ds in training_datasets],
@@ -240,7 +253,8 @@ def submit_job(
240253
"rewards": rewards,
241254
"hyper_parameters": self.tuning.training.hyper_parameters,
242255
"resource_config": self.tuning.training.resources,
243-
"training_type": str(self.tuning.training.type),
256+
"training_type": str(training_type),
257+
"teacher_model": teacher_model,
244258
"job_name": job_name_with_suffix,
245259
}
246260
request = deep_remove_none(request)
@@ -287,6 +301,7 @@ async def run(
287301
hyper_parameters: Optional[Dict[str, str]] = None,
288302
resources: Optional[Dict[str, str]] = None,
289303
job_name: Optional[str] = None,
304+
teacher_model: Optional[str] = None,
290305
**kwargs,
291306
) -> FineTune:
292307
"""
@@ -315,6 +330,7 @@ async def run(
315330
hyper_parameters=hyper_parameters,
316331
resources=resources,
317332
job_name=job_name,
333+
teacher_model=teacher_model,
318334
**kwargs,
319335
)
320336
except Exception as e:

‎dashscope/finetune/reinforcement/common/model.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1397,6 +1397,7 @@ class TuningModel(Models, BaseModel):
13971397

13981398
name: str = Field(default="agentic-rl", min_length=1, max_length=256)
13991399
model: FoundationModel = FoundationModel()
1400+
teacher_model: Optional[str] = None
14001401
functions: List[FunctionComponentUnion] = Field(default_factory=list)
14011402
datasets: List[Dataset] = Field(default_factory=list)
14021403
training: Training = Training()

‎dashscope/finetune/reinforcement/common/model_types.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ def __str__(self):
4747

4848
class TrainingType(str, Enum):
4949
TRAINING_TYPE = "reinforcement"
50+
PG_OPD = "pg_opd"
5051

5152
def __str__(self):
5253
return self.value

‎dashscope/finetune/reinforcement/common/utils.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -381,14 +381,15 @@ def create_deployment_files(
381381
function_layer_used=FC_LAYER_USED,
382382
)
383383

384-
with open(FC_FILES_START, "w", encoding="utf-8") as f:
384+
start_file = os.path.join(dirpath, FC_FILES_START)
385+
with open(start_file, "w", encoding="utf-8") as f:
385386
f.write(content)
386387

387388
# Add execute permission on Unix systems
388389
if os.name == "posix":
389-
os.chmod(FC_FILES_START, 0o755)
390+
os.chmod(start_file, 0o755)
390391

391-
logger.debug(f"Generated startup script: {FC_FILES_START}")
392+
logger.debug(f"Generated startup script: {start_file}")
392393
except Exception as e:
393394
raise RuntimeErrorWithCode(
394395
"Deployment file creation error",

‎dashscope/finetune/reinforcement/examples/workspace/README-zh.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ langchain-openai==1.2.0
5656

5757
> 注:functions/目录下需要包含__init__.py文件
5858
59+
函数组件是否必需取决于训练配置:
60+
61+
| 配置 | 场景 | 自定义 Rollout | 自定义 Reward |
62+
|---|---|---:|---:|
63+
| `rl-job.yaml` | 普通强化学习 | 必选 | 至少一个 |
64+
| `opd-job.yaml` 不保留函数块 | 仅 Teacher | 否 | 否 |
65+
| `opd-job.yaml` 仅保留 Reward | Teacher + Reward | 否 | 是 |
66+
| `opd-job.yaml` 仅保留 Rollout | Teacher + Rollout | 是 | 否 |
67+
| `opd-job.yaml` 保留两个函数块 | Teacher + Rollout + Reward | 是 | 是 |
68+
5969
### 3.2 准备训练数据
6070

6171
在`data`目录下添加数据集文件:
@@ -67,19 +77,56 @@ langchain-openai==1.2.0
6777

6878
### 4.1 函数执行(注册+测试)
6979

80+
普通强化学习必须注册自定义 Rollout 和至少一个 Reward。OPD 的
81+
Rollout、Reward 可选。
82+
7083
```bash
7184
python test_functions.py
7285
```
7386

7487
### 4.2 工作流执行(YAML配置+生命周期管理)
7588

89+
普通强化学习与 OPD 使用完全相同的 SDK 工作流,只需选择对应 YAML:
90+
91+
```python
92+
from dashscope.finetune.agentic_rl import AgenticRL
93+
94+
client = AgenticRL()
95+
# 普通强化学习使用 rl-job.yaml;OPD 使用 opd-job.yaml
96+
client.init(config_path="rl-job.yaml")
97+
result = await client.run()
98+
```
99+
100+
`opd-job.yaml` 默认保留两个函数块,表示 Teacher + Rollout + Reward。删除
101+
Reward 块表示 Teacher + Rollout;删除 Rollout 块表示 Teacher + Reward;
102+
两个都删除表示仅 Teacher。
103+
76104
```bash
105+
# submit_job.py 默认使用普通强化学习 rl-job.yaml
77106
python submit_job.py
78107
```
79108

80109
## 5. 使用CLI执行任务
81110
示例代码:cli.sh
82111

112+
`cli.sh` 展示普通强化学习的完整流程。执行 OPD 时,对 `opd-job.yaml` 中已经
113+
删除的函数块跳过对应注册和测试;数据提交、`rl run` 和任务生命周期命令
114+
保持不变。
115+
116+
CLI 与 SDK 使用同一份 YAML:
117+
118+
```bash
119+
# 普通强化学习
120+
dashscope rl run -c rl-job.yaml
121+
122+
# OPD
123+
dashscope rl run -c opd-job.yaml
124+
125+
# 覆盖 opd-job.yaml 中配置的 Teacher 模型
126+
dashscope rl run -c opd-job.yaml \
127+
--teacher-model qwen3.5-397b-a17b
128+
```
129+
83130
```bash
84131
dashscope rl --help # 查看完整命令帮助
85132

‎dashscope/finetune/reinforcement/examples/workspace/README.md‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,16 @@ Develop functions under the `functions` directory:
5656

5757
> Note: The `functions/` directory must contain an `__init__.py` file
5858
59+
Function components depend on the selected training configuration:
60+
61+
| Configuration | Mode | Custom Rollout | Custom Reward |
62+
|---|---|---:|---:|
63+
| `rl-job.yaml` | Regular reinforcement | Required | At least one |
64+
| `opd-job.yaml` without function blocks | Teacher only | No | No |
65+
| `opd-job.yaml` with Reward only | Teacher + Reward | No | Yes |
66+
| `opd-job.yaml` with Rollout only | Teacher + Rollout | Yes | No |
67+
| `opd-job.yaml` with both blocks | Teacher + Rollout + Reward | Yes | Yes |
68+
5969
### 3.2 Preparing Training Data
6070

6171
Add dataset files under the `data` directory:
@@ -67,19 +77,57 @@ Add dataset files under the `data` directory:
6777

6878
### 4.1 Function Execution (Register + Test)
6979

80+
Regular reinforcement training requires a custom Rollout and at least one
81+
Reward. Rollout and Reward are optional for OPD.
82+
7083
```bash
7184
python test_functions.py
7285
```
7386

7487
### 4.2 Workflow Execution (YAML Config + Lifecycle Management)
7588

89+
Regular reinforcement training and OPD use the same SDK workflow. Select the
90+
corresponding YAML:
91+
92+
```python
93+
from dashscope.finetune.agentic_rl import AgenticRL
94+
95+
client = AgenticRL()
96+
# Use rl-job.yaml for regular reinforcement or opd-job.yaml for OPD
97+
client.init(config_path="rl-job.yaml")
98+
result = await client.run()
99+
```
100+
101+
`opd-job.yaml` keeps both function blocks by default, representing Teacher +
102+
Rollout + Reward. Remove the Reward block for Teacher + Rollout, remove the
103+
Rollout block for Teacher + Reward, or remove both for Teacher only.
104+
76105
```bash
106+
# submit_job.py uses the regular reinforcement rl-job.yaml by default
77107
python submit_job.py
78108
```
79109

80110
## 5. Executing Tasks with CLI
81111
Example code:cli.sh
82112

113+
`cli.sh` demonstrates the regular reinforcement workflow. For OPD, skip
114+
registration and testing for function blocks removed from `opd-job.yaml`; dataset
115+
submission, `rl run`, and job lifecycle commands remain unchanged.
116+
117+
The CLI consumes the same YAML files as the SDK:
118+
119+
```bash
120+
# Regular reinforcement
121+
dashscope rl run -c rl-job.yaml
122+
123+
# OPD
124+
dashscope rl run -c opd-job.yaml
125+
126+
# Override the Teacher configured in opd-job.yaml
127+
dashscope rl run -c opd-job.yaml \
128+
--teacher-model qwen3.5-397b-a17b
129+
```
130+
83131
```bash
84132
dashscope rl --help # View full command help
85133

0 commit comments

Comments
 (0)