Skip to content

Commit 331e006

Browse files
feat(audio-datasets): make dataset manifest stages agent-ready (#2341)
* feat(audio-datasets): make dataset manifest stages agent-ready Adds describe() contracts to the FLEURS and ReadSpeech initial-manifest stages so a planner can tell what each produces before running it. Depends on the agent-ready foundation (nemo_curator/stages/audio/_agent/), which must merge first. Signed-off-by: Shubham Bhawsar <shbhawsar@nvidia.com> * docs(audio-datasets): clarify malformed transcript handling Signed-off-by: Shubham Bhawsar <shbhawsar@nvidia.com> * fix(audio-datasets): correct agent contracts Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * fix(audio-datasets): validate readspeech key remapping Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * test(audio-datasets): satisfy ruff checks Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * test(audio-datasets): align with ci ruff Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * fix(audio): report bounded dataset dependence Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * fix(audio-datasets): reject output key collisions Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * test(audio-datasets): use current agent contract import Signed-off-by: shbhawsar <shbhawsar@nvidia.com> * test(audio-datasets): remove the agent package dependency Signed-off-by: shbhawsar <shbhawsar@nvidia.com> --------- Signed-off-by: Shubham Bhawsar <shbhawsar@nvidia.com> Signed-off-by: shbhawsar <shbhawsar@nvidia.com> Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com>
1 parent 4ce7266 commit 331e006

4 files changed

Lines changed: 347 additions & 15 deletions

File tree

‎nemo_curator/stages/audio/datasets/fleurs/create_initial_manifest.py‎

Lines changed: 45 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414

1515
import os
1616
import shutil
17-
from dataclasses import dataclass
17+
from dataclasses import dataclass, field
18+
from typing import ClassVar
1819

1920
from huggingface_hub import hf_hub_download
2021
from loguru import logger
2122

23+
from nemo_curator.stages.audio._agent._agent_ready import AgentReady, Gates, IOSpec, StageContract, StaticHints
2224
from nemo_curator.stages.audio.datasets.file_utils import extract_archive
2325
from nemo_curator.stages.base import ProcessingStage
2426
from nemo_curator.tasks import AudioTask, EmptyTask
@@ -40,7 +42,7 @@ def get_fleurs_filenames(lang: str, split: str) -> tuple[str, str]:
4042

4143

4244
@dataclass
43-
class CreateInitialManifestFleursStage(ProcessingStage[EmptyTask, AudioTask]):
45+
class CreateInitialManifestFleursStage(AgentReady, ProcessingStage[EmptyTask, AudioTask]):
4446
"""Create initial manifest for the FLEURS dataset.
4547
4648
Dataset link: https://huggingface.co/datasets/google/fleurs
@@ -70,10 +72,19 @@ class CreateInitialManifestFleursStage(ProcessingStage[EmptyTask, AudioTask]):
7072
``benchmarking/data_prep/prepare_fleurs_data.py``).
7173
"""
7274

75+
AGENT_STATIC: ClassVar[StaticHints] = StaticHints(
76+
gates=Gates(
77+
writes_to_disk=True,
78+
output_path_params=["raw_data_dir", "cache_dir"],
79+
requires_internet_first_run=True,
80+
per_row_independent=True,
81+
)
82+
)
83+
7384
name: str = "CreateInitialManifestFleurs"
74-
lang: str = ""
75-
split: str = ""
76-
raw_data_dir: str = ""
85+
lang: str = field(default="", metadata={"agent_required": True})
86+
split: str = field(default="", metadata={"agent_required": True})
87+
raw_data_dir: str = field(default="", metadata={"agent_required": True})
7788
filepath_key: str = "audio_filepath"
7889
text_key: str = "text"
7990
batch_size: int = 1
@@ -85,13 +96,42 @@ def __post_init__(self) -> None:
8596
if not getattr(self, attr):
8697
msg = f"{attr} is required for CreateInitialManifestFleursStage"
8798
raise ValueError(msg)
99+
output_key_fields = {
100+
"filepath_key": self.filepath_key,
101+
"text_key": self.text_key,
102+
}
103+
for field_name, key in output_key_fields.items():
104+
if not isinstance(key, str) or not key.strip():
105+
msg = f"{field_name} must be a non-empty string"
106+
raise ValueError(msg)
107+
if self.filepath_key == self.text_key:
108+
msg = f"filepath_key={self.filepath_key!r} conflicts with text_key"
109+
raise ValueError(msg)
88110

89111
def inputs(self) -> tuple[list[str], list[str]]:
90112
return [], []
91113

92114
def outputs(self) -> tuple[list[str], list[str]]:
93115
return [], [self.filepath_key, self.text_key]
94116

117+
def describe(self) -> StageContract:
118+
return StageContract(
119+
writes=IOSpec(
120+
data_keys=[self.filepath_key, self.text_key],
121+
produces=["disk"] if self.auto_download else [],
122+
),
123+
cardinality="1:N fan-out",
124+
# The download stages the split once; after that every valid transcript line
125+
# emits its own path and text in input order. Malformed lines with fewer than
126+
# three tab-separated columns are skipped.
127+
gates=Gates(
128+
writes_to_disk=self.auto_download,
129+
output_path_params=["raw_data_dir", "cache_dir"] if self.auto_download else [],
130+
requires_internet_first_run=self.auto_download,
131+
per_row_independent=True,
132+
),
133+
)
134+
95135
def language_data_dir(self) -> str:
96136
"""Return the per-language download directory under ``raw_data_dir``.
97137

‎nemo_curator/stages/audio/datasets/readspeech/create_initial_manifest.py‎

Lines changed: 77 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,11 @@
1616
import os
1717
import subprocess
1818
from dataclasses import dataclass
19+
from typing import ClassVar
1920

2021
from loguru import logger
2122

23+
from nemo_curator.stages.audio._agent._agent_ready import AgentReady, Gates, IOSpec, StageContract, StaticHints
2224
from nemo_curator.stages.audio.datasets.file_utils import download_file
2325
from nemo_curator.stages.base import ProcessingStage
2426
from nemo_curator.tasks import AudioTask, EmptyTask
@@ -33,7 +35,7 @@
3335

3436

3537
@dataclass
36-
class CreateInitialManifestReadSpeechStage(ProcessingStage[EmptyTask, AudioTask]):
38+
class CreateInitialManifestReadSpeechStage(AgentReady, ProcessingStage[EmptyTask, AudioTask]):
3739
"""
3840
Stage to create initial manifest for the DNS Challenge Read Speech dataset.
3941
@@ -47,24 +49,90 @@ class CreateInitialManifestReadSpeechStage(ProcessingStage[EmptyTask, AudioTask]
4749
raw_data_dir: Directory where data will be downloaded/extracted to.
4850
max_samples: Maximum number of samples to include (-1 for all).
4951
auto_download: If True, automatically download and extract dataset.
52+
filepath_key: Key name used for each emitted audio path.
53+
text_key: Key name used for each emitted transcript.
54+
sample_rate_key: Key name used for the constant 48 kHz sample rate.
55+
book_id_key: Key name used for the parsed book identifier.
56+
reader_id_key: Key name used for the parsed reader identifier.
5057
"""
5158

59+
AGENT_STATIC: ClassVar[StaticHints] = StaticHints(
60+
gates=Gates(
61+
writes_to_disk=True,
62+
output_path_params=["raw_data_dir"],
63+
requires_internet_first_run=True,
64+
per_row_independent=False,
65+
)
66+
)
67+
INTERNAL_KEY_FIELDS: ClassVar[frozenset[str]] = frozenset({"book_id_key", "reader_id_key"})
68+
5269
raw_data_dir: str
5370
max_samples: int = 5000
5471
auto_download: bool = True
5572
filepath_key: str = "audio_filepath"
5673
text_key: str = "text"
5774
name: str = "CreateInitialManifestReadSpeech"
5875
batch_size: int = 1
76+
sample_rate_key: str = "sample_rate"
77+
book_id_key: str = "book_id"
78+
reader_id_key: str = "reader_id"
5979

6080
def __post_init__(self):
6181
super().__init__()
82+
output_key_fields = {
83+
"filepath_key": self.filepath_key,
84+
"text_key": self.text_key,
85+
"sample_rate_key": self.sample_rate_key,
86+
"book_id_key": self.book_id_key,
87+
"reader_id_key": self.reader_id_key,
88+
}
89+
for field_name, key in output_key_fields.items():
90+
if not isinstance(key, str) or not key.strip():
91+
msg = f"{field_name} must be a non-empty string"
92+
raise ValueError(msg)
93+
conflicting_fields = sorted(
94+
other_name
95+
for other_name, other_key in output_key_fields.items()
96+
if other_name != field_name and other_key == key
97+
)
98+
if conflicting_fields:
99+
msg = f"{field_name}={key!r} conflicts with {', '.join(conflicting_fields)}"
100+
raise ValueError(msg)
62101

63102
def inputs(self) -> tuple[list[str], list[str]]:
64103
return [], []
65104

66105
def outputs(self) -> tuple[list[str], list[str]]:
67-
return [], [self.filepath_key, self.text_key]
106+
return [], [
107+
self.filepath_key,
108+
self.text_key,
109+
self.sample_rate_key,
110+
self.book_id_key,
111+
self.reader_id_key,
112+
]
113+
114+
def describe(self) -> StageContract:
115+
return StageContract(
116+
writes=IOSpec(
117+
data_keys=[
118+
self.filepath_key,
119+
self.text_key,
120+
self.sample_rate_key,
121+
self.book_id_key,
122+
self.reader_id_key,
123+
],
124+
produces=["disk"] if self.auto_download else [],
125+
),
126+
cardinality="1:N fan-out",
127+
gates=Gates(
128+
writes_to_disk=self.auto_download,
129+
requires_internet_first_run=self.auto_download,
130+
output_path_params=["raw_data_dir"] if self.auto_download else [],
131+
# A positive max_samples selects a prefix of the globally sorted corpus, so a
132+
# delta containing only changed files can admit rows a full run would exclude.
133+
per_row_independent=self.max_samples <= 0,
134+
),
135+
)
68136

69137
def download_and_extract(self) -> str:
70138
"""Download and extract DNS Challenge Read Speech dataset (~4.88 GB)."""
@@ -249,9 +317,9 @@ def collect_audio_files(self, search_dir: str) -> list[dict]:
249317
entry = {
250318
self.filepath_key: os.path.abspath(wav_path),
251319
self.text_key: "",
252-
"sample_rate": SAMPLE_RATE_48KHZ,
253-
"book_id": metadata.get("book_id", ""),
254-
"reader_id": metadata.get("reader_id", ""),
320+
self.sample_rate_key: SAMPLE_RATE_48KHZ,
321+
self.book_id_key: metadata.get("book_id", ""),
322+
self.reader_id_key: metadata.get("reader_id", ""),
255323
}
256324
entries.append(entry)
257325

@@ -286,10 +354,10 @@ def verify_dataset_structure(self, entries: list[dict]) -> None:
286354
unique_readers = set()
287355
unique_books = set()
288356
for entry in entries:
289-
if entry.get("reader_id"):
290-
unique_readers.add(entry["reader_id"])
291-
if entry.get("book_id"):
292-
unique_books.add(entry["book_id"])
357+
if entry.get(self.reader_id_key):
358+
unique_readers.add(entry[self.reader_id_key])
359+
if entry.get(self.book_id_key):
360+
unique_books.add(entry[self.book_id_key])
293361

294362
logger.info(f"Unique readers: {len(unique_readers)}")
295363
logger.info(f"Unique books: {len(unique_books)}")

‎tests/stages/audio/datasets/test_fleurs_create_initial_manifest.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,11 @@
2020

2121
import pytest
2222

23+
from nemo_curator.stages.audio._agent._agent_ready import to_json_schema
24+
from nemo_curator.stages.audio._agent._agent_registry import build_contract, stage_params, static_contract
25+
from nemo_curator.stages.audio._agent._catalog import role_index
26+
from nemo_curator.stages.audio._agent._conformance import assert_agent_ready
27+
2328

2429
def _import_stage_module() -> tuple[Any, Any]:
2530
# Inject a stub for optional dependency 'wget' to avoid import errors
@@ -74,6 +79,41 @@ def test_inputs_outputs(tmp_path: Path) -> None:
7479
assert stage.outputs() == ([], ["audio_filepath", "text"])
7580

7681

82+
def test_agent_schema_marks_runtime_validated_params_required() -> None:
83+
stage_cls, _ = _import_stage_module()
84+
params = stage_params(stage_cls)
85+
86+
assert to_json_schema(params)["required"] == ["lang", "split", "raw_data_dir"]
87+
88+
index = role_index()
89+
assert "CreateInitialManifestFleursStage" not in index["unresolved_stages"]
90+
assert "CreateInitialManifestFleursStage" in index["producers"]["audio_filepath"]
91+
assert "CreateInitialManifestFleursStage" in index["producers"]["text"]
92+
93+
94+
def test_contract_has_conditional_download_gate_and_static_hints(tmp_path: Path) -> None:
95+
stage_cls, _ = _import_stage_module()
96+
local = stage_cls(lang="en_us", split="dev", raw_data_dir=str(tmp_path), auto_download=False)
97+
downloading = stage_cls(lang="en_us", split="dev", raw_data_dir=str(tmp_path), auto_download=True)
98+
99+
local_contract = build_contract(local)
100+
assert local_contract.writes.produces == []
101+
assert local_contract.gates.writes_to_disk is False
102+
assert local_contract.gates.requires_internet_first_run is False
103+
assert local_contract.gates.output_path_params == []
104+
105+
download_contract = build_contract(downloading)
106+
assert download_contract.writes.produces == ["disk"]
107+
assert download_contract.gates.writes_to_disk is True
108+
assert download_contract.gates.requires_internet_first_run is True
109+
assert download_contract.gates.output_path_params == ["raw_data_dir", "cache_dir"]
110+
111+
static = static_contract(stage_cls)
112+
assert static.gates.writes_to_disk is True
113+
assert static.gates.requires_internet_first_run is True
114+
assert static.gates.output_path_params == ["raw_data_dir", "cache_dir"]
115+
116+
77117
def test_language_data_dir_is_namespaced_per_language(tmp_path: Path) -> None:
78118
stage_cls, _ = _import_stage_module()
79119
raw = str(tmp_path / "fleurs")
@@ -208,6 +248,63 @@ def test_process_no_download_reads_prestaged(tmp_path: Path) -> None:
208248
assert results[1].data["text"] == "world"
209249

210250

251+
def test_process_uses_renamed_output_keys_declared_by_contract(tmp_path: Path) -> None:
252+
stage_cls, _ = _import_stage_module()
253+
raw_dir = _stage_prestaged_layout(tmp_path)
254+
stage = stage_cls(
255+
lang="hy_am",
256+
split="train",
257+
raw_data_dir=str(raw_dir),
258+
auto_download=False,
259+
filepath_key="path",
260+
text_key="transcript",
261+
)
262+
263+
from nemo_curator.tasks import EmptyTask
264+
265+
results = stage.process(EmptyTask(dataset_name="test", data=None))
266+
267+
assert results
268+
assert set(results[0].data) == {"path", "transcript"}
269+
assert build_contract(stage).writes.data_keys == ["path", "transcript"]
270+
271+
272+
@pytest.mark.parametrize(
273+
"kwargs",
274+
[
275+
{"filepath_key": ""},
276+
{"filepath_key": " "},
277+
{"text_key": ""},
278+
{"filepath_key": "value", "text_key": "value"},
279+
],
280+
)
281+
def test_output_keys_reject_empty_names_and_collisions(tmp_path: Path, kwargs: dict) -> None:
282+
stage_cls, _ = _import_stage_module()
283+
284+
with pytest.raises(ValueError, match=r"must be a non-empty string|conflicts with"):
285+
stage_cls(
286+
lang="hy_am",
287+
split="train",
288+
raw_data_dir=str(tmp_path),
289+
auto_download=False,
290+
**kwargs,
291+
)
292+
293+
294+
def test_agent_conformance(tmp_path: Path) -> None:
295+
stage_cls, _ = _import_stage_module()
296+
raw_dir = _stage_prestaged_layout(tmp_path)
297+
stage = stage_cls(lang="hy_am", split="train", raw_data_dir=str(raw_dir), auto_download=False)
298+
from nemo_curator.tasks import EmptyTask
299+
300+
assert_agent_ready(
301+
stage,
302+
lambda: EmptyTask(dataset_name="test", data=None),
303+
expected_cardinality="1:N fan-out",
304+
available_keys=set(),
305+
)
306+
307+
211308
def _import_prep_module() -> types.ModuleType:
212309
"""Import benchmarking/data_prep/prepare_fleurs_data.py (not on the default path)."""
213310
prep_dir = Path(__file__).resolve().parents[4] / "benchmarking" / "data_prep"

0 commit comments

Comments
 (0)