Skip to content

Commit 3d74b88

Browse files
committed
[test] fix: Speed up test_collate_fn_respects_pad_seq_length_to_mult
Reduce memmap worker count to 1 and sample count to 2 to avoid unnecessary multiprocessing overhead in a unit test. Signed-off-by: Chen Cui <chcui@nvidia.com>
1 parent ca94b04 commit 3d74b88

1 file changed

Lines changed: 2 additions & 1 deletion

File tree

‎tests/unit_tests/data/datasets/test_sft.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@ def test_collate_fn_respects_pad_seq_length_to_mult(self, tmp_path):
331331
path = str(datasets_dir / "sft.jsonl")
332332
line = {"input": "hi how are you?", "output": "I'm fine, thanks."}
333333
with open(path, "w") as f:
334-
for _ in range(10):
334+
for _ in range(2):
335335
f.write(json.dumps(line) + "\n")
336336

337337
tokenizer = create_mock_tokenizer()
@@ -342,6 +342,7 @@ def test_collate_fn_respects_pad_seq_length_to_mult(self, tmp_path):
342342
prompt_template="{input}\n\n### Response:\n{output}",
343343
truncation_field="output",
344344
pad_seq_length_to_mult=32,
345+
memmap_workers=1,
345346
)
346347
batch = [
347348
{

0 commit comments

Comments
 (0)