2525 records = parser.to_llm_dataset(anonymize="strict")
2626 parser.export_jsonl("review.jsonl")
2727
28+ and in 1.5.1 you choose how much of the document comes with each comment, fix
29+ the classifier's tags by hand, and spend fewer tokens doing it::
30+
31+ records = parser.to_llm_dataset(
32+ context={"unit": "sentences", "before": 2, "after": 1},
33+ overrides={4: {"priority": "blocker"}},
34+ compress="balanced",
35+ )
36+
2837Export methods are attached to the compiled ``DocxParser`` / ``BatchParser``
2938classes at import time. That is deliberate: it keeps a single parser object as
3039the one thing a user needs to hold, rather than introducing a second wrapper
7281 from .privacy .policy import AnonymizationPolicy
7382 from .reporting .analytics import build_report_data
7483
75- __version__ = "1.5.0 "
84+ __version__ = "1.5.1 "
7685
7786__all__ = [
7887 # compiled core (unchanged public API)
@@ -319,11 +328,14 @@ def _parser_to_llm_records(
319328 classify : bool = True ,
320329 anonymize : Optional [Any ] = None ,
321330 anonymizer : Optional [Any ] = None ,
331+ ** kwargs : Any ,
322332) -> "LLMDataset" :
323333 """Return this document's comments as an :class:`LLMDataset`.
324334
325335 The object form of :meth:`to_llm_dataset`: same records, plus the
326- anonymisation report and the vault that reverses it.
336+ anonymisation report and the vault that reverses it. Also takes
337+ ``context=`` (sentences, paragraphs, words), ``overrides=`` (manual tags)
338+ and ``compress=`` (token reduction) — see ``llm.build_dataset``.
327339 """
328340 from .llm .dataset import build_dataset
329341
@@ -336,6 +348,7 @@ def _parser_to_llm_records(
336348 classify = classify ,
337349 anonymize = anonymize ,
338350 anonymizer = anonymizer ,
351+ ** kwargs ,
339352 )
340353
341354
@@ -380,17 +393,25 @@ def _parser_to_embeddings_input(
380393 )
381394
382395
396+ #: build_dataset keywords that to_llm_chunks forwards, as opposed to the
397+ #: chunking keywords it passes to Chunk packing.
398+ _DATASET_OPTIONS = frozenset (
399+ {
400+ "chars_before" , "chars_after" , "include_thread" , "include_context" ,
401+ "classify" , "anonymize" , "anonymizer" , "documents" , "context" ,
402+ "overrides" , "compress" ,
403+ }
404+ )
405+
406+
383407def _parser_to_llm_chunks (
384408 self : DocxParser , max_tokens : int = 100_000 , ** kwargs : Any
385409) -> List [Any ]:
386410 """Split this document's records into context-window-sized chunks."""
387411 dataset_kwargs = {
388412 key : kwargs .pop (key )
389413 for key in list (kwargs )
390- if key in {
391- "chars_before" , "chars_after" , "include_thread" ,
392- "include_context" , "classify" , "anonymize" , "anonymizer" ,
393- }
414+ if key in _DATASET_OPTIONS
394415 }
395416 return _parser_to_llm_records (self , ** dataset_kwargs ).chunks (max_tokens , ** kwargs )
396417
0 commit comments