Skip to content

Commit 38a5987

Browse files
nicknick
authored andcommitted
v1.5.1 - fixing text anonymisation & enhancment of exisitng functionalities
1 parent 80f0e28 commit 38a5987

20 files changed

Lines changed: 5080 additions & 238 deletions

‎README.md‎

Lines changed: 526 additions & 46 deletions
Large diffs are not rendered by default.

‎pyproject.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ build-backend = "scikit_build_core.build"
77

88
[project]
99
name = "docx-comment-parser"
10-
version = "1.5.0"
10+
version = "1.5.1"
1111
description = "Fast C++ library for extracting comment metadata from .docx files"
1212
readme = "README.md"
1313
requires-python = ">=3.9"

‎setup.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ def _is_mingw() -> bool:
135135

136136
setup(
137137
name="docx-comment-parser",
138-
version="1.5.0",
138+
version="1.5.1",
139139
author="nick-developer",
140140
description="Fast C++ library for extracting comment metadata from .docx files",
141141
long_description=_read_long_description(),
@@ -156,9 +156,13 @@ def _is_mingw() -> bool:
156156
"polars": ["polars>=1.0"],
157157
"cli": ["typer>=0.12", "rich>=13.0"],
158158
"report": ["jinja2>=3.0"],
159+
# Kept in step with pyproject.toml: accelerated fuzzy matching for the
160+
# diff engine, and exact token counts for chunking and compression.
161+
"diff": ["rapidfuzz>=3.0"],
162+
"llm": ["tiktoken>=0.7"],
159163
"all": [
160164
"pandas>=2.0", "polars>=1.0", "typer>=0.12", "rich>=13.0",
161-
"jinja2>=3.0",
165+
"jinja2>=3.0", "rapidfuzz>=3.0", "tiktoken>=0.7",
162166
],
163167
},
164168
entry_points={

‎src/docx_comment_parser/__init__.py‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,15 @@
2525
records = parser.to_llm_dataset(anonymize="strict")
2626
parser.export_jsonl("review.jsonl")
2727
28+
and in 1.5.1 you choose how much of the document comes with each comment, fix
29+
the classifier's tags by hand, and spend fewer tokens doing it::
30+
31+
records = parser.to_llm_dataset(
32+
context={"unit": "sentences", "before": 2, "after": 1},
33+
overrides={4: {"priority": "blocker"}},
34+
compress="balanced",
35+
)
36+
2837
Export methods are attached to the compiled ``DocxParser`` / ``BatchParser``
2938
classes at import time. That is deliberate: it keeps a single parser object as
3039
the one thing a user needs to hold, rather than introducing a second wrapper
@@ -72,7 +81,7 @@
7281
from .privacy.policy import AnonymizationPolicy
7382
from .reporting.analytics import build_report_data
7483

75-
__version__ = "1.5.0"
84+
__version__ = "1.5.1"
7685

7786
__all__ = [
7887
# compiled core (unchanged public API)
@@ -319,11 +328,14 @@ def _parser_to_llm_records(
319328
classify: bool = True,
320329
anonymize: Optional[Any] = None,
321330
anonymizer: Optional[Any] = None,
331+
**kwargs: Any,
322332
) -> "LLMDataset":
323333
"""Return this document's comments as an :class:`LLMDataset`.
324334
325335
The object form of :meth:`to_llm_dataset`: same records, plus the
326-
anonymisation report and the vault that reverses it.
336+
anonymisation report and the vault that reverses it. Also takes
337+
``context=`` (sentences, paragraphs, words), ``overrides=`` (manual tags)
338+
and ``compress=`` (token reduction) — see ``llm.build_dataset``.
327339
"""
328340
from .llm.dataset import build_dataset
329341

@@ -336,6 +348,7 @@ def _parser_to_llm_records(
336348
classify=classify,
337349
anonymize=anonymize,
338350
anonymizer=anonymizer,
351+
**kwargs,
339352
)
340353

341354

@@ -380,17 +393,25 @@ def _parser_to_embeddings_input(
380393
)
381394

382395

396+
#: build_dataset keywords that to_llm_chunks forwards, as opposed to the
397+
#: chunking keywords it passes to Chunk packing.
398+
_DATASET_OPTIONS = frozenset(
399+
{
400+
"chars_before", "chars_after", "include_thread", "include_context",
401+
"classify", "anonymize", "anonymizer", "documents", "context",
402+
"overrides", "compress",
403+
}
404+
)
405+
406+
383407
def _parser_to_llm_chunks(
384408
self: DocxParser, max_tokens: int = 100_000, **kwargs: Any
385409
) -> List[Any]:
386410
"""Split this document's records into context-window-sized chunks."""
387411
dataset_kwargs = {
388412
key: kwargs.pop(key)
389413
for key in list(kwargs)
390-
if key in {
391-
"chars_before", "chars_after", "include_thread",
392-
"include_context", "classify", "anonymize", "anonymizer",
393-
}
414+
if key in _DATASET_OPTIONS
394415
}
395416
return _parser_to_llm_records(self, **dataset_kwargs).chunks(max_tokens, **kwargs)
396417

0 commit comments

Comments
 (0)