fix(text): drop whitespace tokens from the Chinese word splitter (#2492)

jieba.cut keeps the spaces between words as tokens, so every zh heuristic
filter counted them as words, and SymbolsToWordsFilter divided by the
length of a stripped space and raised ZeroDivisionError on any document
with a space.

Closes #2491

Signed-off-by: usmanmateen <74776275+usmanmateen@users.noreply.github.com>
Co-authored-by: Sarah Yurick <53962159+sarahyurick@users.noreply.github.com>

Author: usmanmateen

nemo_curator/stages/text/utils/text_utils.py

@@ -44,7 +44,8 @@ def get_word_splitter(language: str) -> Callable[[str], list[str]]:
         import jieba
 
         def jieba_splitter(text: str) -> list[str]:
-            return list(jieba.cut(text))
+            # Jieba returns the whitespace between words as tokens, so drop them like str.split() does
+            return [word for word in jieba.cut(text) if word.strip()]
 
         return jieba_splitter
 

tests/stages/text/modules/test_filters.py

@@ -505,6 +505,18 @@ def test_symbolswords(self) -> None:
         )
         assert all_equal(expected_data, filtered_data), f"Expected {expected_data} but got {filtered_data}"
 
+    def test_symbolswords_zh_with_spaces(self) -> None:
+        dataset = list_to_dataset(["我用 Python 写代码。", "... ... 标题"])
+        filters = ScoreFilter(SymbolsToWordsFilter(lang="zh"))
+
+        filtered_data = filters.process(dataset)
+
+        expected_data = DocumentBatch(
+            data=pd.DataFrame({"text": ["我用 Python 写代码。"]}),
+            dataset_name="test_1",
+        )
+        assert all_equal(expected_data, filtered_data), f"Expected {expected_data} but got {filtered_data}"
+
     def test_numbers(self) -> None:
         dataset = list_to_dataset(["purely letters", "34134543", "$!@$@!$!@", "abcdefghi1"])
         filters = ScoreFilter(NumbersFilter(max_number_to_text_ratio=0.1))
@@ -728,6 +740,19 @@ def test_wordcount_zh(self) -> None:
         )
         assert all_equal(expected_data, filtered_data), f"Expected {expected_data} but got {filtered_data}"
 
+    def test_wordcount_zh_ignores_spaces(self) -> None:
+        # 5 words: 我用 / Python / 写 / 代码 / 。, the two spaces are not words
+        dataset = list_to_dataset(["我用 Python 写代码。"])
+        filters = ScoreFilter(WordCountFilter(min_words=2, max_words=5, lang="zh"))
+
+        filtered_data = filters.process(dataset)
+
+        expected_data = DocumentBatch(
+            data=pd.DataFrame({"text": ["我用 Python 写代码。"]}),
+            dataset_name="test_1",
+        )
+        assert all_equal(expected_data, filtered_data), f"Expected {expected_data} but got {filtered_data}"
+
     @pytest.mark.skip(reason="Skipping MeCab runtime error for now")
     def test_wordcount_ja(self) -> None:
         dataset = list_to_dataset(["", "猫が寝ます。", "私は日本語のテキストを分割します。"])