test: enable mypy typing checks for test/components/preprocessors

pyproject.toml

@@ -150,7 +150,7 @@ all = 'pytest {args:test}'
 e2e = 'pytest {args:e2e}'
 
 # TODO We want to eventually type the whole test folder
-types = "mypy --install-types --non-interactive --cache-dir=.mypy_cache/ {args:haystack test/core/ test/marshal/ test/testing/ test/tracing/ test/tools/ test/hooks/human_in_the_loop test/evaluation test/document_stores test/dataclasses test/utils/ test/fuzz/ test/skill_stores/ test/token_counters/ test/components/agents/test_state_class.py test/components/builders/ test/components/caching/ test/components/converters/test_utils.py test/components/embedders/ test/components/evaluators/ test/components/extractors/ test/components/generators/ test/components/joiners/ test/components/query/ test/components/rankers/ test/components/routers/ test/components/samplers/ test/components/validators/ test/components/writers/}"
+types = "mypy --install-types --non-interactive --cache-dir=.mypy_cache/ {args:haystack test/core/ test/marshal/ test/testing/ test/tracing/ test/tools/ test/hooks/human_in_the_loop test/evaluation test/document_stores test/dataclasses test/utils/ test/fuzz/ test/skill_stores/ test/token_counters/ test/components/agents/test_state_class.py test/components/builders/ test/components/caching/ test/components/converters/test_utils.py test/components/embedders/ test/components/evaluators/ test/components/extractors/ test/components/generators/ test/components/joiners/ test/components/preprocessors/ test/components/query/ test/components/rankers/ test/components/routers/ test/components/samplers/ test/components/validators/ test/components/writers/}"
 
 
 [project.urls]

test/components/preprocessors/test_csv_document_splitter.py

@@ -7,6 +7,7 @@
 
 import pytest
 from pandas import read_csv
+from pytest import LogCaptureFixture
 
 from haystack import Document
 from haystack.components.preprocessors.csv_document_splitter import CSVDocumentSplitter
@@ -62,14 +63,14 @@ class TestFindSplitIndices:
     def test_find_split_indices_row_two_tables(
         self, splitter: CSVDocumentSplitter, two_tables_sep_by_two_empty_rows: str
     ) -> None:
-        df = read_csv(StringIO(two_tables_sep_by_two_empty_rows), header=None, dtype=object)  # type: ignore
+        df = read_csv(StringIO(two_tables_sep_by_two_empty_rows), header=None, dtype=object)
         result = splitter._find_split_indices(df, split_threshold=2, axis="row")
         assert result == [(2, 3)]
 
     def test_find_split_indices_row_two_tables_with_empty_row(
         self, splitter: CSVDocumentSplitter, three_tables_sep_by_empty_rows: str
     ) -> None:
-        df = read_csv(StringIO(three_tables_sep_by_empty_rows), header=None, dtype=object)  # type: ignore
+        df = read_csv(StringIO(three_tables_sep_by_empty_rows), header=None, dtype=object)
         result = splitter._find_split_indices(df, split_threshold=2, axis="row")
         assert result == [(3, 4)]
 
@@ -84,7 +85,7 @@ def test_find_split_indices_row_three_tables(self, splitter: CSVDocumentSplitter
 ,,
 P,Q,R
 """
-        df = read_csv(StringIO(csv_content), header=None, dtype=object)  # type: ignore
+        df = read_csv(StringIO(csv_content), header=None, dtype=object)
         result = splitter._find_split_indices(df, split_threshold=2, axis="row")
         assert result == [(2, 3), (6, 7)]
 
@@ -108,7 +109,7 @@ def test_find_split_indices_returns_positional_indices_for_non_zero_index(
     def test_find_split_indices_column_two_tables(
         self, splitter: CSVDocumentSplitter, two_tables_sep_by_two_empty_columns: str
     ) -> None:
-        df = read_csv(StringIO(two_tables_sep_by_two_empty_columns), header=None, dtype=object)  # type: ignore
+        df = read_csv(StringIO(two_tables_sep_by_two_empty_columns), header=None, dtype=object)
         result = splitter._find_split_indices(df, split_threshold=1, axis="column")
         assert result == [(2, 3)]
 
@@ -117,7 +118,7 @@ def test_find_split_indices_column_two_tables_with_empty_column(self, splitter:
 1,,2,,,7,8
 3,,4,,,9,10
 """
-        df = read_csv(StringIO(csv_content), header=None, dtype=object)  # type: ignore
+        df = read_csv(StringIO(csv_content), header=None, dtype=object)
         result = splitter._find_split_indices(df, split_threshold=2, axis="column")
         assert result == [(3, 4)]
 
@@ -126,7 +127,7 @@ def test_find_split_indices_column_three_tables(self, splitter: CSVDocumentSplit
 1,2,,,7,8,,,11,12
 3,4,,,9,10,,,13,14
 """
-        df = read_csv(StringIO(csv_content), header=None, dtype=object)  # type: ignore
+        df = read_csv(StringIO(csv_content), header=None, dtype=object)
         result = splitter._find_split_indices(df, split_threshold=2, axis="column")
         assert result == [(2, 3), (6, 7)]
 
@@ -386,7 +387,7 @@ def test_split_by_row(self, csv_with_four_rows: str) -> None:
         assert result[1].content == "1,2,3\n"
         assert result[2].content == "X,Y,Z\n"
 
-    def test_split_by_row_with_empty_rows(self, caplog) -> None:
+    def test_split_by_row_with_empty_rows(self, caplog: LogCaptureFixture) -> None:
         splitter = CSVDocumentSplitter(split_mode="row-wise")
         doc = Document(content="")
         with caplog.at_level(logging.ERROR):
@@ -396,4 +397,4 @@ def test_split_by_row_with_empty_rows(self, caplog) -> None:
 
     def test_incorrect_split_mode(self) -> None:
         with pytest.raises(ValueError, match="not recognized"):
-            CSVDocumentSplitter(split_mode="incorrect_mode")
+            CSVDocumentSplitter(split_mode="incorrect_mode")  # type: ignore[arg-type]

test/components/preprocessors/test_document_cleaner.py

@@ -30,7 +30,7 @@ def test_non_text_document(self, caplog):
     def test_single_document(self):
         with pytest.raises(TypeError, match="DocumentCleaner expects a List of Documents as input."):
             cleaner = DocumentCleaner()
-            cleaner.run(documents=Document())
+            cleaner.run(documents=Document())  # type: ignore[arg-type]
 
     def test_empty_list(self):
         cleaner = DocumentCleaner()
@@ -127,6 +127,7 @@ def test_remove_repeated_substrings_preserves_unique_middle_page(self):
         )
         text = "PAGE ONE\fThe quick brown fox jumps high\fPAGE THREE"
         result = cleaner.run(documents=[Document(content=text)])["documents"][0]
+        assert result.content is not None
         assert result.content.split("\f")[1] == "The quick brown fox jumps high"
         # With no genuine repeated header/footer, all three pages must round-trip unchanged and in order.
         assert result.content.split("\f") == ["PAGE ONE", "The quick brown fox jumps high", "PAGE THREE"]

test/components/preprocessors/test_document_preprocessor.py

@@ -7,7 +7,9 @@
 import pytest
 
 from haystack import Document, Pipeline
+from haystack.components.preprocessors.document_cleaner import DocumentCleaner
 from haystack.components.preprocessors.document_preprocessor import DocumentPreprocessor
+from haystack.components.preprocessors.document_splitter import DocumentSplitter
 
 
 class TestDocumentPreprocessor:
@@ -33,12 +35,14 @@ def test_init(self, preprocessor: DocumentPreprocessor) -> None:
         assert preprocessor.output_mapping == {"cleaner.documents": "documents"}
 
         cleaner = preprocessor.pipeline.get_component("cleaner")
+        assert isinstance(cleaner, DocumentCleaner)
         assert cleaner.remove_empty_lines is True
         assert cleaner.remove_extra_whitespaces is True
         assert cleaner.remove_repeated_substrings is False
         assert cleaner.keep_id is True
 
         splitter = preprocessor.pipeline.get_component("splitter")
+        assert isinstance(splitter, DocumentSplitter)
         assert splitter.split_by == "word"
         assert splitter.split_length == 3
         assert splitter.split_overlap == 1
@@ -116,6 +120,7 @@ def test_run(self, preprocessor: DocumentPreprocessor) -> None:
 
         # Check that the content was cleaned and split
         for doc in processed_docs:
+            assert doc.content is not None
             assert doc.content.strip() == doc.content
             assert len(doc.content.split()) <= 3  # Split length of 3 words
             assert doc.id is not None
@@ -131,4 +136,6 @@ def custom_split(text: str) -> list[str]:
 
         processed_docs = result["documents"]
         assert len(processed_docs) == 3  # Should be split into 3 sentences
-        assert all("." not in doc.content for doc in processed_docs)  # Each doc should be a single sentence
+        for doc in processed_docs:
+            assert doc.content is not None
+            assert "." not in doc.content  # Each doc should be a single sentence

test/components/preprocessors/test_document_splitter.py

@@ -48,7 +48,7 @@ def test_non_text_document(self, caplog):
     def test_single_doc(self):
         with pytest.raises(TypeError, match="DocumentSplitter expects a List of Documents as input."):
             splitter = DocumentSplitter()
-            splitter.run(documents=Document())
+            splitter.run(documents=Document())  # type: ignore[arg-type]
 
     def test_empty_list(self):
         splitter = DocumentSplitter()
@@ -57,7 +57,7 @@ def test_empty_list(self):
 
     def test_unsupported_split_by(self):
         with pytest.raises(ValueError, match="split_by must be one of "):
-            DocumentSplitter(split_by="unsupported")
+            DocumentSplitter(split_by="unsupported")  # type: ignore[arg-type]
 
     def test_undefined_function(self):
         with pytest.raises(ValueError, match="When 'split_by' is set to 'function', a valid 'splitting_function'"):
@@ -117,6 +117,7 @@ def test_split_by_word_with_threshold_and_overlap_does_not_duplicate_overlap(sel
         result = splitter.run(documents=[Document(content=text)])
         contents = [doc.content for doc in result["documents"]]
         for content in contents:
+            assert content is not None
             assert content in text, f"chunk {content!r} is not present in the source text"
         assert contents == ["a b c ", "c d e f"]
 
@@ -251,6 +252,7 @@ def test_split_by_word_with_overlap(self):
         assert docs[0].meta["split_id"] == 0
         assert docs[0].meta["split_idx_start"] == text.index(docs[0].content)
         assert docs[0].meta["_split_overlap"][0]["range"] == (0, 5)
+        assert docs[1].content is not None
         assert docs[1].content[0:5] == "is a "
         # doc 1
         assert docs[1].content == "is a second sentence. And there is a third sentence."
@@ -409,6 +411,7 @@ def test_add_split_overlap_information(self):
         assert docs[0].meta["split_id"] == 0
         assert docs[0].meta["split_idx_start"] == text.index(docs[0].content)  # 0
         assert docs[0].meta["_split_overlap"][0]["range"] == (0, 23)
+        assert docs[1].content is not None
         assert docs[1].content[0:23] == "some words. There is a "
         # doc 1
         assert docs[1].content == "some words. There is a second sentence. And a third "
@@ -417,6 +420,7 @@ def test_add_split_overlap_information(self):
         assert docs[1].meta["_split_overlap"][0]["range"] == (20, 43)
         assert docs[1].meta["_split_overlap"][1]["range"] == (0, 29)
         assert docs[0].content[20:43] == "some words. There is a "
+        assert docs[2].content is not None
         assert docs[2].content[0:29] == "second sentence. And a third "
         # doc 2
         assert docs[2].content == "second sentence. And a third sentence."
@@ -747,7 +751,9 @@ def test_run_split_by_word_respect_sentence_boundary_no_repeats(self) -> None:
             documents[0].content
             == "This is a test sentence with many many words that exceeds the split length and should not be repeated. "
         )
+        assert documents[1].content is not None
         assert "This is a test sentence with many many words" not in documents[1].content
+        assert documents[2].content is not None
         assert "This is a test sentence with many many words" not in documents[2].content
 
     def test_run_split_by_word_respect_sentence_boundary_with_split_overlap_and_page_breaks(self) -> None:

test/components/preprocessors/test_embedding_based_document_splitter.py

@@ -59,7 +59,7 @@ def test_run_invalid_input(self):
         splitter.sentence_splitter = Mock()
 
         with pytest.raises(TypeError, match="expects a List of Documents"):
-            splitter.run(documents="not a list")
+            splitter.run(documents="not a list")  # type: ignore[arg-type]
 
     @pytest.mark.asyncio
     async def test_run_invalid_input_async(self) -> None:
@@ -68,7 +68,7 @@ async def test_run_invalid_input_async(self) -> None:
         splitter.sentence_splitter = AsyncMock()
 
         with pytest.raises(TypeError, match="expects a List of Documents"):
-            await splitter.run_async(documents="not a list")
+            await splitter.run_async(documents="not a list")  # type: ignore[arg-type]
 
     def test_run_document_with_none_content(self):
         mock_embedder = Mock()
@@ -185,7 +185,7 @@ def test_create_splits_from_points_no_points(self):
         splitter = EmbeddingBasedDocumentSplitter(document_embedder=mock_embedder)
 
         sentence_groups = ["Group 1 ", "Group 2 ", "Group 3"]
-        split_points = []
+        split_points: list[int] = []
 
         splits = splitter._create_splits_from_points(sentence_groups, split_points)
         assert splits == ["Group 1 Group 2 Group 3"]
@@ -313,6 +313,7 @@ def test_create_documents_from_splits_split_idx_start(self):
         assert documents[2].meta["split_idx_start"] == len("First chunk. ") + len("Second chunk. ")
         # Cross-check: split_idx_start correctly points into the original text
         for doc in documents:
+            assert doc.content is not None
             start = doc.meta["split_idx_start"]
             assert text[start : start + len(doc.content)] == doc.content
 
@@ -351,6 +352,7 @@ def mock_run(documents):
 
         # split_idx_start must point to the correct position in the original text
         for chunk in chunks:
+            assert chunk.content is not None
             start = chunk.meta["split_idx_start"]
             assert text[start : start + len(chunk.content)] == chunk.content
 
@@ -438,11 +440,14 @@ def test_split_document_with_multiple_topics(self):
         # There should be more than one split
         assert len(split_docs) > 1
         # Each split should be non-empty and respect min_length
+        split_contents: list[str] = []
         for split_doc in split_docs:
+            assert split_doc.content is not None
             assert split_doc.content.strip() != ""
             assert len(split_doc.content) >= 30
+            split_contents.append(split_doc.content)
         # The splits should cover the original text
-        combined = "".join([d.content for d in split_docs])
+        combined = "".join(split_contents)
         original = text
         assert combined in original or original in combined
 
@@ -474,11 +479,14 @@ async def test_split_document_with_multiple_topics_async(self) -> None:
         # There should be more than one split
         assert len(split_docs) > 1
         # Each split should be non-empty and respect min_length
+        split_contents: list[str] = []
         for split_doc in split_docs:
+            assert split_doc.content is not None
             assert split_doc.content.strip() != ""
             assert len(split_doc.content) >= 30
+            split_contents.append(split_doc.content)
         # The splits should cover the original text
-        combined = "".join([d.content for d in split_docs])
+        combined = "".join(split_contents)
         original = text
         assert combined in original or original in combined
 
@@ -612,12 +620,17 @@ def test_split_large_splits_recursion(self):
 
         assert len(split_docs) == 1
 
+        split_contents: list[str] = []
+        for split_doc in split_docs:
+            assert split_doc.content is not None
+            split_contents.append(split_doc.content)
+
         # If the chunk cannot be split further, it is allowed to be larger than max_length
         # At least one split should be larger than max_length in this test case
-        assert any(len(split_doc.content) > 1000 for split_doc in split_docs)
+        assert any(len(split_content) > 1000 for split_content in split_contents)
 
         # Verify that the splits cover the original content
-        combined_content = "".join([d.content for d in split_docs])
+        combined_content = "".join(split_contents)
         assert combined_content == text
 
         for i, split_doc in enumerate(split_docs):
@@ -653,12 +666,17 @@ async def test_split_large_splits_recursion_async(self) -> None:
 
         assert len(split_docs) == 1
 
+        split_contents: list[str] = []
+        for split_doc in split_docs:
+            assert split_doc.content is not None
+            split_contents.append(split_doc.content)
+
         # If the chunk cannot be split further, it is allowed to be larger than max_length
         # At least one split should be larger than max_length in this test case
-        assert any(len(split_doc.content) > 1000 for split_doc in split_docs)
+        assert any(len(split_content) > 1000 for split_content in split_contents)
 
         # Verify that the splits cover the original content
-        combined_content = "".join([d.content for d in split_docs])
+        combined_content = "".join(split_contents)
         assert combined_content == text
 
         for i, split_doc in enumerate(split_docs):
@@ -730,8 +748,13 @@ def test_split_large_splits_actually_splits(self):
 
         assert len(split_docs) == 11
 
+        split_contents: list[str] = []
+        for split_doc in split_docs:
+            assert split_doc.content is not None
+            split_contents.append(split_doc.content)
+
         # Verify that the splits cover the original content
-        combined_content = "".join([d.content for d in split_docs])
+        combined_content = "".join(split_contents)
         assert combined_content == text
 
         for i, split_doc in enumerate(split_docs):
@@ -813,8 +836,13 @@ async def test_split_large_splits_actually_splits_async(self) -> None:
 
         assert len(split_docs) == 11
 
+        split_contents: list[str] = []
+        for split_doc in split_docs:
+            assert split_doc.content is not None
+            split_contents.append(split_doc.content)
+
         # Verify that the splits cover the original content
-        combined_content = "".join([d.content for d in split_docs])
+        combined_content = "".join(split_contents)
         assert combined_content == text
 
         for i, split_doc in enumerate(split_docs):