docs: clarify Markdown summary threshold

Author: yaodong-shen

README.md

@@ -184,6 +184,9 @@ You can customize the processing with additional optional arguments (the structu
 --if-add-node-id        Add node ID (yes/no, default: yes)
 --if-add-node-summary   Add node summary (yes/no, default: yes)
 --if-add-doc-description Add doc description (yes/no, default: yes)
+--summary-token-threshold Markdown only: nodes below this token count copy
+                          their text as the summary; other nodes call the LLM
+                          (default: 200; set 0 to summarize every node)
 ```
 </details>
 

run_pageindex.py

@@ -49,7 +49,7 @@
     parser.add_argument('--thinning-threshold', type=int, default=5000,
                       help='Minimum token threshold for thinning (markdown only)')
     parser.add_argument('--summary-token-threshold', type=int, default=200,
-                      help='Token threshold for generating summaries (markdown only)')
+                      help='Markdown only: nodes below this token count copy text as their summary; other nodes call the LLM (default: 200)')
     args = parser.parse_args()
     if args.flash:
         args.mode = 'flash'

tests/test_page_index_md.py

@@ -1,6 +1,7 @@
 import unittest
+from unittest.mock import AsyncMock, patch
 
-from pageindex.page_index_md import extract_nodes_from_markdown
+from pageindex.page_index_md import extract_nodes_from_markdown, get_node_summary
 
 
 class ExtractNodesFromMarkdownTest(unittest.TestCase):
@@ -19,5 +20,31 @@ def test_skips_bold_heading_with_only_whitespace(self):
         )
 
 
+class GetNodeSummaryTest(unittest.IsolatedAsyncioTestCase):
+    async def test_below_threshold_uses_node_text_without_calling_llm(self):
+        node = {"text": "short section"}
+
+        with patch("pageindex.page_index_md.count_tokens", return_value=199), patch(
+            "pageindex.page_index_md.generate_node_summary", new_callable=AsyncMock
+        ) as generate_summary:
+            summary = await get_node_summary(node, summary_token_threshold=200)
+
+        self.assertEqual(summary, node["text"])
+        generate_summary.assert_not_awaited()
+
+    async def test_threshold_boundary_generates_a_summary(self):
+        node = {"text": "boundary section"}
+
+        with patch("pageindex.page_index_md.count_tokens", return_value=200), patch(
+            "pageindex.page_index_md.generate_node_summary",
+            new_callable=AsyncMock,
+            return_value="generated summary",
+        ) as generate_summary:
+            summary = await get_node_summary(node, summary_token_threshold=200)
+
+        self.assertEqual(summary, "generated summary")
+        generate_summary.assert_awaited_once_with(node, model=None)
+
+
 if __name__ == "__main__":
     unittest.main()