Understanding_Document_Pipeline_Langchain/text_splitters.py at main · MsheesAI/Understanding_Document_Pipeline_Langchain · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
#Length Based Text Splitter

from langchain_text_splitters import CharacterTextSplitter
from langchain_community.document_loaders import TextLoader

text = TextLoader("text.txt",encoding="utf-8")
docs = text.load()

splitter = CharacterTextSplitter(chunk_size=700, chunk_overlap=100)

result = splitter.split_documents(docs)

print(result[0].page_content)

#Text-Structure Based Text Splitter

from langchain_text_splitters import RecursiveCharacterTextSplitter
from langchain_community.document_loaders import TextLoader

text = TextLoader("text.txt",encoding="utf-8")
docs = text.load()

splitter = RecursiveCharacterTextSplitter(chunk_size=100, chunk_overlap=0,separators=["\n\n","\n",".","!","?"," ", ""])
result = splitter.split_documents(docs)
print(result[0].page_content)