Browse Source
Update init
Signed-off-by: shiyu22 <shiyu.chen@zilliz.com>
main
1 changed files with
1 additions and
0 deletions
-
spliter.py
|
|
@ -6,6 +6,7 @@ from langchain.text_splitter import RecursiveCharacterTextSplitter |
|
|
class TextSpliter(PyOperator): |
|
|
class TextSpliter(PyOperator): |
|
|
'''Split data into a list.''' |
|
|
'''Split data into a list.''' |
|
|
def __init__(self, chunk_size: int = 300): |
|
|
def __init__(self, chunk_size: int = 300): |
|
|
|
|
|
super().__init__() |
|
|
self.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size) |
|
|
self.splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size) |
|
|
|
|
|
|
|
|
def __call__(self, data: str) -> List[str]: |
|
|
def __call__(self, data: str) -> List[str]: |
|
|
|
