Browse Source
Update eqa insert
Signed-off-by: shiyu22 <shiyu.chen@zilliz.com>
main
1 changed files with
1 additions and
1 deletions
-
eqa_insert.py
|
|
@ -78,7 +78,7 @@ def enhanced_qa_insert_pipe(config): |
|
|
return ( |
|
|
return ( |
|
|
pipe.input('doc') |
|
|
pipe.input('doc') |
|
|
.map('doc', 'text', ops.text_loader()) |
|
|
.map('doc', 'text', ops.text_loader()) |
|
|
.flat_map('text', 'sentence', ops.text_spilter(chunk_size=config.chunk_size)) |
|
|
|
|
|
|
|
|
.flat_map('text', 'sentence', ops.text_spliter(chunk_size=config.chunk_size)) |
|
|
.map('sentence', 'embedding', sentence_embedding_op, config=sentence_embedding_config) |
|
|
.map('sentence', 'embedding', sentence_embedding_op, config=sentence_embedding_config) |
|
|
.map('embedding', 'embedding', ops.towhee.np_normalize()) |
|
|
.map('embedding', 'embedding', ops.towhee.np_normalize()) |
|
|
.map(('doc', 'sentence', 'embedding'), 'mr', insert_milvus_op) |
|
|
.map(('doc', 'sentence', 'embedding'), 'mr', insert_milvus_op) |
|
|
|
