Browse Source
Count chunk
Signed-off-by: Jael Gu <mengjia.gu@zilliz.com>
main
1 changed files with
1 additions and
1 deletions
-
osschat_insert.py
|
|
@ -98,8 +98,8 @@ def osschat_insert_pipe(config): |
|
|
p = ( |
|
|
p = ( |
|
|
pipe.input('doc', 'project_name') |
|
|
pipe.input('doc', 'project_name') |
|
|
.map('doc', 'text', data_loader) |
|
|
.map('doc', 'text', data_loader) |
|
|
.map('doc', 'token_count', ops.token_counter(config.token_model)) |
|
|
|
|
|
.flat_map('text', 'chunk', text_split_op) |
|
|
.flat_map('text', 'chunk', text_split_op) |
|
|
|
|
|
.map('chunk', 'token_count', ops.token_counter(config.token_model)) |
|
|
.map('chunk', 'embedding', sentence_embedding_op, config=sentence_embedding_config) |
|
|
.map('chunk', 'embedding', sentence_embedding_op, config=sentence_embedding_config) |
|
|
) |
|
|
) |
|
|
if config.embedding_normalize: |
|
|
if config.embedding_normalize: |
|
|
|
