Count chunk

Signed-off-by: Jael Gu <mengjia.gu@zilliz.com>
2 years ago · 723998f09f
1 changed files with 1 additions and 1 deletions
--- a/osschat_insert.py
+++ b/osschat_insert.py
@ -98,8 +98,8 @@ def osschat_insert_pipe(config):
    p = (
        pipe.input('doc', 'project_name')
            .map('doc', 'text', data_loader)
-            .map('doc', 'token_count', ops.token_counter(config.token_model))
            .flat_map('text', 'chunk', text_split_op)
+            .map('chunk', 'token_count', ops.token_counter(config.token_model))
            .map('chunk', 'embedding', sentence_embedding_op, config=sentence_embedding_config)
    )
    if config.embedding_normalize: