From 723998f09f92eabe94249cffb0591521163c442d Mon Sep 17 00:00:00 2001 From: Jael Gu Date: Mon, 28 Aug 2023 17:46:43 +0800 Subject: [PATCH] Count chunk Signed-off-by: Jael Gu --- osschat_insert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/osschat_insert.py b/osschat_insert.py index 5e69af5..710033d 100644 --- a/osschat_insert.py +++ b/osschat_insert.py @@ -98,8 +98,8 @@ def osschat_insert_pipe(config): p = ( pipe.input('doc', 'project_name') .map('doc', 'text', data_loader) - .map('doc', 'token_count', ops.token_counter(config.token_model)) .flat_map('text', 'chunk', text_split_op) + .map('chunk', 'token_count', ops.token_counter(config.token_model)) .map('chunk', 'embedding', sentence_embedding_op, config=sentence_embedding_config) ) if config.embedding_normalize: