|
|
@ -74,12 +74,6 @@ def osschat_insert_pipe(config): |
|
|
|
chunk_size=config.chunk_size, |
|
|
|
**config.splitter_kwargs) |
|
|
|
|
|
|
|
es_example_input = lambda x: { |
|
|
|
'title': x, |
|
|
|
'author': 'OSSChat', |
|
|
|
'content': x, |
|
|
|
'timestamp': datetime.now() |
|
|
|
} |
|
|
|
es_index_op = ops.elasticsearch.osschat_index(host=config.es_host, |
|
|
|
port=config.es_port, |
|
|
|
user=config.es_user, |
|
|
@ -105,7 +99,7 @@ def osschat_insert_pipe(config): |
|
|
|
pipe.input('doc', 'milvus_collection', 'es_index') |
|
|
|
.map('doc', 'text', ops.text_loader()) |
|
|
|
.flat_map('text', 'sentence', text_split_op) |
|
|
|
.map('sentence', 'es_sentence', es_example_input) |
|
|
|
.map('sentence', 'es_sentence', lambda x: {'sentence': x}) |
|
|
|
.map(('es_index', 'es_sentence'), es_index_op) |
|
|
|
.map('sentence', 'embedding', sentence_embedding_op, config=sentence_embedding_config) |
|
|
|
.map('embedding', 'embedding', ops.towhee.np_normalize()) |
|
|
|