logo
Browse Source

Update with es

main
shiyu22 2 years ago
parent
commit
14d8fb7a37
  1. 8
      osschat_insert.py

8
osschat_insert.py

@ -74,12 +74,6 @@ def osschat_insert_pipe(config):
chunk_size=config.chunk_size,
**config.splitter_kwargs)
es_example_input = lambda x: {
'title': x,
'author': 'OSSChat',
'content': x,
'timestamp': datetime.now()
}
es_index_op = ops.elasticsearch.osschat_index(host=config.es_host,
port=config.es_port,
user=config.es_user,
@ -105,7 +99,7 @@ def osschat_insert_pipe(config):
pipe.input('doc', 'milvus_collection', 'es_index')
.map('doc', 'text', ops.text_loader())
.flat_map('text', 'sentence', text_split_op)
.map('sentence', 'es_sentence', es_example_input)
.map('sentence', 'es_sentence', lambda x: {'sentence': x})
.map(('es_index', 'es_sentence'), es_index_op)
.map('sentence', 'embedding', sentence_embedding_op, config=sentence_embedding_config)
.map('embedding', 'embedding', ops.towhee.np_normalize())

Loading…
Cancel
Save