logo
Browse Source

Update with es

main
shiyu22 2 years ago
parent
commit
0ab6375eb6
  1. 4
      README.md
  2. 6
      osschat_insert.py

4
README.md

@ -2,7 +2,7 @@
## Description
**[OSSChat](https://osschat.io/)** is enhanced ChatGPT with documentation, issues, blog posts, community Q&A as knowledge bases. Built for every community and developer. The osschat-insert pipeline is a pipeline to insert data.
**[OSSChat](https://osschat.io/)** is enhanced ChatGPT with documentation, issues, blog posts, community Q&A as knowledge bases. Built for every community and developer. The osschat-insert pipeline is used to insert data.
<br />
@ -30,7 +30,7 @@ fields = [
FieldSchema(name='text', dtype=DataType.VARCHAR, description='text', max_length=1000),
FieldSchema(name='embedding', dtype=DataType.FLOAT_VECTOR, description='embedding vectors', dim=dim)
]
schema = CollectionSchema(fields=fields, description='enhanced qa')
schema = CollectionSchema(fields=fields, description='osschat')
collection = Collection(name=collection_name, schema=schema)
index_params = {

6
osschat_insert.py

@ -100,9 +100,9 @@ def osschat_insert_pipe(config):
.map('doc', 'text', ops.text_loader())
.flat_map('text', 'sentence', text_split_op)
.map('sentence', 'es_sentence', lambda x: {'sentence': x})
.map(('es_index', 'es_sentence'), es_index_op)
.map(('es_index', 'es_sentence'), 'es_res', es_index_op)
.map('sentence', 'embedding', sentence_embedding_op, config=sentence_embedding_config)
.map('embedding', 'embedding', ops.towhee.np_normalize())
.map(('milvus_collection', 'doc', 'sentence', 'embedding'), 'mr', insert_milvus_op)
.output('mr', 'es_res')
.map(('milvus_collection', 'doc', 'sentence', 'embedding'), 'milvus_res', insert_milvus_op)
.output('milvus_res', 'es_res')
)

Loading…
Cancel
Save