| if dataset_documents: | if dataset_documents: | ||||
| # save vector index | # save vector index | ||||
| index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=True) | index = IndexBuilder.get_index(dataset, 'high_quality', ignore_high_quality_check=True) | ||||
| documents = [] | |||||
| for dataset_document in dataset_documents: | for dataset_document in dataset_documents: | ||||
| # delete from vector index | # delete from vector index | ||||
| segments = db.session.query(DocumentSegment).filter( | segments = db.session.query(DocumentSegment).filter( | ||||
| DocumentSegment.document_id == dataset_document.id, | DocumentSegment.document_id == dataset_document.id, | ||||
| DocumentSegment.enabled == True | DocumentSegment.enabled == True | ||||
| ) .order_by(DocumentSegment.position.asc()).all() | ) .order_by(DocumentSegment.position.asc()).all() | ||||
| documents = [] | |||||
| for segment in segments: | for segment in segments: | ||||
| document = Document( | document = Document( | ||||
| page_content=segment.content, | page_content=segment.content, | ||||
| documents.append(document) | documents.append(document) | ||||
| # save vector index | |||||
| index.add_texts(documents) | |||||
| # save vector index | |||||
| index.add_texts(documents) | |||||
| end_at = time.perf_counter() | end_at = time.perf_counter() | ||||
| logging.info( | logging.info( |