|
|
|
@@ -339,26 +339,7 @@ def create_qdrant_indexes(): |
|
|
|
|
|
|
|
) |
|
|
|
except Exception: |
|
|
|
try: |
|
|
|
embedding_model = model_manager.get_default_model_instance( |
|
|
|
tenant_id=dataset.tenant_id, |
|
|
|
model_type=ModelType.TEXT_EMBEDDING, |
|
|
|
) |
|
|
|
dataset.embedding_model = embedding_model.model |
|
|
|
dataset.embedding_model_provider = embedding_model.provider |
|
|
|
except Exception: |
|
|
|
|
|
|
|
provider = Provider( |
|
|
|
id='provider_id', |
|
|
|
tenant_id=dataset.tenant_id, |
|
|
|
provider_name='openai', |
|
|
|
provider_type=ProviderType.SYSTEM.value, |
|
|
|
encrypted_config=json.dumps({'openai_api_key': 'TEST'}), |
|
|
|
is_valid=True, |
|
|
|
) |
|
|
|
model_provider = OpenAIProvider(provider=provider) |
|
|
|
embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", |
|
|
|
model_provider=model_provider) |
|
|
|
continue |
|
|
|
embeddings = CacheEmbedding(embedding_model) |
|
|
|
|
|
|
|
from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex |
|
|
|
@@ -405,7 +386,7 @@ def update_qdrant_indexes(): |
|
|
|
.order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50) |
|
|
|
except NotFound: |
|
|
|
break |
|
|
|
|
|
|
|
model_manager = ModelManager() |
|
|
|
page += 1 |
|
|
|
for dataset in datasets: |
|
|
|
if dataset.index_struct_dict: |
|
|
|
@@ -413,23 +394,15 @@ def update_qdrant_indexes(): |
|
|
|
try: |
|
|
|
click.echo('Update dataset qdrant index: {}'.format(dataset.id)) |
|
|
|
try: |
|
|
|
embedding_model = ModelFactory.get_embedding_model( |
|
|
|
embedding_model = model_manager.get_model_instance( |
|
|
|
tenant_id=dataset.tenant_id, |
|
|
|
model_provider_name=dataset.embedding_model_provider, |
|
|
|
model_name=dataset.embedding_model |
|
|
|
provider=dataset.embedding_model_provider, |
|
|
|
model_type=ModelType.TEXT_EMBEDDING, |
|
|
|
model=dataset.embedding_model |
|
|
|
|
|
|
|
) |
|
|
|
except Exception: |
|
|
|
provider = Provider( |
|
|
|
id='provider_id', |
|
|
|
tenant_id=dataset.tenant_id, |
|
|
|
provider_name='openai', |
|
|
|
provider_type=ProviderType.CUSTOM.value, |
|
|
|
encrypted_config=json.dumps({'openai_api_key': 'TEST'}), |
|
|
|
is_valid=True, |
|
|
|
) |
|
|
|
model_provider = OpenAIProvider(provider=provider) |
|
|
|
embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", |
|
|
|
model_provider=model_provider) |
|
|
|
continue |
|
|
|
embeddings = CacheEmbedding(embedding_model) |
|
|
|
|
|
|
|
from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex |
|
|
|
@@ -524,23 +497,17 @@ def deal_dataset_vector(flask_app: Flask, dataset: Dataset, normalization_count: |
|
|
|
try: |
|
|
|
click.echo('restore dataset index: {}'.format(dataset.id)) |
|
|
|
try: |
|
|
|
embedding_model = ModelFactory.get_embedding_model( |
|
|
|
model_manager = ModelManager() |
|
|
|
|
|
|
|
embedding_model = model_manager.get_model_instance( |
|
|
|
tenant_id=dataset.tenant_id, |
|
|
|
model_provider_name=dataset.embedding_model_provider, |
|
|
|
model_name=dataset.embedding_model |
|
|
|
provider=dataset.embedding_model_provider, |
|
|
|
model_type=ModelType.TEXT_EMBEDDING, |
|
|
|
model=dataset.embedding_model |
|
|
|
|
|
|
|
) |
|
|
|
except Exception: |
|
|
|
provider = Provider( |
|
|
|
id='provider_id', |
|
|
|
tenant_id=dataset.tenant_id, |
|
|
|
provider_name='openai', |
|
|
|
provider_type=ProviderType.CUSTOM.value, |
|
|
|
encrypted_config=json.dumps({'openai_api_key': 'TEST'}), |
|
|
|
is_valid=True, |
|
|
|
) |
|
|
|
model_provider = OpenAIProvider(provider=provider) |
|
|
|
embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", |
|
|
|
model_provider=model_provider) |
|
|
|
pass |
|
|
|
embeddings = CacheEmbedding(embedding_model) |
|
|
|
dataset_collection_binding = db.session.query(DatasetCollectionBinding). \ |
|
|
|
filter(DatasetCollectionBinding.provider_name == embedding_model.model_provider.provider_name, |