| @@ -339,26 +339,7 @@ def create_qdrant_indexes(): | |||
| ) | |||
| except Exception: | |||
| try: | |||
| embedding_model = model_manager.get_default_model_instance( | |||
| tenant_id=dataset.tenant_id, | |||
| model_type=ModelType.TEXT_EMBEDDING, | |||
| ) | |||
| dataset.embedding_model = embedding_model.model | |||
| dataset.embedding_model_provider = embedding_model.provider | |||
| except Exception: | |||
| provider = Provider( | |||
| id='provider_id', | |||
| tenant_id=dataset.tenant_id, | |||
| provider_name='openai', | |||
| provider_type=ProviderType.SYSTEM.value, | |||
| encrypted_config=json.dumps({'openai_api_key': 'TEST'}), | |||
| is_valid=True, | |||
| ) | |||
| model_provider = OpenAIProvider(provider=provider) | |||
| embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", | |||
| model_provider=model_provider) | |||
| continue | |||
| embeddings = CacheEmbedding(embedding_model) | |||
| from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex | |||
| @@ -405,7 +386,7 @@ def update_qdrant_indexes(): | |||
| .order_by(Dataset.created_at.desc()).paginate(page=page, per_page=50) | |||
| except NotFound: | |||
| break | |||
| model_manager = ModelManager() | |||
| page += 1 | |||
| for dataset in datasets: | |||
| if dataset.index_struct_dict: | |||
| @@ -413,23 +394,15 @@ def update_qdrant_indexes(): | |||
| try: | |||
| click.echo('Update dataset qdrant index: {}'.format(dataset.id)) | |||
| try: | |||
| embedding_model = ModelFactory.get_embedding_model( | |||
| embedding_model = model_manager.get_model_instance( | |||
| tenant_id=dataset.tenant_id, | |||
| model_provider_name=dataset.embedding_model_provider, | |||
| model_name=dataset.embedding_model | |||
| provider=dataset.embedding_model_provider, | |||
| model_type=ModelType.TEXT_EMBEDDING, | |||
| model=dataset.embedding_model | |||
| ) | |||
| except Exception: | |||
| provider = Provider( | |||
| id='provider_id', | |||
| tenant_id=dataset.tenant_id, | |||
| provider_name='openai', | |||
| provider_type=ProviderType.CUSTOM.value, | |||
| encrypted_config=json.dumps({'openai_api_key': 'TEST'}), | |||
| is_valid=True, | |||
| ) | |||
| model_provider = OpenAIProvider(provider=provider) | |||
| embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", | |||
| model_provider=model_provider) | |||
| continue | |||
| embeddings = CacheEmbedding(embedding_model) | |||
| from core.index.vector_index.qdrant_vector_index import QdrantConfig, QdrantVectorIndex | |||
| @@ -524,23 +497,17 @@ def deal_dataset_vector(flask_app: Flask, dataset: Dataset, normalization_count: | |||
| try: | |||
| click.echo('restore dataset index: {}'.format(dataset.id)) | |||
| try: | |||
| embedding_model = ModelFactory.get_embedding_model( | |||
| model_manager = ModelManager() | |||
| embedding_model = model_manager.get_model_instance( | |||
| tenant_id=dataset.tenant_id, | |||
| model_provider_name=dataset.embedding_model_provider, | |||
| model_name=dataset.embedding_model | |||
| provider=dataset.embedding_model_provider, | |||
| model_type=ModelType.TEXT_EMBEDDING, | |||
| model=dataset.embedding_model | |||
| ) | |||
| except Exception: | |||
| provider = Provider( | |||
| id='provider_id', | |||
| tenant_id=dataset.tenant_id, | |||
| provider_name='openai', | |||
| provider_type=ProviderType.CUSTOM.value, | |||
| encrypted_config=json.dumps({'openai_api_key': 'TEST'}), | |||
| is_valid=True, | |||
| ) | |||
| model_provider = OpenAIProvider(provider=provider) | |||
| embedding_model = OpenAIEmbedding(name="text-embedding-ada-002", | |||
| model_provider=model_provider) | |||
| pass | |||
| embeddings = CacheEmbedding(embedding_model) | |||
| dataset_collection_binding = db.session.query(DatasetCollectionBinding). \ | |||
| filter(DatasetCollectionBinding.provider_name == embedding_model.model_provider.provider_name, | |||