| skipped_count = 0 | skipped_count = 0 | ||||
| total_count = 0 | total_count = 0 | ||||
| vector_type = dify_config.VECTOR_STORE | vector_type = dify_config.VECTOR_STORE | ||||
| upper_colletion_vector_types = { | |||||
| VectorType.MILVUS, | |||||
| VectorType.PGVECTOR, | |||||
| VectorType.RELYT, | |||||
| VectorType.WEAVIATE, | |||||
| VectorType.ORACLE, | |||||
| VectorType.ELASTICSEARCH, | |||||
| } | |||||
| lower_colletion_vector_types = { | |||||
| VectorType.ANALYTICDB, | |||||
| VectorType.CHROMA, | |||||
| VectorType.MYSCALE, | |||||
| VectorType.PGVECTO_RS, | |||||
| VectorType.TIDB_VECTOR, | |||||
| VectorType.OPENSEARCH, | |||||
| VectorType.TENCENT, | |||||
| VectorType.BAIDU, | |||||
| VectorType.VIKINGDB, | |||||
| } | |||||
| page = 1 | page = 1 | ||||
| while True: | while True: | ||||
| try: | try: | ||||
| skipped_count = skipped_count + 1 | skipped_count = skipped_count + 1 | ||||
| continue | continue | ||||
| collection_name = "" | collection_name = "" | ||||
| if vector_type == VectorType.WEAVIATE: | |||||
| dataset_id = dataset.id | |||||
| dataset_id = dataset.id | |||||
| if vector_type in upper_colletion_vector_types: | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | collection_name = Dataset.gen_collection_name_by_id(dataset_id) | ||||
| index_struct_dict = {"type": VectorType.WEAVIATE, "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.QDRANT: | elif vector_type == VectorType.QDRANT: | ||||
| if dataset.collection_binding_id: | if dataset.collection_binding_id: | ||||
| dataset_collection_binding = ( | dataset_collection_binding = ( | ||||
| else: | else: | ||||
| raise ValueError("Dataset Collection Binding not found") | raise ValueError("Dataset Collection Binding not found") | ||||
| else: | else: | ||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | collection_name = Dataset.gen_collection_name_by_id(dataset_id) | ||||
| index_struct_dict = {"type": VectorType.QDRANT, "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.MILVUS: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = {"type": VectorType.MILVUS, "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.RELYT: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = {"type": "relyt", "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.TENCENT: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = {"type": VectorType.TENCENT, "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.PGVECTOR: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = {"type": VectorType.PGVECTOR, "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.OPENSEARCH: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = { | |||||
| "type": VectorType.OPENSEARCH, | |||||
| "vector_store": {"class_prefix": collection_name}, | |||||
| } | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.ANALYTICDB: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = { | |||||
| "type": VectorType.ANALYTICDB, | |||||
| "vector_store": {"class_prefix": collection_name}, | |||||
| } | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.ELASTICSEARCH: | |||||
| dataset_id = dataset.id | |||||
| index_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = {"type": "elasticsearch", "vector_store": {"class_prefix": index_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type == VectorType.BAIDU: | |||||
| dataset_id = dataset.id | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id) | |||||
| index_struct_dict = { | |||||
| "type": VectorType.BAIDU, | |||||
| "vector_store": {"class_prefix": collection_name}, | |||||
| } | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| elif vector_type in lower_colletion_vector_types: | |||||
| collection_name = Dataset.gen_collection_name_by_id(dataset_id).lower() | |||||
| else: | else: | ||||
| raise ValueError(f"Vector store {vector_type} is not supported.") | raise ValueError(f"Vector store {vector_type} is not supported.") | ||||
| index_struct_dict = {"type": vector_type, "vector_store": {"class_prefix": collection_name}} | |||||
| dataset.index_struct = json.dumps(index_struct_dict) | |||||
| vector = Vector(dataset) | vector = Vector(dataset) | ||||
| click.echo(f"Migrating dataset {dataset.id}.") | click.echo(f"Migrating dataset {dataset.id}.") | ||||