add built-in field check when doing old metadata migrate (#16371)

api/commands.py

@@ -12,6 +12,7 @@ from configs import dify_config
 from constants.languages import languages
 from core.rag.datasource.vdb.vector_factory import Vector
 from core.rag.datasource.vdb.vector_type import VectorType
+from core.rag.index_processor.constant.built_in_field import BuiltInField
 from core.rag.models.document import Document
 from events.app_event import app_was_created
 from extensions.ext_database import db
@@ -559,36 +560,25 @@ def old_metadata_migration():
             if document.doc_metadata:
                 doc_metadata = document.doc_metadata
                 for key, value in doc_metadata.items():
-                    dataset_metadata = (
-                        db.session.query(DatasetMetadata)
-                        .filter(DatasetMetadata.dataset_id == document.dataset_id, DatasetMetadata.name == key)
-                        .first()
-                    )
-                    if not dataset_metadata:
-                        dataset_metadata = DatasetMetadata(
-                            tenant_id=document.tenant_id,
-                            dataset_id=document.dataset_id,
-                            name=key,
-                            type="string",
-                            created_by=document.created_by,
-                        )
-                        db.session.add(dataset_metadata)
-                        db.session.flush()
-                        dataset_metadata_binding = DatasetMetadataBinding(
-                            tenant_id=document.tenant_id,
-                            dataset_id=document.dataset_id,
-                            metadata_id=dataset_metadata.id,
-                            document_id=document.id,
-                            created_by=document.created_by,
-                        )
-                        db.session.add(dataset_metadata_binding)
+                    for field in BuiltInField:
+                        if field.value == key:
+                            break
                     else:
-                        dataset_metadata_binding = DatasetMetadataBinding.query.filter(
-                            DatasetMetadataBinding.dataset_id == document.dataset_id,
-                            DatasetMetadataBinding.document_id == document.id,
-                            DatasetMetadataBinding.metadata_id == dataset_metadata.id,
-                        ).first()
-                        if not dataset_metadata_binding:
+                        dataset_metadata = (
+                            db.session.query(DatasetMetadata)
+                            .filter(DatasetMetadata.dataset_id == document.dataset_id, DatasetMetadata.name == key)
+                            .first()
+                        )
+                        if not dataset_metadata:
+                            dataset_metadata = DatasetMetadata(
+                                tenant_id=document.tenant_id,
+                                dataset_id=document.dataset_id,
+                                name=key,
+                                type="string",
+                                created_by=document.created_by,
+                            )
+                            db.session.add(dataset_metadata)
+                            db.session.flush()
                             dataset_metadata_binding = DatasetMetadataBinding(
                                 tenant_id=document.tenant_id,
                                 dataset_id=document.dataset_id,
@@ -597,7 +587,22 @@ def old_metadata_migration():
                                 created_by=document.created_by,
                             )
                             db.session.add(dataset_metadata_binding)
-                db.session.commit()
+                        else:
+                            dataset_metadata_binding = DatasetMetadataBinding.query.filter(
+                                DatasetMetadataBinding.dataset_id == document.dataset_id,
+                                DatasetMetadataBinding.document_id == document.id,
+                                DatasetMetadataBinding.metadata_id == dataset_metadata.id,
+                            ).first()
+                            if not dataset_metadata_binding:
+                                dataset_metadata_binding = DatasetMetadataBinding(
+                                    tenant_id=document.tenant_id,
+                                    dataset_id=document.dataset_id,
+                                    metadata_id=dataset_metadata.id,
+                                    document_id=document.id,
+                                    created_by=document.created_by,
+                                )
+                                db.session.add(dataset_metadata_binding)
+                        db.session.commit()
         page += 1
     click.echo(click.style("Old metadata migration completed.", fg="green"))