Browse Source

deal dataset metadata and metadata binding (#17544)

tags/1.2.0
Jyong 6 months ago
parent
commit
e1439c4a5b
No account linked to committer's email address
2 changed files with 12 additions and 2 deletions
  1. 5
    1
      api/tasks/clean_dataset_task.py
  2. 7
    1
      api/tasks/clean_document_task.py

+ 5
- 1
api/tasks/clean_dataset_task.py View File

from models.dataset import ( from models.dataset import (
AppDatasetJoin, AppDatasetJoin,
Dataset, Dataset,
DatasetMetadata,
DatasetMetadataBinding,
DatasetProcessRule, DatasetProcessRule,
DatasetQuery, DatasetQuery,
Document, Document,
db.session.query(DatasetProcessRule).filter(DatasetProcessRule.dataset_id == dataset_id).delete() db.session.query(DatasetProcessRule).filter(DatasetProcessRule.dataset_id == dataset_id).delete()
db.session.query(DatasetQuery).filter(DatasetQuery.dataset_id == dataset_id).delete() db.session.query(DatasetQuery).filter(DatasetQuery.dataset_id == dataset_id).delete()
db.session.query(AppDatasetJoin).filter(AppDatasetJoin.dataset_id == dataset_id).delete() db.session.query(AppDatasetJoin).filter(AppDatasetJoin.dataset_id == dataset_id).delete()

# delete dataset metadata
db.session.query(DatasetMetadata).filter(DatasetMetadata.dataset_id == dataset_id).delete()
db.session.query(DatasetMetadataBinding).filter(DatasetMetadataBinding.dataset_id == dataset_id).delete()
# delete files # delete files
if documents: if documents:
for document in documents: for document in documents:

+ 7
- 1
api/tasks/clean_document_task.py View File

from core.tools.utils.rag_web_reader import get_image_upload_file_ids from core.tools.utils.rag_web_reader import get_image_upload_file_ids
from extensions.ext_database import db from extensions.ext_database import db
from extensions.ext_storage import storage from extensions.ext_storage import storage
from models.dataset import Dataset, DocumentSegment
from models.dataset import Dataset, DatasetMetadataBinding, DocumentSegment
from models.model import UploadFile from models.model import UploadFile




db.session.delete(file) db.session.delete(file)
db.session.commit() db.session.commit()


# delete dataset metadata binding
db.session.query(DatasetMetadataBinding).filter(
DatasetMetadataBinding.dataset_id == dataset_id,
DatasetMetadataBinding.document_id == document_id,
).delete()

end_at = time.perf_counter() end_at = time.perf_counter()
logging.info( logging.info(
click.style( click.style(

Loading…
Cancel
Save