Przeglądaj źródła

Fix: filed_map was incorrectly persisted (#7443)

### What problem does this PR solve?

Fix `filed_map` was incorrectly persisted. #7412 

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.19.0
Yongteng Lei 6 miesięcy temu
rodzic
commit
f29a5de9f5
No account linked to committer's email address

+ 23
- 0
api/apps/document_app.py Wyświetl plik

pf_id = root_folder["id"] pf_id = root_folder["id"]
FileService.init_knowledgebase_docs(pf_id, current_user.id) FileService.init_knowledgebase_docs(pf_id, current_user.id)
errors = "" errors = ""
kb_table_num_map = {}
for doc_id in doc_ids: for doc_id in doc_ids:
try: try:
e, doc = DocumentService.get_by_id(doc_id) e, doc = DocumentService.get_by_id(doc_id)
File2DocumentService.delete_by_document_id(doc_id) File2DocumentService.delete_by_document_id(doc_id)
if deleted_file_count > 0: if deleted_file_count > 0:
STORAGE_IMPL.rm(b, n) STORAGE_IMPL.rm(b, n)

doc_parser = doc.parser_id
if doc_parser == ParserType.TABLE:
kb_id = doc.kb_id
if kb_id not in kb_table_num_map:
counts = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
kb_table_num_map[kb_id] = counts
kb_table_num_map[kb_id] -= 1
if kb_table_num_map[kb_id] <= 0:
KnowledgebaseService.delete_field_map(kb_id)
except Exception as e: except Exception as e:
errors += str(e) errors += str(e)


code=settings.RetCode.AUTHENTICATION_ERROR code=settings.RetCode.AUTHENTICATION_ERROR
) )
try: try:
kb_table_num_map = {}
for id in req["doc_ids"]: for id in req["doc_ids"]:
info = {"run": str(req["run"]), "progress": 0} info = {"run": str(req["run"]), "progress": 0}
if str(req["run"]) == TaskStatus.RUNNING.value and req.get("delete", False): if str(req["run"]) == TaskStatus.RUNNING.value and req.get("delete", False):
e, doc = DocumentService.get_by_id(id) e, doc = DocumentService.get_by_id(id)
doc = doc.to_dict() doc = doc.to_dict()
doc["tenant_id"] = tenant_id doc["tenant_id"] = tenant_id

doc_parser = doc.get("parser_id", ParserType.NAIVE)
if doc_parser == ParserType.TABLE:
kb_id = doc.get("kb_id")
if not kb_id:
continue
if kb_id not in kb_table_num_map:
count = DocumentService.count_by_kb_id(kb_id=kb_id, keywords="", run_status=[TaskStatus.DONE], types=[])
kb_table_num_map[kb_id] = count
if kb_table_num_map[kb_id] <=0:
KnowledgebaseService.delete_field_map(kb_id)
bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"]) bucket, name = File2DocumentService.get_storage_address(doc_id=doc["id"])
queue_tasks(doc, bucket, name, 0) queue_tasks(doc, bucket, name, 0)



+ 20
- 0
api/db/services/document_service.py Wyświetl plik



return list(docs.dicts()), count return list(docs.dicts()), count


@classmethod
@DB.connection_context()
def count_by_kb_id(cls, kb_id, keywords, run_status, types):
if keywords:
docs = cls.model.select().where(
(cls.model.kb_id == kb_id),
(fn.LOWER(cls.model.name).contains(keywords.lower()))
)
else:
docs = cls.model.select().where(cls.model.kb_id == kb_id)

if run_status:
docs = docs.where(cls.model.run.in_(run_status))
if types:
docs = docs.where(cls.model.type.in_(types))

count = docs.count()

return count

@classmethod @classmethod
@DB.connection_context() @DB.connection_context()
def insert(cls, doc): def insert(cls, doc):

+ 10
- 0
api/db/services/knowledgebase_service.py Wyświetl plik

dfs_update(m.parser_config, config) dfs_update(m.parser_config, config)
cls.update_by_id(id, {"parser_config": m.parser_config}) cls.update_by_id(id, {"parser_config": m.parser_config})


@classmethod
@DB.connection_context()
def delete_field_map(cls, id):
e, m = cls.get_by_id(id)
if not e:
raise LookupError(f"knowledgebase({id}) not found.")

m.parser_config.pop("field_map", None)
cls.update_by_id(id, {"parser_config": m.parser_config})

@classmethod @classmethod
@DB.connection_context() @DB.connection_context()
def get_field_map(cls, ids): def get_field_map(cls, ids):

Ładowanie…
Anuluj
Zapisz