### What problem does this PR solve? Fix typo process_duration. ### Type of change - [x] Documentation Update - [x] Refactoringtags/v0.20.0
| if "parser_config" in req: | if "parser_config" in req: | ||||
| DocumentService.update_parser_config(doc.id, req["parser_config"]) | DocumentService.update_parser_config(doc.id, req["parser_config"]) | ||||
| if doc.token_num > 0: | if doc.token_num > 0: | ||||
| e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, doc.process_duation * -1) | |||||
| e = DocumentService.increment_chunk_num(doc.id, doc.kb_id, doc.token_num * -1, doc.chunk_num * -1, doc.process_duration * -1) | |||||
| if not e: | if not e: | ||||
| return get_data_error_result(message="Document not found!") | return get_data_error_result(message="Document not found!") | ||||
| tenant_id = DocumentService.get_tenant_id(req["doc_id"]) | tenant_id = DocumentService.get_tenant_id(req["doc_id"]) |
| doc.kb_id, | doc.kb_id, | ||||
| doc.token_num * -1, | doc.token_num * -1, | ||||
| doc.chunk_num * -1, | doc.chunk_num * -1, | ||||
| doc.process_duation * -1, | |||||
| doc.process_duration * -1, | |||||
| ) | ) | ||||
| if not e: | if not e: | ||||
| return get_error_data_result(message="Document not found!") | return get_error_data_result(message="Document not found!") | ||||
| "parser_id": "chunk_method", | "parser_id": "chunk_method", | ||||
| } | } | ||||
| run_mapping = { | run_mapping = { | ||||
| "0": "UNSTART", | |||||
| "1": "RUNNING", | |||||
| "2": "CANCEL", | |||||
| "3": "DONE", | |||||
| "4": "FAIL", | |||||
| "0": "UNSTART", | |||||
| "1": "RUNNING", | |||||
| "2": "CANCEL", | |||||
| "3": "DONE", | |||||
| "4": "FAIL", | |||||
| } | } | ||||
| for doc in docs: | for doc in docs: | ||||
| renamed_doc = {} | renamed_doc = {} |
| progress = FloatField(default=0, index=True) | progress = FloatField(default=0, index=True) | ||||
| progress_msg = TextField(null=True, help_text="process message", default="") | progress_msg = TextField(null=True, help_text="process message", default="") | ||||
| process_begin_at = DateTimeField(null=True, index=True) | process_begin_at = DateTimeField(null=True, index=True) | ||||
| process_duation = FloatField(default=0) | |||||
| process_duration = FloatField(default=0) | |||||
| meta_fields = JSONField(null=True, default={}) | meta_fields = JSONField(null=True, default={}) | ||||
| run = CharField(max_length=1, null=True, help_text="start to run processing or cancel.(1: run it; 2: cancel)", default="0", index=True) | run = CharField(max_length=1, null=True, help_text="start to run processing or cancel.(1: run it; 2: cancel)", default="0", index=True) | ||||
| priority = IntegerField(default=0) | priority = IntegerField(default=0) | ||||
| begin_at = DateTimeField(null=True, index=True) | begin_at = DateTimeField(null=True, index=True) | ||||
| process_duation = FloatField(default=0) | |||||
| process_duration = FloatField(default=0) | |||||
| progress = FloatField(default=0, index=True) | progress = FloatField(default=0, index=True) | ||||
| progress_msg = TextField(null=True, help_text="process message", default="") | progress_msg = TextField(null=True, help_text="process message", default="") | ||||
| migrate(migrator.add_column("mcp_server", "variables", JSONField(null=True, help_text="MCP Server variables", default=dict))) | migrate(migrator.add_column("mcp_server", "variables", JSONField(null=True, help_text="MCP Server variables", default=dict))) | ||||
| except Exception: | except Exception: | ||||
| pass | pass | ||||
| try: | |||||
| migrate(migrator.rename_column("task", "process_duation", "process_duration")) | |||||
| except Exception: | |||||
| pass | |||||
| try: | |||||
| migrate(migrator.rename_column("document", "process_duation", "process_duration")) | |||||
| except Exception: | |||||
| pass |
| @classmethod | @classmethod | ||||
| @DB.connection_context() | @DB.connection_context() | ||||
| def increment_chunk_num(cls, doc_id, kb_id, token_num, chunk_num, duation): | |||||
| def increment_chunk_num(cls, doc_id, kb_id, token_num, chunk_num, duration): | |||||
| num = cls.model.update(token_num=cls.model.token_num + token_num, | num = cls.model.update(token_num=cls.model.token_num + token_num, | ||||
| chunk_num=cls.model.chunk_num + chunk_num, | chunk_num=cls.model.chunk_num + chunk_num, | ||||
| process_duation=cls.model.process_duation + duation).where( | |||||
| process_duration=cls.model.process_duration + duration).where( | |||||
| cls.model.id == doc_id).execute() | cls.model.id == doc_id).execute() | ||||
| if num == 0: | if num == 0: | ||||
| raise LookupError( | raise LookupError( | ||||
| @classmethod | @classmethod | ||||
| @DB.connection_context() | @DB.connection_context() | ||||
| def decrement_chunk_num(cls, doc_id, kb_id, token_num, chunk_num, duation): | |||||
| def decrement_chunk_num(cls, doc_id, kb_id, token_num, chunk_num, duration): | |||||
| num = cls.model.update(token_num=cls.model.token_num - token_num, | num = cls.model.update(token_num=cls.model.token_num - token_num, | ||||
| chunk_num=cls.model.chunk_num - chunk_num, | chunk_num=cls.model.chunk_num - chunk_num, | ||||
| process_duation=cls.model.process_duation + duation).where( | |||||
| process_duration=cls.model.process_duration + duration).where( | |||||
| cls.model.id == doc_id).execute() | cls.model.id == doc_id).execute() | ||||
| if num == 0: | if num == 0: | ||||
| raise LookupError( | raise LookupError( | ||||
| msg = "\n".join(sorted(msg)) | msg = "\n".join(sorted(msg)) | ||||
| info = { | info = { | ||||
| "process_duation": datetime.timestamp( | |||||
| "process_duration": datetime.timestamp( | |||||
| datetime.now()) - | datetime.now()) - | ||||
| d["process_begin_at"].timestamp(), | d["process_begin_at"].timestamp(), | ||||
| "run": status} | "run": status} |
| }, | }, | ||||
| "chunk_method": "naive", | "chunk_method": "naive", | ||||
| "process_begin_at": null, | "process_begin_at": null, | ||||
| "process_duation": 0.0, | |||||
| "process_duration": 0.0, | |||||
| "progress": 0.0, | "progress": 0.0, | ||||
| "progress_msg": "", | "progress_msg": "", | ||||
| "run": "0", | "run": "0", | ||||
| } | } | ||||
| }, | }, | ||||
| "process_begin_at": "Thu, 24 Oct 2024 09:56:44 GMT", | "process_begin_at": "Thu, 24 Oct 2024 09:56:44 GMT", | ||||
| "process_duation": 0.54213, | |||||
| "process_duration": 0.54213, | |||||
| "progress": 0.0, | "progress": 0.0, | ||||
| "progress_msg": "Task dispatched...", | "progress_msg": "Task dispatched...", | ||||
| "run": "2", | "run": "2", |
| - `progress`: `float` The current processing progress as a percentage. Defaults to `0.0`. | - `progress`: `float` The current processing progress as a percentage. Defaults to `0.0`. | ||||
| - `progress_msg`: `str` A message indicating the current progress status. Defaults to `""`. | - `progress_msg`: `str` A message indicating the current progress status. Defaults to `""`. | ||||
| - `process_begin_at`: `datetime` The start time of document processing. Defaults to `None`. | - `process_begin_at`: `datetime` The start time of document processing. Defaults to `None`. | ||||
| - `process_duation`: `float` Duration of the processing in seconds. Defaults to `0.0`. | |||||
| - `process_duration`: `float` Duration of the processing in seconds. Defaults to `0.0`. | |||||
| - `run`: `str` The document's processing status: | - `run`: `str` The document's processing status: | ||||
| - `"UNSTART"` (default) | - `"UNSTART"` (default) | ||||
| - `"RUNNING"` | - `"RUNNING"` |
| self.progress = 0.0 | self.progress = 0.0 | ||||
| self.progress_msg = "" | self.progress_msg = "" | ||||
| self.process_begin_at = None | self.process_begin_at = None | ||||
| self.process_duation = 0.0 | |||||
| self.process_duration = 0.0 | |||||
| self.run = "0" | self.run = "0" | ||||
| self.status = "1" | self.status = "1" | ||||
| for k in list(res_dict.keys()): | for k in list(res_dict.keys()): |
| doc = res["data"]["docs"][0] | doc = res["data"]["docs"][0] | ||||
| assert doc["run"] == "DONE" | assert doc["run"] == "DONE" | ||||
| assert len(doc["process_begin_at"]) > 0 | assert len(doc["process_begin_at"]) > 0 | ||||
| assert doc["process_duation"] > 0 | |||||
| assert doc["process_duration"] > 0 | |||||
| assert doc["progress"] > 0 | assert doc["progress"] > 0 | ||||
| assert "Task done" in doc["progress_msg"] | assert "Task done" in doc["progress_msg"] | ||||
| doc = res["data"]["docs"][0] | doc = res["data"]["docs"][0] | ||||
| assert doc["run"] == "DONE" | assert doc["run"] == "DONE" | ||||
| assert len(doc["process_begin_at"]) > 0 | assert len(doc["process_begin_at"]) > 0 | ||||
| assert doc["process_duation"] > 0 | |||||
| assert doc["process_duration"] > 0 | |||||
| assert doc["progress"] > 0 | assert doc["progress"] > 0 | ||||
| assert "Task done" in doc["progress_msg"] | assert "Task done" in doc["progress_msg"] | ||||
| marks=pytest.mark.skip(reason="issues/6104"), | marks=pytest.mark.skip(reason="issues/6104"), | ||||
| ), | ), | ||||
| pytest.param( | pytest.param( | ||||
| {"process_duation": 1.0}, | |||||
| {"process_duration": 1.0}, | |||||
| 102, | 102, | ||||
| "The input parameters are invalid.", | "The input parameters are invalid.", | ||||
| marks=pytest.mark.skip(reason="issues/6104"), | marks=pytest.mark.skip(reason="issues/6104"), |
| doc = res["data"]["docs"][0] | doc = res["data"]["docs"][0] | ||||
| assert doc["run"] == "DONE" | assert doc["run"] == "DONE" | ||||
| assert len(doc["process_begin_at"]) > 0 | assert len(doc["process_begin_at"]) > 0 | ||||
| assert doc["process_duation"] > 0 | |||||
| assert doc["process_duration"] > 0 | |||||
| assert doc["progress"] > 0 | assert doc["progress"] > 0 | ||||
| assert "Task done" in doc["progress_msg"] | assert "Task done" in doc["progress_msg"] | ||||
| doc = res["data"]["docs"][0] | doc = res["data"]["docs"][0] | ||||
| assert doc["run"] == "DONE" | assert doc["run"] == "DONE" | ||||
| assert len(doc["process_begin_at"]) > 0 | assert len(doc["process_begin_at"]) > 0 | ||||
| assert doc["process_duation"] > 0 | |||||
| assert doc["process_duration"] > 0 | |||||
| assert doc["progress"] > 0 | assert doc["progress"] > 0 | ||||
| assert "Task done" in doc["progress_msg"] | assert "Task done" in doc["progress_msg"] | ||||
| marks=pytest.mark.skip(reason="issues/6104"), | marks=pytest.mark.skip(reason="issues/6104"), | ||||
| ), | ), | ||||
| pytest.param( | pytest.param( | ||||
| {"process_duation": 1.0}, | |||||
| {"process_duration": 1.0}, | |||||
| 102, | 102, | ||||
| "The input parameters are invalid.", | "The input parameters are invalid.", | ||||
| marks=pytest.mark.skip(reason="issues/6104"), | marks=pytest.mark.skip(reason="issues/6104"), |
| if document.id in document_ids: | if document.id in document_ids: | ||||
| assert document.run == "DONE" | assert document.run == "DONE" | ||||
| assert len(document.process_begin_at) > 0 | assert len(document.process_begin_at) > 0 | ||||
| assert document.process_duation > 0 | |||||
| assert document.process_duration > 0 | |||||
| assert document.progress > 0 | assert document.progress > 0 | ||||
| assert "Task done" in document.progress_msg | assert "Task done" in document.progress_msg | ||||
| if document.id in document_ids: | if document.id in document_ids: | ||||
| assert document.run == "DONE" | assert document.run == "DONE" | ||||
| assert len(document.process_begin_at) > 0 | assert len(document.process_begin_at) > 0 | ||||
| assert document.process_duation > 0 | |||||
| assert document.process_duration > 0 | |||||
| assert document.progress > 0 | assert document.progress > 0 | ||||
| assert "Task done" in document.progress_msg | assert "Task done" in document.progress_msg | ||||
| marks=pytest.mark.skip(reason="issues/6104"), | marks=pytest.mark.skip(reason="issues/6104"), | ||||
| ), | ), | ||||
| pytest.param( | pytest.param( | ||||
| {"process_duation": 1.0}, | |||||
| {"process_duration": 1.0}, | |||||
| "The input parameters are invalid", | "The input parameters are invalid", | ||||
| marks=pytest.mark.skip(reason="issues/6104"), | marks=pytest.mark.skip(reason="issues/6104"), | ||||
| ), | ), |
| continue | continue | ||||
| assert doc["run"] == "3" | assert doc["run"] == "3" | ||||
| assert len(doc["process_begin_at"]) > 0 | assert len(doc["process_begin_at"]) > 0 | ||||
| assert doc["process_duation"] > 0 | |||||
| assert doc["process_duration"] > 0 | |||||
| assert doc["progress"] > 0 | assert doc["progress"] > 0 | ||||
| assert "Task done" in doc["progress_msg"] | assert "Task done" in doc["progress_msg"] | ||||
| parser_config: IParserConfig; | parser_config: IParserConfig; | ||||
| parser_id: string; | parser_id: string; | ||||
| process_begin_at?: string; | process_begin_at?: string; | ||||
| process_duation: number; | |||||
| process_duration: number; | |||||
| progress: number; | progress: number; | ||||
| progress_msg: string; | progress_msg: string; | ||||
| run: RunningStatus; | run: RunningStatus; |
| name: string; | name: string; | ||||
| parser_id: string; | parser_id: string; | ||||
| process_begin_at?: any; | process_begin_at?: any; | ||||
| process_duation: number; | |||||
| process_duration: number; | |||||
| progress: number; // parsing process | progress: number; // parsing process | ||||
| progress_msg: string; // parsing log | progress_msg: string; // parsing log | ||||
| run: RunningStatus; // parsing status | run: RunningStatus; // parsing status |
| children: record.process_begin_at, | children: record.process_begin_at, | ||||
| }, | }, | ||||
| { | { | ||||
| key: 'process_duation', | |||||
| key: 'process_duration', | |||||
| label: t('processDuration'), | label: t('processDuration'), | ||||
| children: `${record.process_duation.toFixed(2)} s`, | |||||
| children: `${record.process_duration.toFixed(2)} s`, | |||||
| }, | }, | ||||
| { | { | ||||
| key: 'progress_msg', | key: 'progress_msg', |
| children: record.process_begin_at, | children: record.process_begin_at, | ||||
| }, | }, | ||||
| { | { | ||||
| key: 'knowledgeDetails.process_duation', | |||||
| key: 'knowledgeDetails.process_duration', | |||||
| label: t('processDuration'), | label: t('processDuration'), | ||||
| children: `${record.process_duation.toFixed(2)} s`, | |||||
| children: `${record.process_duration.toFixed(2)} s`, | |||||
| }, | }, | ||||
| { | { | ||||
| key: 'progress_msg', | key: 'progress_msg', |