| def gen(): | def gen(): | ||||
| for message in generator: | for message in generator: | ||||
| if isinstance(message, (Mapping, dict)): | |||||
| if isinstance(message, Mapping | dict): | |||||
| yield f"data: {json.dumps(message)}\n\n" | yield f"data: {json.dumps(message)}\n\n" | ||||
| else: | else: | ||||
| yield f"event: {message}\n\n" | yield f"event: {message}\n\n" |
| VariableValue = Union[str, int, float, dict, list, File] | VariableValue = Union[str, int, float, dict, list, File] | ||||
| VARIABLE_PATTERN = re.compile(r"\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\}\}") | VARIABLE_PATTERN = re.compile(r"\{\{#([a-zA-Z0-9_]{1,50}(?:\.[a-zA-Z_][a-zA-Z0-9_]{0,29}){1,10})#\}\}") | ||||
| if attr not in {item.value for item in FileAttribute}: | if attr not in {item.value for item in FileAttribute}: | ||||
| return None | return None | ||||
| value = self.get(selector) | value = self.get(selector) | ||||
| if not isinstance(value, (FileSegment, NoneSegment)): | |||||
| if not isinstance(value, FileSegment | NoneSegment): | |||||
| return None | return None | ||||
| if isinstance(value, FileSegment): | if isinstance(value, FileSegment): | ||||
| attr = FileAttribute(attr) | attr = FileAttribute(attr) |
| import logging | import logging | ||||
| from collections.abc import Generator, Mapping, Sequence | from collections.abc import Generator, Mapping, Sequence | ||||
| from datetime import datetime, timezone | |||||
| from datetime import UTC, datetime | |||||
| from typing import Any, cast | from typing import Any, cast | ||||
| from configs import dify_config | from configs import dify_config | ||||
| thread_pool_id=self.thread_pool_id, | thread_pool_id=self.thread_pool_id, | ||||
| ) | ) | ||||
| start_at = datetime.now(timezone.utc).replace(tzinfo=None) | |||||
| start_at = datetime.now(UTC).replace(tzinfo=None) | |||||
| condition_processor = ConditionProcessor() | condition_processor = ConditionProcessor() | ||||
| # Start Loop event | # Start Loop event |
| optional = true | optional = true | ||||
| [tool.poetry.group.lint.dependencies] | [tool.poetry.group.lint.dependencies] | ||||
| dotenv-linter = "~0.5.0" | dotenv-linter = "~0.5.0" | ||||
| ruff = "~0.9.9" | |||||
| ruff = "~0.11.0" |
| ).first() | ).first() | ||||
| if document: | if document: | ||||
| document.dataset_process_rule_id = dataset_process_rule.id # type: ignore | document.dataset_process_rule_id = dataset_process_rule.id # type: ignore | ||||
| document.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| document.created_from = created_from | document.created_from = created_from | ||||
| document.doc_form = knowledge_config.doc_form | document.doc_form = knowledge_config.doc_form | ||||
| document.doc_language = knowledge_config.doc_language | document.doc_language = knowledge_config.doc_language | ||||
| if cache_result is not None: | if cache_result is not None: | ||||
| continue | continue | ||||
| segment.enabled = False | segment.enabled = False | ||||
| segment.disabled_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| segment.disabled_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| segment.disabled_by = current_user.id | segment.disabled_by = current_user.id | ||||
| db.session.add(segment) | db.session.add(segment) | ||||
| real_deal_segmment_ids.append(segment.id) | real_deal_segmment_ids.append(segment.id) | ||||
| child_chunk.content = child_chunk_update_args.content | child_chunk.content = child_chunk_update_args.content | ||||
| child_chunk.word_count = len(child_chunk.content) | child_chunk.word_count = len(child_chunk.content) | ||||
| child_chunk.updated_by = current_user.id | child_chunk.updated_by = current_user.id | ||||
| child_chunk.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| child_chunk.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| child_chunk.type = "customized" | child_chunk.type = "customized" | ||||
| update_child_chunks.append(child_chunk) | update_child_chunks.append(child_chunk) | ||||
| else: | else: | ||||
| child_chunk.content = content | child_chunk.content = content | ||||
| child_chunk.word_count = len(content) | child_chunk.word_count = len(content) | ||||
| child_chunk.updated_by = current_user.id | child_chunk.updated_by = current_user.id | ||||
| child_chunk.updated_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| child_chunk.updated_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| child_chunk.type = "customized" | child_chunk.type = "customized" | ||||
| db.session.add(child_chunk) | db.session.add(child_chunk) | ||||
| VectorService.update_child_chunk_vector([], [child_chunk], [], dataset) | VectorService.update_child_chunk_vector([], [child_chunk], [], dataset) |
| if document: | if document: | ||||
| document.indexing_status = "error" | document.indexing_status = "error" | ||||
| document.error = str(e) | document.error = str(e) | ||||
| document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.stopped_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| return | return | ||||
| db.session.commit() | db.session.commit() | ||||
| document.indexing_status = "parsing" | document.indexing_status = "parsing" | ||||
| document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.processing_started_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| documents.append(document) | documents.append(document) | ||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() |
| { | { | ||||
| "error": str(e), | "error": str(e), | ||||
| "status": "error", | "status": "error", | ||||
| "disabled_at": datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None), | |||||
| "disabled_at": datetime.datetime.now(datetime.UTC).replace(tzinfo=None), | |||||
| "enabled": False, | "enabled": False, | ||||
| } | } | ||||
| ) | ) |
| if document: | if document: | ||||
| document.indexing_status = "error" | document.indexing_status = "error" | ||||
| document.error = str(e) | document.error = str(e) | ||||
| document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.stopped_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| redis_client.delete(retry_indexing_cache_key) | redis_client.delete(retry_indexing_cache_key) | ||||
| db.session.commit() | db.session.commit() | ||||
| document.indexing_status = "parsing" | document.indexing_status = "parsing" | ||||
| document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.processing_started_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| except Exception as ex: | except Exception as ex: | ||||
| document.indexing_status = "error" | document.indexing_status = "error" | ||||
| document.error = str(ex) | document.error = str(ex) | ||||
| document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.stopped_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| logging.info(click.style(str(ex), fg="yellow")) | logging.info(click.style(str(ex), fg="yellow")) |
| if document: | if document: | ||||
| document.indexing_status = "error" | document.indexing_status = "error" | ||||
| document.error = str(e) | document.error = str(e) | ||||
| document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.stopped_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| redis_client.delete(sync_indexing_cache_key) | redis_client.delete(sync_indexing_cache_key) | ||||
| db.session.commit() | db.session.commit() | ||||
| document.indexing_status = "parsing" | document.indexing_status = "parsing" | ||||
| document.processing_started_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.processing_started_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| except Exception as ex: | except Exception as ex: | ||||
| document.indexing_status = "error" | document.indexing_status = "error" | ||||
| document.error = str(ex) | document.error = str(ex) | ||||
| document.stopped_at = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) | |||||
| document.stopped_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None) | |||||
| db.session.add(document) | db.session.add(document) | ||||
| db.session.commit() | db.session.commit() | ||||
| logging.info(click.style(str(ex), fg="yellow")) | logging.info(click.style(str(ex), fg="yellow")) |