| dataset_id = kwargs.get("dataset_id") | dataset_id = kwargs.get("dataset_id") | ||||
| doc_form = kwargs.get("doc_form") | doc_form = kwargs.get("doc_form") | ||||
| file_id = kwargs.get("file_id") | file_id = kwargs.get("file_id") | ||||
| assert dataset_id is not None | |||||
| assert doc_form is not None | |||||
| clean_document_task.delay(document_id, dataset_id, doc_form, file_id) | clean_document_task.delay(document_id, dataset_id, doc_form, file_id) |
| from typing import Any, Optional | from typing import Any, Optional | ||||
| import pytz | import pytz | ||||
| from celery import Celery, Task # type: ignore | |||||
| from celery.schedules import crontab # type: ignore | |||||
| from celery import Celery, Task | |||||
| from celery.schedules import crontab | |||||
| from configs import dify_config | from configs import dify_config | ||||
| from dify_app import DifyApp | from dify_app import DifyApp | ||||
| task_cls=FlaskTask, | task_cls=FlaskTask, | ||||
| broker=dify_config.CELERY_BROKER_URL, | broker=dify_config.CELERY_BROKER_URL, | ||||
| backend=dify_config.CELERY_BACKEND, | backend=dify_config.CELERY_BACKEND, | ||||
| task_ignore_result=True, | |||||
| ) | ) | ||||
| celery_app.conf.update( | celery_app.conf.update( | ||||
| worker_task_log_format=dify_config.LOG_FORMAT, | worker_task_log_format=dify_config.LOG_FORMAT, | ||||
| worker_hijack_root_logger=False, | worker_hijack_root_logger=False, | ||||
| timezone=pytz.timezone(dify_config.LOG_TZ or "UTC"), | timezone=pytz.timezone(dify_config.LOG_TZ or "UTC"), | ||||
| task_ignore_result=True, | |||||
| ) | ) | ||||
| # Apply SSL configuration if enabled | # Apply SSL configuration if enabled |
| from typing import Union | from typing import Union | ||||
| import flask | import flask | ||||
| from celery.signals import worker_init # type: ignore | |||||
| from celery.signals import worker_init | |||||
| from flask_login import user_loaded_from_request, user_logged_in # type: ignore | from flask_login import user_loaded_from_request, user_logged_in # type: ignore | ||||
| from configs import dify_config | from configs import dify_config |
| from typing import Optional | from typing import Optional | ||||
| import sqlalchemy as sa | import sqlalchemy as sa | ||||
| from celery import states # type: ignore | |||||
| from celery import states | |||||
| from sqlalchemy import DateTime, String | from sqlalchemy import DateTime, String | ||||
| from sqlalchemy.orm import Mapped, mapped_column | from sqlalchemy.orm import Mapped, mapped_column | ||||
| "dotenv-linter~=0.5.0", | "dotenv-linter~=0.5.0", | ||||
| "faker~=32.1.0", | "faker~=32.1.0", | ||||
| "lxml-stubs~=0.5.1", | "lxml-stubs~=0.5.1", | ||||
| "mypy~=1.16.0", | |||||
| "mypy~=1.17.1", | |||||
| "ruff~=0.12.3", | "ruff~=0.12.3", | ||||
| "pytest~=8.3.2", | "pytest~=8.3.2", | ||||
| "pytest-benchmark~=4.0.0", | "pytest-benchmark~=4.0.0", | ||||
| "scipy-stubs>=1.15.3.0", | "scipy-stubs>=1.15.3.0", | ||||
| "types-python-http-client>=3.3.7.20240910", | "types-python-http-client>=3.3.7.20240910", | ||||
| "types-redis>=4.6.0.20241004", | "types-redis>=4.6.0.20241004", | ||||
| "celery-types>=0.23.0", | |||||
| ] | ] | ||||
| ############################################################ | ############################################################ |
| cls, | cls, | ||||
| account: Optional[Account] = None, | account: Optional[Account] = None, | ||||
| email: Optional[str] = None, | email: Optional[str] = None, | ||||
| language: Optional[str] = "en-US", | |||||
| language: str = "en-US", | |||||
| ): | ): | ||||
| account_email = account.email if account else email | account_email = account.email if account else email | ||||
| if account_email is None: | if account_email is None: | ||||
| account: Optional[Account] = None, | account: Optional[Account] = None, | ||||
| email: Optional[str] = None, | email: Optional[str] = None, | ||||
| old_email: Optional[str] = None, | old_email: Optional[str] = None, | ||||
| language: Optional[str] = "en-US", | |||||
| language: str = "en-US", | |||||
| phase: Optional[str] = None, | phase: Optional[str] = None, | ||||
| ): | ): | ||||
| account_email = account.email if account else email | account_email = account.email if account else email | ||||
| if account_email is None: | if account_email is None: | ||||
| raise ValueError("Email must be provided.") | raise ValueError("Email must be provided.") | ||||
| if not phase: | |||||
| raise ValueError("phase must be provided.") | |||||
| if cls.change_email_rate_limiter.is_rate_limited(account_email): | if cls.change_email_rate_limiter.is_rate_limited(account_email): | ||||
| from controllers.console.auth.error import EmailChangeRateLimitExceededError | from controllers.console.auth.error import EmailChangeRateLimitExceededError | ||||
| cls, | cls, | ||||
| account: Optional[Account] = None, | account: Optional[Account] = None, | ||||
| email: Optional[str] = None, | email: Optional[str] = None, | ||||
| language: Optional[str] = "en-US", | |||||
| language: str = "en-US", | |||||
| ): | ): | ||||
| account_email = account.email if account else email | account_email = account.email if account else email | ||||
| if account_email is None: | if account_email is None: | ||||
| cls, | cls, | ||||
| account: Optional[Account] = None, | account: Optional[Account] = None, | ||||
| email: Optional[str] = None, | email: Optional[str] = None, | ||||
| language: Optional[str] = "en-US", | |||||
| language: str = "en-US", | |||||
| workspace_name: Optional[str] = "", | workspace_name: Optional[str] = "", | ||||
| ): | ): | ||||
| account_email = account.email if account else email | account_email = account.email if account else email | ||||
| raise OwnerTransferRateLimitExceededError() | raise OwnerTransferRateLimitExceededError() | ||||
| code, token = cls.generate_owner_transfer_token(account_email, account) | code, token = cls.generate_owner_transfer_token(account_email, account) | ||||
| workspace_name = workspace_name or "" | |||||
| send_owner_transfer_confirm_task.delay( | send_owner_transfer_confirm_task.delay( | ||||
| language=language, | language=language, | ||||
| cls, | cls, | ||||
| account: Optional[Account] = None, | account: Optional[Account] = None, | ||||
| email: Optional[str] = None, | email: Optional[str] = None, | ||||
| language: Optional[str] = "en-US", | |||||
| language: str = "en-US", | |||||
| workspace_name: Optional[str] = "", | workspace_name: Optional[str] = "", | ||||
| new_owner_email: Optional[str] = "", | |||||
| new_owner_email: str = "", | |||||
| ): | ): | ||||
| account_email = account.email if account else email | account_email = account.email if account else email | ||||
| if account_email is None: | if account_email is None: | ||||
| raise ValueError("Email must be provided.") | raise ValueError("Email must be provided.") | ||||
| workspace_name = workspace_name or "" | |||||
| send_old_owner_transfer_notify_email_task.delay( | send_old_owner_transfer_notify_email_task.delay( | ||||
| language=language, | language=language, | ||||
| cls, | cls, | ||||
| account: Optional[Account] = None, | account: Optional[Account] = None, | ||||
| email: Optional[str] = None, | email: Optional[str] = None, | ||||
| language: Optional[str] = "en-US", | |||||
| language: str = "en-US", | |||||
| workspace_name: Optional[str] = "", | workspace_name: Optional[str] = "", | ||||
| ): | ): | ||||
| account_email = account.email if account else email | account_email = account.email if account else email | ||||
| if account_email is None: | if account_email is None: | ||||
| raise ValueError("Email must be provided.") | raise ValueError("Email must be provided.") | ||||
| workspace_name = workspace_name or "" | |||||
| send_new_owner_transfer_notify_email_task.delay( | send_new_owner_transfer_notify_email_task.delay( | ||||
| language=language, | language=language, | ||||
| @classmethod | @classmethod | ||||
| def send_email_code_login_email( | def send_email_code_login_email( | ||||
| cls, account: Optional[Account] = None, email: Optional[str] = None, language: Optional[str] = "en-US" | |||||
| cls, | |||||
| account: Optional[Account] = None, | |||||
| email: Optional[str] = None, | |||||
| language: str = "en-US", | |||||
| ): | ): | ||||
| email = account.email if account else email | email = account.email if account else email | ||||
| if email is None: | if email is None: | ||||
| raise AccountAlreadyInTenantError("Account already in tenant.") | raise AccountAlreadyInTenantError("Account already in tenant.") | ||||
| token = cls.generate_invite_token(tenant, account) | token = cls.generate_invite_token(tenant, account) | ||||
| language = account.interface_language or "en-US" | |||||
| # send email | # send email | ||||
| send_invite_member_mail_task.delay( | send_invite_member_mail_task.delay( | ||||
| language=account.interface_language, | |||||
| language=language, | |||||
| to=email, | to=email, | ||||
| token=token, | token=token, | ||||
| inviter_name=inviter.name if inviter else "Dify", | inviter_name=inviter.name if inviter else "Dify", |
| @classmethod | @classmethod | ||||
| def send_email_code_login_email( | def send_email_code_login_email( | ||||
| cls, account: Optional[Account] = None, email: Optional[str] = None, language: Optional[str] = "en-US" | |||||
| cls, account: Optional[Account] = None, email: Optional[str] = None, language: str = "en-US" | |||||
| ): | ): | ||||
| email = account.email if account else email | email = account.email if account else email | ||||
| if email is None: | if email is None: |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.constant.index_type import IndexType | from core.rag.index_processor.constant.index_type import IndexType | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.datasource.vdb.vector_factory import Vector | from core.rag.datasource.vdb.vector_factory import Vector | ||||
| from core.rag.models.document import Document | from core.rag.models.document import Document |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from werkzeug.exceptions import NotFound | from werkzeug.exceptions import NotFound | ||||
| from core.rag.datasource.vdb.vector_factory import Vector | from core.rag.datasource.vdb.vector_factory import Vector |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.datasource.vdb.vector_factory import Vector | from core.rag.datasource.vdb.vector_factory import Vector | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.datasource.vdb.vector_factory import Vector | from core.rag.datasource.vdb.vector_factory import Vector | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.datasource.vdb.vector_factory import Vector | from core.rag.datasource.vdb.vector_factory import Vector | ||||
| from core.rag.models.document import Document | from core.rag.models.document import Document |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.datasource.vdb.vector_factory import Vector | from core.rag.datasource.vdb.vector_factory import Vector | ||||
| from core.rag.models.document import Document | from core.rag.models.document import Document |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from core.tools.utils.web_reader_tool import get_image_upload_file_ids | from core.tools.utils.web_reader_tool import get_image_upload_file_ids |
| import click | import click | ||||
| import pandas as pd | import pandas as pd | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from sqlalchemy import func | from sqlalchemy import func | ||||
| from sqlalchemy.orm import Session | from sqlalchemy.orm import Session | ||||
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from core.tools.utils.web_reader_tool import get_image_upload_file_ids | from core.tools.utils.web_reader_tool import get_image_upload_file_ids |
| from typing import Optional | from typing import Optional | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from core.tools.utils.web_reader_tool import get_image_upload_file_ids | from core.tools.utils.web_reader_tool import get_image_upload_file_ids |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| from typing import Optional | from typing import Optional | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from core.rag.models.document import Document | from core.rag.models.document import Document |
| from typing import Literal | from typing import Literal | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.constant.index_type import IndexType | from core.rag.index_processor.constant.index_type import IndexType | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import logging | import logging | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from extensions.ext_database import db | from extensions.ext_database import db | ||||
| from models.account import Account | from models.account import Account |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.indexing_runner import DocumentIsPausedError, IndexingRunner | from core.indexing_runner import DocumentIsPausedError, IndexingRunner | ||||
| from core.rag.extractor.notion_extractor import NotionExtractor | from core.rag.extractor.notion_extractor import NotionExtractor |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from configs import dify_config | from configs import dify_config | ||||
| from core.indexing_runner import DocumentIsPausedError, IndexingRunner | from core.indexing_runner import DocumentIsPausedError, IndexingRunner |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.indexing_runner import DocumentIsPausedError, IndexingRunner | from core.indexing_runner import DocumentIsPausedError, IndexingRunner | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from configs import dify_config | from configs import dify_config | ||||
| from core.indexing_runner import DocumentIsPausedError, IndexingRunner | from core.indexing_runner import DocumentIsPausedError, IndexingRunner |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.constant.index_type import IndexType | from core.rag.index_processor.constant.index_type import IndexType | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.constant.index_type import IndexType | from core.rag.index_processor.constant.index_type import IndexType | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail | ||||
| from libs.email_i18n import EmailType, get_email_i18n_service | from libs.email_i18n import EmailType, get_email_i18n_service |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail | ||||
| from libs.email_i18n import EmailType, get_email_i18n_service | from libs.email_i18n import EmailType, get_email_i18n_service |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail | ||||
| from libs.email_i18n import EmailType, get_email_i18n_service | from libs.email_i18n import EmailType, get_email_i18n_service |
| from collections.abc import Mapping | from collections.abc import Mapping | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from flask import render_template_string | from flask import render_template_string | ||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from configs import dify_config | from configs import dify_config | ||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail | ||||
| from libs.email_i18n import EmailType, get_email_i18n_service | from libs.email_i18n import EmailType, get_email_i18n_service |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from extensions.ext_mail import mail | from extensions.ext_mail import mail | ||||
| from libs.email_i18n import EmailType, get_email_i18n_service | from libs.email_i18n import EmailType, get_email_i18n_service |
| import json | import json | ||||
| import logging | import logging | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from flask import current_app | from flask import current_app | ||||
| from core.ops.entities.config_entity import OPS_FILE_PATH, OPS_TRACE_FAILED_KEY | from core.ops.entities.config_entity import OPS_FILE_PATH, OPS_TRACE_FAILED_KEY |
| import typing | import typing | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.helper import marketplace | from core.helper import marketplace | ||||
| from core.helper.marketplace import MarketplacePluginDeclaration | from core.helper.marketplace import MarketplacePluginDeclaration |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.indexing_runner import DocumentIsPausedError, IndexingRunner | from core.indexing_runner import DocumentIsPausedError, IndexingRunner | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import click | import click | ||||
| import sqlalchemy as sa | import sqlalchemy as sa | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from sqlalchemy import delete | from sqlalchemy import delete | ||||
| from sqlalchemy.exc import SQLAlchemyError | from sqlalchemy.exc import SQLAlchemyError | ||||
| from sqlalchemy.orm import sessionmaker | from sqlalchemy.orm import sessionmaker | ||||
| with db.engine.begin() as conn: | with db.engine.begin() as conn: | ||||
| # Get a batch of draft variable IDs | # Get a batch of draft variable IDs | ||||
| query_sql = """ | query_sql = """ | ||||
| SELECT id FROM workflow_draft_variables | |||||
| WHERE app_id = :app_id | |||||
| SELECT id FROM workflow_draft_variables | |||||
| WHERE app_id = :app_id | |||||
| LIMIT :batch_size | LIMIT :batch_size | ||||
| """ | """ | ||||
| result = conn.execute(sa.text(query_sql), {"app_id": app_id, "batch_size": batch_size}) | result = conn.execute(sa.text(query_sql), {"app_id": app_id, "batch_size": batch_size}) | ||||
| # Delete the batch | # Delete the batch | ||||
| delete_sql = """ | delete_sql = """ | ||||
| DELETE FROM workflow_draft_variables | |||||
| DELETE FROM workflow_draft_variables | |||||
| WHERE id IN :ids | WHERE id IN :ids | ||||
| """ | """ | ||||
| deleted_result = conn.execute(sa.text(delete_sql), {"ids": tuple(draft_var_ids)}) | deleted_result = conn.execute(sa.text(delete_sql), {"ids": tuple(draft_var_ids)}) |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | ||||
| from extensions.ext_database import db | from extensions.ext_database import db |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.indexing_runner import IndexingRunner | from core.indexing_runner import IndexingRunner | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import time | import time | ||||
| import click | import click | ||||
| from celery import shared_task # type: ignore | |||||
| from celery import shared_task | |||||
| from core.indexing_runner import IndexingRunner | from core.indexing_runner import IndexingRunner | ||||
| from core.rag.index_processor.index_processor_factory import IndexProcessorFactory | from core.rag.index_processor.index_processor_factory import IndexProcessorFactory |
| import json | import json | ||||
| import logging | import logging | ||||
| from celery import shared_task # type: ignore[import-untyped] | |||||
| from celery import shared_task | |||||
| from sqlalchemy import select | from sqlalchemy import select | ||||
| from sqlalchemy.orm import sessionmaker | from sqlalchemy.orm import sessionmaker | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| from celery import shared_task # type: ignore[import-untyped] | |||||
| from celery import shared_task | |||||
| from sqlalchemy import select | from sqlalchemy import select | ||||
| from sqlalchemy.orm import sessionmaker | from sqlalchemy.orm import sessionmaker | ||||
| { url = "https://files.pythonhosted.org/packages/c9/af/0dcccc7fdcdf170f9a1585e5e96b6fb0ba1749ef6be8c89a6202284759bd/celery-5.5.3-py3-none-any.whl", hash = "sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525", size = 438775, upload-time = "2025-06-01T11:08:09.94Z" }, | { url = "https://files.pythonhosted.org/packages/c9/af/0dcccc7fdcdf170f9a1585e5e96b6fb0ba1749ef6be8c89a6202284759bd/celery-5.5.3-py3-none-any.whl", hash = "sha256:0b5761a07057acee94694464ca482416b959568904c9dfa41ce8413a7d65d525", size = 438775, upload-time = "2025-06-01T11:08:09.94Z" }, | ||||
| ] | ] | ||||
| [[package]] | |||||
| name = "celery-types" | |||||
| version = "0.23.0" | |||||
| source = { registry = "https://pypi.org/simple" } | |||||
| dependencies = [ | |||||
| { name = "typing-extensions" }, | |||||
| ] | |||||
| sdist = { url = "https://files.pythonhosted.org/packages/e9/d1/0823e71c281e4ad0044e278cf1577d1a68e05f2809424bf94e1614925c5d/celery_types-0.23.0.tar.gz", hash = "sha256:402ed0555aea3cd5e1e6248f4632e4f18eec8edb2435173f9e6dc08449fa101e", size = 31479, upload-time = "2025-03-03T23:56:51.547Z" } | |||||
| wheels = [ | |||||
| { url = "https://files.pythonhosted.org/packages/6f/8b/92bb54dd74d145221c3854aa245c84f4dc04cc9366147496182cec8e88e3/celery_types-0.23.0-py3-none-any.whl", hash = "sha256:0cc495b8d7729891b7e070d0ec8d4906d2373209656a6e8b8276fe1ed306af9a", size = 50189, upload-time = "2025-03-03T23:56:50.458Z" }, | |||||
| ] | |||||
| [[package]] | [[package]] | ||||
| name = "certifi" | name = "certifi" | ||||
| version = "2025.6.15" | version = "2025.6.15" | ||||
| [package.dev-dependencies] | [package.dev-dependencies] | ||||
| dev = [ | dev = [ | ||||
| { name = "boto3-stubs" }, | { name = "boto3-stubs" }, | ||||
| { name = "celery-types" }, | |||||
| { name = "coverage" }, | { name = "coverage" }, | ||||
| { name = "dotenv-linter" }, | { name = "dotenv-linter" }, | ||||
| { name = "faker" }, | { name = "faker" }, | ||||
| [package.metadata.requires-dev] | [package.metadata.requires-dev] | ||||
| dev = [ | dev = [ | ||||
| { name = "boto3-stubs", specifier = ">=1.38.20" }, | { name = "boto3-stubs", specifier = ">=1.38.20" }, | ||||
| { name = "celery-types", specifier = ">=0.23.0" }, | |||||
| { name = "coverage", specifier = "~=7.2.4" }, | { name = "coverage", specifier = "~=7.2.4" }, | ||||
| { name = "dotenv-linter", specifier = "~=0.5.0" }, | { name = "dotenv-linter", specifier = "~=0.5.0" }, | ||||
| { name = "faker", specifier = "~=32.1.0" }, | { name = "faker", specifier = "~=32.1.0" }, | ||||
| { name = "hypothesis", specifier = ">=6.131.15" }, | { name = "hypothesis", specifier = ">=6.131.15" }, | ||||
| { name = "lxml-stubs", specifier = "~=0.5.1" }, | { name = "lxml-stubs", specifier = "~=0.5.1" }, | ||||
| { name = "mypy", specifier = "~=1.16.0" }, | |||||
| { name = "mypy", specifier = "~=1.17.1" }, | |||||
| { name = "pandas-stubs", specifier = "~=2.2.3" }, | { name = "pandas-stubs", specifier = "~=2.2.3" }, | ||||
| { name = "pytest", specifier = "~=8.3.2" }, | { name = "pytest", specifier = "~=8.3.2" }, | ||||
| { name = "pytest-benchmark", specifier = "~=4.0.0" }, | { name = "pytest-benchmark", specifier = "~=4.0.0" }, | ||||
| [[package]] | [[package]] | ||||
| name = "mypy" | name = "mypy" | ||||
| version = "1.16.1" | |||||
| version = "1.17.1" | |||||
| source = { registry = "https://pypi.org/simple" } | source = { registry = "https://pypi.org/simple" } | ||||
| dependencies = [ | dependencies = [ | ||||
| { name = "mypy-extensions" }, | { name = "mypy-extensions" }, | ||||
| { name = "pathspec" }, | { name = "pathspec" }, | ||||
| { name = "typing-extensions" }, | { name = "typing-extensions" }, | ||||
| ] | ] | ||||
| sdist = { url = "https://files.pythonhosted.org/packages/81/69/92c7fa98112e4d9eb075a239caa4ef4649ad7d441545ccffbd5e34607cbb/mypy-1.16.1.tar.gz", hash = "sha256:6bd00a0a2094841c5e47e7374bb42b83d64c527a502e3334e1173a0c24437bab", size = 3324747, upload-time = "2025-06-16T16:51:35.145Z" } | |||||
| wheels = [ | |||||
| { url = "https://files.pythonhosted.org/packages/9a/61/ec1245aa1c325cb7a6c0f8570a2eee3bfc40fa90d19b1267f8e50b5c8645/mypy-1.16.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:472e4e4c100062488ec643f6162dd0d5208e33e2f34544e1fc931372e806c0cc", size = 10890557, upload-time = "2025-06-16T16:37:21.421Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/6b/bb/6eccc0ba0aa0c7a87df24e73f0ad34170514abd8162eb0c75fd7128171fb/mypy-1.16.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ea16e2a7d2714277e349e24d19a782a663a34ed60864006e8585db08f8ad1782", size = 10012921, upload-time = "2025-06-16T16:51:28.659Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/5f/80/b337a12e2006715f99f529e732c5f6a8c143bb58c92bb142d5ab380963a5/mypy-1.16.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08e850ea22adc4d8a4014651575567b0318ede51e8e9fe7a68f25391af699507", size = 11802887, upload-time = "2025-06-16T16:50:53.627Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/d9/59/f7af072d09793d581a745a25737c7c0a945760036b16aeb620f658a017af/mypy-1.16.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:22d76a63a42619bfb90122889b903519149879ddbf2ba4251834727944c8baca", size = 12531658, upload-time = "2025-06-16T16:33:55.002Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/82/c4/607672f2d6c0254b94a646cfc45ad589dd71b04aa1f3d642b840f7cce06c/mypy-1.16.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2c7ce0662b6b9dc8f4ed86eb7a5d505ee3298c04b40ec13b30e572c0e5ae17c4", size = 12732486, upload-time = "2025-06-16T16:37:03.301Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/b6/5e/136555ec1d80df877a707cebf9081bd3a9f397dedc1ab9750518d87489ec/mypy-1.16.1-cp311-cp311-win_amd64.whl", hash = "sha256:211287e98e05352a2e1d4e8759c5490925a7c784ddc84207f4714822f8cf99b6", size = 9479482, upload-time = "2025-06-16T16:47:37.48Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/b4/d6/39482e5fcc724c15bf6280ff5806548c7185e0c090712a3736ed4d07e8b7/mypy-1.16.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:af4792433f09575d9eeca5c63d7d90ca4aeceda9d8355e136f80f8967639183d", size = 11066493, upload-time = "2025-06-16T16:47:01.683Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/e6/e5/26c347890efc6b757f4d5bb83f4a0cf5958b8cf49c938ac99b8b72b420a6/mypy-1.16.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:66df38405fd8466ce3517eda1f6640611a0b8e70895e2a9462d1d4323c5eb4b9", size = 10081687, upload-time = "2025-06-16T16:48:19.367Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/44/c7/b5cb264c97b86914487d6a24bd8688c0172e37ec0f43e93b9691cae9468b/mypy-1.16.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:44e7acddb3c48bd2713994d098729494117803616e116032af192871aed80b79", size = 11839723, upload-time = "2025-06-16T16:49:20.912Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/15/f8/491997a9b8a554204f834ed4816bda813aefda31cf873bb099deee3c9a99/mypy-1.16.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0ab5eca37b50188163fa7c1b73c685ac66c4e9bdee4a85c9adac0e91d8895e15", size = 12722980, upload-time = "2025-06-16T16:37:40.929Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/df/f0/2bd41e174b5fd93bc9de9a28e4fb673113633b8a7f3a607fa4a73595e468/mypy-1.16.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb6229b2c9086247e21a83c309754b9058b438704ad2f6807f0d8227f6ebdd", size = 12903328, upload-time = "2025-06-16T16:34:35.099Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/61/81/5572108a7bec2c46b8aff7e9b524f371fe6ab5efb534d38d6b37b5490da8/mypy-1.16.1-cp312-cp312-win_amd64.whl", hash = "sha256:1f0435cf920e287ff68af3d10a118a73f212deb2ce087619eb4e648116d1fe9b", size = 9562321, upload-time = "2025-06-16T16:48:58.823Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/cf/d3/53e684e78e07c1a2bf7105715e5edd09ce951fc3f47cf9ed095ec1b7a037/mypy-1.16.1-py3-none-any.whl", hash = "sha256:5fc2ac4027d0ef28d6ba69a0343737a23c4d1b83672bf38d1fe237bdc0643b37", size = 2265923, upload-time = "2025-06-16T16:48:02.366Z" }, | |||||
| sdist = { url = "https://files.pythonhosted.org/packages/8e/22/ea637422dedf0bf36f3ef238eab4e455e2a0dcc3082b5cc067615347ab8e/mypy-1.17.1.tar.gz", hash = "sha256:25e01ec741ab5bb3eec8ba9cdb0f769230368a22c959c4937360efb89b7e9f01", size = 3352570, upload-time = "2025-07-31T07:54:19.204Z" } | |||||
| wheels = [ | |||||
| { url = "https://files.pythonhosted.org/packages/46/cf/eadc80c4e0a70db1c08921dcc220357ba8ab2faecb4392e3cebeb10edbfa/mypy-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ad37544be07c5d7fba814eb370e006df58fed8ad1ef33ed1649cb1889ba6ff58", size = 10921009, upload-time = "2025-07-31T07:53:23.037Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/5d/c1/c869d8c067829ad30d9bdae051046561552516cfb3a14f7f0347b7d973ee/mypy-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:064e2ff508e5464b4bd807a7c1625bc5047c5022b85c70f030680e18f37273a5", size = 10047482, upload-time = "2025-07-31T07:53:26.151Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/98/b9/803672bab3fe03cee2e14786ca056efda4bb511ea02dadcedde6176d06d0/mypy-1.17.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70401bbabd2fa1aa7c43bb358f54037baf0586f41e83b0ae67dd0534fc64edfd", size = 11832883, upload-time = "2025-07-31T07:53:47.948Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/88/fb/fcdac695beca66800918c18697b48833a9a6701de288452b6715a98cfee1/mypy-1.17.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e92bdc656b7757c438660f775f872a669b8ff374edc4d18277d86b63edba6b8b", size = 12566215, upload-time = "2025-07-31T07:54:04.031Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/7f/37/a932da3d3dace99ee8eb2043b6ab03b6768c36eb29a02f98f46c18c0da0e/mypy-1.17.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:c1fdf4abb29ed1cb091cf432979e162c208a5ac676ce35010373ff29247bcad5", size = 12751956, upload-time = "2025-07-31T07:53:36.263Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/8c/cf/6438a429e0f2f5cab8bc83e53dbebfa666476f40ee322e13cac5e64b79e7/mypy-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:ff2933428516ab63f961644bc49bc4cbe42bbffb2cd3b71cc7277c07d16b1a8b", size = 9507307, upload-time = "2025-07-31T07:53:59.734Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/17/a2/7034d0d61af8098ec47902108553122baa0f438df8a713be860f7407c9e6/mypy-1.17.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:69e83ea6553a3ba79c08c6e15dbd9bfa912ec1e493bf75489ef93beb65209aeb", size = 11086295, upload-time = "2025-07-31T07:53:28.124Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/14/1f/19e7e44b594d4b12f6ba8064dbe136505cec813549ca3e5191e40b1d3cc2/mypy-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1b16708a66d38abb1e6b5702f5c2c87e133289da36f6a1d15f6a5221085c6403", size = 10112355, upload-time = "2025-07-31T07:53:21.121Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/5b/69/baa33927e29e6b4c55d798a9d44db5d394072eef2bdc18c3e2048c9ed1e9/mypy-1.17.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:89e972c0035e9e05823907ad5398c5a73b9f47a002b22359b177d40bdaee7056", size = 11875285, upload-time = "2025-07-31T07:53:55.293Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/90/13/f3a89c76b0a41e19490b01e7069713a30949d9a6c147289ee1521bcea245/mypy-1.17.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:03b6d0ed2b188e35ee6d5c36b5580cffd6da23319991c49ab5556c023ccf1341", size = 12737895, upload-time = "2025-07-31T07:53:43.623Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/23/a1/c4ee79ac484241301564072e6476c5a5be2590bc2e7bfd28220033d2ef8f/mypy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c837b896b37cd103570d776bda106eabb8737aa6dd4f248451aecf53030cdbeb", size = 12931025, upload-time = "2025-07-31T07:54:17.125Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/89/b8/7409477be7919a0608900e6320b155c72caab4fef46427c5cc75f85edadd/mypy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:665afab0963a4b39dff7c1fa563cc8b11ecff7910206db4b2e64dd1ba25aed19", size = 9584664, upload-time = "2025-07-31T07:54:12.842Z" }, | |||||
| { url = "https://files.pythonhosted.org/packages/1d/f3/8fcd2af0f5b806f6cf463efaffd3c9548a28f84220493ecd38d127b6b66d/mypy-1.17.1-py3-none-any.whl", hash = "sha256:a9f52c0351c21fe24c21d8c0eb1f62967b262d6729393397b6f443c3b773c3b9", size = 2283411, upload-time = "2025-07-31T07:53:24.664Z" }, | |||||
| ] | ] | ||||
| [[package]] | [[package]] |