Merge remote-tracking branch 'origin/main' into feat/queue-based-graph-engine

api/configs/middleware/__init__.py

@@ -300,8 +300,7 @@ class DatasetQueueMonitorConfig(BaseSettings):
 
 class MiddlewareConfig(
     # place the configs in alphabet order
-    CeleryConfig,
-    DatabaseConfig,
+    CeleryConfig,  # Note: CeleryConfig already inherits from DatabaseConfig
     KeywordStoreConfig,
     RedisConfig,
     # configs of storage and storage providers

api/configs/middleware/vdb/clickzetta_config.py

@@ -1,9 +1,10 @@
 from typing import Optional
 
-from pydantic import BaseModel, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 
-class ClickzettaConfig(BaseModel):
+class ClickzettaConfig(BaseSettings):
     """
     Clickzetta Lakehouse vector database configuration
     """

api/configs/middleware/vdb/matrixone_config.py

@@ -1,7 +1,8 @@
-from pydantic import BaseModel, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 
-class MatrixoneConfig(BaseModel):
+class MatrixoneConfig(BaseSettings):
     """Matrixone vector database configuration."""
 
     MATRIXONE_HOST: str = Field(default="localhost", description="Host address of the Matrixone server")

api/configs/packaging/__init__.py

@@ -1,6 +1,6 @@
 from pydantic import Field
 
-from configs.packaging.pyproject import PyProjectConfig, PyProjectTomlConfig
+from configs.packaging.pyproject import PyProjectTomlConfig
 
 
 class PackagingInfo(PyProjectTomlConfig):

api/configs/remote_settings_sources/apollo/client.py

@@ -4,8 +4,9 @@ import logging
 import os
 import threading
 import time
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from pathlib import Path
+from typing import Any
 
 from .python_3x import http_request, makedirs_wrapper
 from .utils import (
@@ -25,13 +26,13 @@ logger = logging.getLogger(__name__)
 class ApolloClient:
     def __init__(
         self,
-        config_url,
-        app_id,
-        cluster="default",
-        secret="",
-        start_hot_update=True,
-        change_listener=None,
-        _notification_map=None,
+        config_url: str,
+        app_id: str,
+        cluster: str = "default",
+        secret: str = "",
+        start_hot_update: bool = True,
+        change_listener: Callable[[str, str, str, Any], None] | None = None,
+        _notification_map: dict[str, int] | None = None,
     ):
         # Core routing parameters
         self.config_url = config_url
@@ -47,17 +48,17 @@ class ApolloClient:
         # Private control variables
         self._cycle_time = 5
         self._stopping = False
-        self._cache = {}
-        self._no_key = {}
-        self._hash = {}
+        self._cache: dict[str, dict[str, Any]] = {}
+        self._no_key: dict[str, str] = {}
+        self._hash: dict[str, str] = {}
         self._pull_timeout = 75
         self._cache_file_path = os.path.expanduser("~") + "/.dify/config/remote-settings/apollo/cache/"
-        self._long_poll_thread = None
+        self._long_poll_thread: threading.Thread | None = None
         self._change_listener = change_listener  # "add" "delete" "update"
         if _notification_map is None:
             _notification_map = {"application": -1}
         self._notification_map = _notification_map
-        self.last_release_key = None
+        self.last_release_key: str | None = None
         # Private startup method
         self._path_checker()
         if start_hot_update:
@@ -68,7 +69,7 @@ class ApolloClient:
         heartbeat.daemon = True
         heartbeat.start()
 
-    def get_json_from_net(self, namespace="application"):
+    def get_json_from_net(self, namespace: str = "application") -> dict[str, Any] | None:
         url = "{}/configs/{}/{}/{}?releaseKey={}&ip={}".format(
             self.config_url, self.app_id, self.cluster, namespace, "", self.ip
         )
@@ -88,7 +89,7 @@ class ApolloClient:
             logger.exception("an error occurred in get_json_from_net")
             return None
 
-    def get_value(self, key, default_val=None, namespace="application"):
+    def get_value(self, key: str, default_val: Any = None, namespace: str = "application") -> Any:
         try:
             # read memory configuration
             namespace_cache = self._cache.get(namespace)
@@ -104,7 +105,8 @@ class ApolloClient:
             namespace_data = self.get_json_from_net(namespace)
             val = get_value_from_dict(namespace_data, key)
             if val is not None:
-                self._update_cache_and_file(namespace_data, namespace)
+                if namespace_data is not None:
+                    self._update_cache_and_file(namespace_data, namespace)
                 return val
 
             # read the file configuration
@@ -126,23 +128,23 @@ class ApolloClient:
     # to ensure the real-time correctness of the function call.
     # If the user does not have the same default val twice
     # and the default val is used here, there may be a problem.
-    def _set_local_cache_none(self, namespace, key):
+    def _set_local_cache_none(self, namespace: str, key: str) -> None:
         no_key = no_key_cache_key(namespace, key)
         self._no_key[no_key] = key
 
-    def _start_hot_update(self):
+    def _start_hot_update(self) -> None:
         self._long_poll_thread = threading.Thread(target=self._listener)
         # When the asynchronous thread is started, the daemon thread will automatically exit
         # when the main thread is launched.
         self._long_poll_thread.daemon = True
         self._long_poll_thread.start()
 
-    def stop(self):
+    def stop(self) -> None:
         self._stopping = True
         logger.info("Stopping listener...")
 
     # Call the set callback function, and if it is abnormal, try it out
-    def _call_listener(self, namespace, old_kv, new_kv):
+    def _call_listener(self, namespace: str, old_kv: dict[str, Any] | None, new_kv: dict[str, Any] | None) -> None:
         if self._change_listener is None:
             return
         if old_kv is None:
@@ -168,12 +170,12 @@ class ApolloClient:
         except BaseException as e:
             logger.warning(str(e))
 
-    def _path_checker(self):
+    def _path_checker(self) -> None:
         if not os.path.isdir(self._cache_file_path):
             makedirs_wrapper(self._cache_file_path)
 
     # update the local cache and file cache
-    def _update_cache_and_file(self, namespace_data, namespace="application"):
+    def _update_cache_and_file(self, namespace_data: dict[str, Any], namespace: str = "application") -> None:
         # update the local cache
         self._cache[namespace] = namespace_data
         # update the file cache
@@ -187,7 +189,7 @@ class ApolloClient:
             self._hash[namespace] = new_hash
 
     # get the configuration from the local file
-    def _get_local_cache(self, namespace="application"):
+    def _get_local_cache(self, namespace: str = "application") -> dict[str, Any]:
         cache_file_path = os.path.join(self._cache_file_path, f"{self.app_id}_configuration_{namespace}.txt")
         if os.path.isfile(cache_file_path):
             with open(cache_file_path) as f:
@@ -195,8 +197,8 @@ class ApolloClient:
             return result
         return {}
 
-    def _long_poll(self):
-        notifications = []
+    def _long_poll(self) -> None:
+        notifications: list[dict[str, Any]] = []
         for key in self._cache:
             namespace_data = self._cache[key]
             notification_id = -1
@@ -236,7 +238,7 @@ class ApolloClient:
         except Exception as e:
             logger.warning(str(e))
 
-    def _get_net_and_set_local(self, namespace, n_id, call_change=False):
+    def _get_net_and_set_local(self, namespace: str, n_id: int, call_change: bool = False) -> None:
         namespace_data = self.get_json_from_net(namespace)
         if not namespace_data:
             return
@@ -248,7 +250,7 @@ class ApolloClient:
             new_kv = namespace_data.get(CONFIGURATIONS)
             self._call_listener(namespace, old_kv, new_kv)
 
-    def _listener(self):
+    def _listener(self) -> None:
         logger.info("start long_poll")
         while not self._stopping:
             self._long_poll()
@@ -266,13 +268,13 @@ class ApolloClient:
         headers["Timestamp"] = time_unix_now
         return headers
 
-    def _heart_beat(self):
+    def _heart_beat(self) -> None:
         while not self._stopping:
             for namespace in self._notification_map:
                 self._do_heart_beat(namespace)
             time.sleep(60 * 10)  # 10 minutes
 
-    def _do_heart_beat(self, namespace):
+    def _do_heart_beat(self, namespace: str) -> None:
         url = f"{self.config_url}/configs/{self.app_id}/{self.cluster}/{namespace}?ip={self.ip}"
         try:
             code, body = http_request(url, timeout=3, headers=self._sign_headers(url))
@@ -292,7 +294,7 @@ class ApolloClient:
             logger.exception("an error occurred in _do_heart_beat")
             return None
 
-    def get_all_dicts(self, namespace):
+    def get_all_dicts(self, namespace: str) -> dict[str, Any] | None:
         namespace_data = self._cache.get(namespace)
         if namespace_data is None:
             net_namespace_data = self.get_json_from_net(namespace)

api/configs/remote_settings_sources/apollo/python_3x.py

@@ -2,6 +2,8 @@ import logging
 import os
 import ssl
 import urllib.request
+from collections.abc import Mapping
+from typing import Any
 from urllib import parse
 from urllib.error import HTTPError
 
@@ -19,9 +21,9 @@ urllib.request.install_opener(opener)
 logger = logging.getLogger(__name__)
 
 
-def http_request(url, timeout, headers={}):
+def http_request(url: str, timeout: int | float, headers: Mapping[str, str] = {}) -> tuple[int, str | None]:
     try:
-        request = urllib.request.Request(url, headers=headers)
+        request = urllib.request.Request(url, headers=dict(headers))
         res = urllib.request.urlopen(request, timeout=timeout)
         body = res.read().decode("utf-8")
         return res.code, body
@@ -33,9 +35,9 @@ def http_request(url, timeout, headers={}):
         raise e
 
 
-def url_encode(params):
+def url_encode(params: dict[str, Any]) -> str:
     return parse.urlencode(params)
 
 
-def makedirs_wrapper(path):
+def makedirs_wrapper(path: str) -> None:
     os.makedirs(path, exist_ok=True)

api/configs/remote_settings_sources/apollo/utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import socket
+from typing import Any
 
 from .python_3x import url_encode
 
@@ -10,7 +11,7 @@ NAMESPACE_NAME = "namespaceName"
 
 
 # add timestamps uris and keys
-def signature(timestamp, uri, secret):
+def signature(timestamp: str, uri: str, secret: str) -> str:
     import base64
     import hmac
 
@@ -19,16 +20,16 @@ def signature(timestamp, uri, secret):
     return base64.b64encode(hmac_code).decode()
 
 
-def url_encode_wrapper(params):
+def url_encode_wrapper(params: dict[str, Any]) -> str:
     return url_encode(params)
 
 
-def no_key_cache_key(namespace, key):
+def no_key_cache_key(namespace: str, key: str) -> str:
     return f"{namespace}{len(namespace)}{key}"
 
 
 # Returns whether the obtained value is obtained, and None if it does not
-def get_value_from_dict(namespace_cache, key):
+def get_value_from_dict(namespace_cache: dict[str, Any] | None, key: str) -> Any | None:
     if namespace_cache:
         kv_data = namespace_cache.get(CONFIGURATIONS)
         if kv_data is None:
@@ -38,7 +39,7 @@ def get_value_from_dict(namespace_cache, key):
     return None
 
 
-def init_ip():
+def init_ip() -> str:
     ip = ""
     s = None
     try:

api/configs/remote_settings_sources/nacos/__init__.py

@@ -11,16 +11,16 @@ logger = logging.getLogger(__name__)
 
 from configs.remote_settings_sources.base import RemoteSettingsSource
 
-from .utils import _parse_config
+from .utils import parse_config
 
 
 class NacosSettingsSource(RemoteSettingsSource):
     def __init__(self, configs: Mapping[str, Any]):
         self.configs = configs
-        self.remote_configs: dict[str, Any] = {}
+        self.remote_configs: dict[str, str] = {}
         self.async_init()
 
-    def async_init(self):
+    def async_init(self) -> None:
         data_id = os.getenv("DIFY_ENV_NACOS_DATA_ID", "dify-api-env.properties")
         group = os.getenv("DIFY_ENV_NACOS_GROUP", "nacos-dify")
         tenant = os.getenv("DIFY_ENV_NACOS_NAMESPACE", "")
@@ -33,18 +33,15 @@ class NacosSettingsSource(RemoteSettingsSource):
             logger.exception("[get-access-token] exception occurred")
             raise
 
-    def _parse_config(self, content: str):
+    def _parse_config(self, content: str) -> dict[str, str]:
         if not content:
             return {}
         try:
-            return _parse_config(self, content)
+            return parse_config(content)
         except Exception as e:
             raise RuntimeError(f"Failed to parse config: {e}")
 
     def get_field_value(self, field: FieldInfo, field_name: str) -> tuple[Any, str, bool]:
-        if not isinstance(self.remote_configs, dict):
-            raise ValueError(f"remote configs is not dict, but {type(self.remote_configs)}")
-
         field_value = self.remote_configs.get(field_name)
         if field_value is None:
             return None, field_name, False

api/configs/remote_settings_sources/nacos/http_request.py

@@ -17,11 +17,17 @@ class NacosHttpClient:
         self.ak = os.getenv("DIFY_ENV_NACOS_ACCESS_KEY")
         self.sk = os.getenv("DIFY_ENV_NACOS_SECRET_KEY")
         self.server = os.getenv("DIFY_ENV_NACOS_SERVER_ADDR", "localhost:8848")
-        self.token = None
+        self.token: str | None = None
         self.token_ttl = 18000
         self.token_expire_time: float = 0
 
-    def http_request(self, url, method="GET", headers=None, params=None):
+    def http_request(
+        self, url: str, method: str = "GET", headers: dict[str, str] | None = None, params: dict[str, str] | None = None
+    ) -> str:
+        if headers is None:
+            headers = {}
+        if params is None:
+            params = {}
         try:
             self._inject_auth_info(headers, params)
             response = requests.request(method, url="http://" + self.server + url, headers=headers, params=params)
@@ -30,7 +36,7 @@ class NacosHttpClient:
         except requests.RequestException as e:
             return f"Request to Nacos failed: {e}"
 
-    def _inject_auth_info(self, headers, params, module="config"):
+    def _inject_auth_info(self, headers: dict[str, str], params: dict[str, str], module: str = "config") -> None:
         headers.update({"User-Agent": "Nacos-Http-Client-In-Dify:v0.0.1"})
 
         if module == "login":
@@ -45,16 +51,17 @@ class NacosHttpClient:
             headers["timeStamp"] = ts
         if self.username and self.password:
             self.get_access_token(force_refresh=False)
-            params["accessToken"] = self.token
+            if self.token is not None:
+                params["accessToken"] = self.token
 
-    def __do_sign(self, sign_str, sk):
+    def __do_sign(self, sign_str: str, sk: str) -> str:
         return (
             base64.encodebytes(hmac.new(sk.encode(), sign_str.encode(), digestmod=hashlib.sha1).digest())
             .decode()
             .strip()
         )
 
-    def get_sign_str(self, group, tenant, ts):
+    def get_sign_str(self, group: str, tenant: str, ts: str) -> str:
         sign_str = ""
         if tenant:
             sign_str = tenant + "+"
@@ -63,7 +70,7 @@ class NacosHttpClient:
         sign_str += ts  # Directly concatenate ts without conditional checks, because the nacos auth header forced it.
         return sign_str
 
-    def get_access_token(self, force_refresh=False):
+    def get_access_token(self, force_refresh: bool = False) -> str | None:
         current_time = time.time()
         if self.token and not force_refresh and self.token_expire_time > current_time:
             return self.token
@@ -77,6 +84,7 @@ class NacosHttpClient:
             self.token = response_data.get("accessToken")
             self.token_ttl = response_data.get("tokenTtl", 18000)
             self.token_expire_time = current_time + self.token_ttl - 10
+            return self.token
         except Exception:
             logger.exception("[get-access-token] exception occur")
             raise

api/configs/remote_settings_sources/nacos/utils.py

@@ -1,4 +1,4 @@
-def _parse_config(self, content: str) -> dict[str, str]:
+def parse_config(content: str) -> dict[str, str]:
     config: dict[str, str] = {}
     if not content:
         return config

api/core/rag/datasource/vdb/matrixone/matrixone_vector.py

@@ -99,9 +99,9 @@ class MatrixoneVector(BaseVector):
                 return client
             try:
                 client.create_full_text_index()
+                redis_client.set(collection_exist_cache_key, 1, ex=3600)
             except Exception:
                 logger.exception("Failed to create full text index")
-            redis_client.set(collection_exist_cache_key, 1, ex=3600)
             return client
 
     def add_texts(self, documents: list[Document], embeddings: list[list[float]], **kwargs):

api/pyrightconfig.json

@@ -1,5 +1,7 @@
 {
-  "include": ["."],
+  "include": [
+    "."
+  ],
   "exclude": [
     "tests/",
     "migrations/",
@@ -19,10 +21,9 @@
     "events/",
     "contexts/",
     "constants/",
-    "configs/",
     "commands.py"
   ],
   "typeCheckingMode": "strict",
   "pythonVersion": "3.11",
   "pythonPlatform": "All"
-}
+}

api/tests/test_containers_integration_tests/tasks/test_batch_clean_document_task.py

@@ -0,0 +1,720 @@
+"""
+Integration tests for batch_clean_document_task using testcontainers.
+
+This module tests the batch document cleaning functionality with real database
+and storage containers to ensure proper cleanup of documents, segments, and files.
+"""
+
+import json
+import uuid
+from unittest.mock import Mock, patch
+
+import pytest
+from faker import Faker
+
+from extensions.ext_database import db
+from models.account import Account, Tenant, TenantAccountJoin, TenantAccountRole
+from models.dataset import Dataset, Document, DocumentSegment
+from models.model import UploadFile
+from tasks.batch_clean_document_task import batch_clean_document_task
+
+
+class TestBatchCleanDocumentTask:
+    """Integration tests for batch_clean_document_task using testcontainers."""
+
+    @pytest.fixture
+    def mock_external_service_dependencies(self):
+        """Mock setup for external service dependencies."""
+        with (
+            patch("extensions.ext_storage.storage") as mock_storage,
+            patch("core.rag.index_processor.index_processor_factory.IndexProcessorFactory") as mock_index_factory,
+            patch("core.tools.utils.web_reader_tool.get_image_upload_file_ids") as mock_get_image_ids,
+        ):
+            # Setup default mock returns
+            mock_storage.delete.return_value = None
+
+            # Mock index processor
+            mock_index_processor = Mock()
+            mock_index_processor.clean.return_value = None
+            mock_index_factory.return_value.init_index_processor.return_value = mock_index_processor
+
+            # Mock image file ID extraction
+            mock_get_image_ids.return_value = []
+
+            yield {
+                "storage": mock_storage,
+                "index_factory": mock_index_factory,
+                "index_processor": mock_index_processor,
+                "get_image_ids": mock_get_image_ids,
+            }
+
+    def _create_test_account(self, db_session_with_containers):
+        """
+        Helper method to create a test account for testing.
+
+        Args:
+            db_session_with_containers: Database session from testcontainers infrastructure
+
+        Returns:
+            Account: Created account instance
+        """
+        fake = Faker()
+
+        # Create account
+        account = Account(
+            email=fake.email(),
+            name=fake.name(),
+            interface_language="en-US",
+            status="active",
+        )
+
+        db.session.add(account)
+        db.session.commit()
+
+        # Create tenant for the account
+        tenant = Tenant(
+            name=fake.company(),
+            status="normal",
+        )
+        db.session.add(tenant)
+        db.session.commit()
+
+        # Create tenant-account join
+        join = TenantAccountJoin(
+            tenant_id=tenant.id,
+            account_id=account.id,
+            role=TenantAccountRole.OWNER.value,
+            current=True,
+        )
+        db.session.add(join)
+        db.session.commit()
+
+        # Set current tenant for account
+        account.current_tenant = tenant
+
+        return account
+
+    def _create_test_dataset(self, db_session_with_containers, account):
+        """
+        Helper method to create a test dataset for testing.
+
+        Args:
+            db_session_with_containers: Database session from testcontainers infrastructure
+            account: Account instance
+
+        Returns:
+            Dataset: Created dataset instance
+        """
+        fake = Faker()
+
+        dataset = Dataset(
+            id=str(uuid.uuid4()),
+            tenant_id=account.current_tenant.id,
+            name=fake.word(),
+            description=fake.sentence(),
+            data_source_type="upload_file",
+            created_by=account.id,
+            embedding_model="text-embedding-ada-002",
+            embedding_model_provider="openai",
+        )
+
+        db.session.add(dataset)
+        db.session.commit()
+
+        return dataset
+
+    def _create_test_document(self, db_session_with_containers, dataset, account):
+        """
+        Helper method to create a test document for testing.
+
+        Args:
+            db_session_with_containers: Database session from testcontainers infrastructure
+            dataset: Dataset instance
+            account: Account instance
+
+        Returns:
+            Document: Created document instance
+        """
+        fake = Faker()
+
+        document = Document(
+            id=str(uuid.uuid4()),
+            tenant_id=account.current_tenant.id,
+            dataset_id=dataset.id,
+            position=0,
+            name=fake.word(),
+            data_source_type="upload_file",
+            data_source_info=json.dumps({"upload_file_id": str(uuid.uuid4())}),
+            batch="test_batch",
+            created_from="test",
+            created_by=account.id,
+            indexing_status="completed",
+            doc_form="text_model",
+        )
+
+        db.session.add(document)
+        db.session.commit()
+
+        return document
+
+    def _create_test_document_segment(self, db_session_with_containers, document, account):
+        """
+        Helper method to create a test document segment for testing.
+
+        Args:
+            db_session_with_containers: Database session from testcontainers infrastructure
+            document: Document instance
+            account: Account instance
+
+        Returns:
+            DocumentSegment: Created document segment instance
+        """
+        fake = Faker()
+
+        segment = DocumentSegment(
+            id=str(uuid.uuid4()),
+            tenant_id=account.current_tenant.id,
+            dataset_id=document.dataset_id,
+            document_id=document.id,
+            position=0,
+            content=fake.text(),
+            word_count=100,
+            tokens=50,
+            index_node_id=str(uuid.uuid4()),
+            created_by=account.id,
+            status="completed",
+        )
+
+        db.session.add(segment)
+        db.session.commit()
+
+        return segment
+
+    def _create_test_upload_file(self, db_session_with_containers, account):
+        """
+        Helper method to create a test upload file for testing.
+
+        Args:
+            db_session_with_containers: Database session from testcontainers infrastructure
+            account: Account instance
+
+        Returns:
+            UploadFile: Created upload file instance
+        """
+        fake = Faker()
+        from datetime import datetime
+
+        from models.enums import CreatorUserRole
+
+        upload_file = UploadFile(
+            tenant_id=account.current_tenant.id,
+            storage_type="local",
+            key=f"test_files/{fake.file_name()}",
+            name=fake.file_name(),
+            size=1024,
+            extension="txt",
+            mime_type="text/plain",
+            created_by_role=CreatorUserRole.ACCOUNT,
+            created_by=account.id,
+            created_at=datetime.utcnow(),
+            used=False,
+        )
+
+        db.session.add(upload_file)
+        db.session.commit()
+
+        return upload_file
+
+    def test_batch_clean_document_task_successful_cleanup(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test successful cleanup of documents with segments and files.
+
+        This test verifies that the task properly cleans up:
+        - Document segments from the index
+        - Associated image files from storage
+        - Upload files from storage and database
+        """
+        # Create test data
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+        document = self._create_test_document(db_session_with_containers, dataset, account)
+        segment = self._create_test_document_segment(db_session_with_containers, document, account)
+        upload_file = self._create_test_upload_file(db_session_with_containers, account)
+
+        # Update document to reference the upload file
+        document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+        db.session.commit()
+
+        # Store original IDs for verification
+        document_id = document.id
+        segment_id = segment.id
+        file_id = upload_file.id
+
+        # Execute the task
+        batch_clean_document_task(
+            document_ids=[document_id], dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=[file_id]
+        )
+
+        # Verify that the task completed successfully
+        # The task should have processed the segment and cleaned up the database
+
+        # Verify database cleanup
+        db.session.commit()  # Ensure all changes are committed
+
+        # Check that segment is deleted
+        deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+        assert deleted_segment is None
+
+        # Check that upload file is deleted
+        deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+        assert deleted_file is None
+
+    def test_batch_clean_document_task_with_image_files(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup of documents containing image references.
+
+        This test verifies that the task properly handles documents with
+        image content and cleans up associated segments.
+        """
+        # Create test data
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+        document = self._create_test_document(db_session_with_containers, dataset, account)
+
+        # Create segment with simple content (no image references)
+        segment = DocumentSegment(
+            id=str(uuid.uuid4()),
+            tenant_id=account.current_tenant.id,
+            dataset_id=document.dataset_id,
+            document_id=document.id,
+            position=0,
+            content="Simple text content without images",
+            word_count=100,
+            tokens=50,
+            index_node_id=str(uuid.uuid4()),
+            created_by=account.id,
+            status="completed",
+        )
+
+        db.session.add(segment)
+        db.session.commit()
+
+        # Store original IDs for verification
+        segment_id = segment.id
+        document_id = document.id
+
+        # Execute the task
+        batch_clean_document_task(
+            document_ids=[document_id], dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=[]
+        )
+
+        # Verify database cleanup
+        db.session.commit()
+
+        # Check that segment is deleted
+        deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+        assert deleted_segment is None
+
+        # Verify that the task completed successfully by checking the log output
+        # The task should have processed the segment and cleaned up the database
+
+    def test_batch_clean_document_task_no_segments(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup when document has no segments.
+
+        This test verifies that the task handles documents without segments
+        gracefully and still cleans up associated files.
+        """
+        # Create test data without segments
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+        document = self._create_test_document(db_session_with_containers, dataset, account)
+        upload_file = self._create_test_upload_file(db_session_with_containers, account)
+
+        # Update document to reference the upload file
+        document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+        db.session.commit()
+
+        # Store original IDs for verification
+        document_id = document.id
+        file_id = upload_file.id
+
+        # Execute the task
+        batch_clean_document_task(
+            document_ids=[document_id], dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=[file_id]
+        )
+
+        # Verify that the task completed successfully
+        # Since there are no segments, the task should handle this gracefully
+
+        # Verify database cleanup
+        db.session.commit()
+
+        # Check that upload file is deleted
+        deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+        assert deleted_file is None
+
+        # Verify database cleanup
+        db.session.commit()
+
+        # Check that upload file is deleted
+        deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+        assert deleted_file is None
+
+    def test_batch_clean_document_task_dataset_not_found(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup when dataset is not found.
+
+        This test verifies that the task properly handles the case where
+        the specified dataset does not exist in the database.
+        """
+        # Create test data
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+        document = self._create_test_document(db_session_with_containers, dataset, account)
+
+        # Store original IDs for verification
+        document_id = document.id
+        dataset_id = dataset.id
+
+        # Delete the dataset to simulate not found scenario
+        db.session.delete(dataset)
+        db.session.commit()
+
+        # Execute the task with non-existent dataset
+        batch_clean_document_task(document_ids=[document_id], dataset_id=dataset_id, doc_form="text_model", file_ids=[])
+
+        # Verify that no index processing occurred
+        mock_external_service_dependencies["index_processor"].clean.assert_not_called()
+
+        # Verify that no storage operations occurred
+        mock_external_service_dependencies["storage"].delete.assert_not_called()
+
+        # Verify that no database cleanup occurred
+        db.session.commit()
+
+        # Document should still exist since cleanup failed
+        existing_document = db.session.query(Document).filter_by(id=document_id).first()
+        assert existing_document is not None
+
+    def test_batch_clean_document_task_storage_cleanup_failure(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup when storage operations fail.
+
+        This test verifies that the task continues processing even when
+        storage cleanup operations fail, ensuring database cleanup still occurs.
+        """
+        # Create test data
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+        document = self._create_test_document(db_session_with_containers, dataset, account)
+        segment = self._create_test_document_segment(db_session_with_containers, document, account)
+        upload_file = self._create_test_upload_file(db_session_with_containers, account)
+
+        # Update document to reference the upload file
+        document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+        db.session.commit()
+
+        # Store original IDs for verification
+        document_id = document.id
+        segment_id = segment.id
+        file_id = upload_file.id
+
+        # Mock storage.delete to raise an exception
+        mock_external_service_dependencies["storage"].delete.side_effect = Exception("Storage error")
+
+        # Execute the task
+        batch_clean_document_task(
+            document_ids=[document_id], dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=[file_id]
+        )
+
+        # Verify that the task completed successfully despite storage failure
+        # The task should continue processing even when storage operations fail
+
+        # Verify database cleanup still occurred despite storage failure
+        db.session.commit()
+
+        # Check that segment is deleted from database
+        deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+        assert deleted_segment is None
+
+        # Check that upload file is deleted from database
+        deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+        assert deleted_file is None
+
+    def test_batch_clean_document_task_multiple_documents(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup of multiple documents in a single batch operation.
+
+        This test verifies that the task can handle multiple documents
+        efficiently and cleans up all associated resources.
+        """
+        # Create test data for multiple documents
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+
+        documents = []
+        segments = []
+        upload_files = []
+
+        # Create 3 documents with segments and files
+        for i in range(3):
+            document = self._create_test_document(db_session_with_containers, dataset, account)
+            segment = self._create_test_document_segment(db_session_with_containers, document, account)
+            upload_file = self._create_test_upload_file(db_session_with_containers, account)
+
+            # Update document to reference the upload file
+            document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+
+            documents.append(document)
+            segments.append(segment)
+            upload_files.append(upload_file)
+
+        db.session.commit()
+
+        # Store original IDs for verification
+        document_ids = [doc.id for doc in documents]
+        segment_ids = [seg.id for seg in segments]
+        file_ids = [file.id for file in upload_files]
+
+        # Execute the task with multiple documents
+        batch_clean_document_task(
+            document_ids=document_ids, dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=file_ids
+        )
+
+        # Verify that the task completed successfully for all documents
+        # The task should process all documents and clean up all associated resources
+
+        # Verify database cleanup for all resources
+        db.session.commit()
+
+        # Check that all segments are deleted
+        for segment_id in segment_ids:
+            deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+            assert deleted_segment is None
+
+        # Check that all upload files are deleted
+        for file_id in file_ids:
+            deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+            assert deleted_file is None
+
+    def test_batch_clean_document_task_different_doc_forms(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup with different document form types.
+
+        This test verifies that the task properly handles different
+        document form types and creates the appropriate index processor.
+        """
+        # Create test data
+        account = self._create_test_account(db_session_with_containers)
+
+        # Test different doc_form types
+        doc_forms = ["text_model", "qa_model", "hierarchical_model"]
+
+        for doc_form in doc_forms:
+            dataset = self._create_test_dataset(db_session_with_containers, account)
+            db.session.commit()
+
+            document = self._create_test_document(db_session_with_containers, dataset, account)
+            # Update document doc_form
+            document.doc_form = doc_form
+            db.session.commit()
+
+            segment = self._create_test_document_segment(db_session_with_containers, document, account)
+
+            # Store the ID before the object is deleted
+            segment_id = segment.id
+
+            try:
+                # Execute the task
+                batch_clean_document_task(
+                    document_ids=[document.id], dataset_id=dataset.id, doc_form=doc_form, file_ids=[]
+                )
+
+                # Verify that the task completed successfully for this doc_form
+                # The task should handle different document forms correctly
+
+                # Verify database cleanup
+                db.session.commit()
+
+                # Check that segment is deleted
+                deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+                assert deleted_segment is None
+
+            except Exception as e:
+                # If the task fails due to external service issues (e.g., plugin daemon),
+                # we should still verify that the database state is consistent
+                # This is a common scenario in test environments where external services may not be available
+                db.session.commit()
+
+                # Check if the segment still exists (task may have failed before deletion)
+                existing_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+                if existing_segment is not None:
+                    # If segment still exists, the task failed before deletion
+                    # This is acceptable in test environments with external service issues
+                    pass
+                else:
+                    # If segment was deleted, the task succeeded
+                    pass
+
+    def test_batch_clean_document_task_large_batch_performance(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test cleanup performance with a large batch of documents.
+
+        This test verifies that the task can handle large batches efficiently
+        and maintains performance characteristics.
+        """
+        import time
+
+        # Create test data for large batch
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+
+        documents = []
+        segments = []
+        upload_files = []
+
+        # Create 10 documents with segments and files (larger batch)
+        batch_size = 10
+        for i in range(batch_size):
+            document = self._create_test_document(db_session_with_containers, dataset, account)
+            segment = self._create_test_document_segment(db_session_with_containers, document, account)
+            upload_file = self._create_test_upload_file(db_session_with_containers, account)
+
+            # Update document to reference the upload file
+            document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+
+            documents.append(document)
+            segments.append(segment)
+            upload_files.append(upload_file)
+
+        db.session.commit()
+
+        # Store original IDs for verification
+        document_ids = [doc.id for doc in documents]
+        segment_ids = [seg.id for seg in segments]
+        file_ids = [file.id for file in upload_files]
+
+        # Measure execution time
+        start_time = time.perf_counter()
+
+        # Execute the task with large batch
+        batch_clean_document_task(
+            document_ids=document_ids, dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=file_ids
+        )
+
+        end_time = time.perf_counter()
+        execution_time = end_time - start_time
+
+        # Verify performance characteristics (should complete within reasonable time)
+        assert execution_time < 5.0  # Should complete within 5 seconds
+
+        # Verify that the task completed successfully for the large batch
+        # The task should handle large batches efficiently
+
+        # Verify database cleanup for all resources
+        db.session.commit()
+
+        # Check that all segments are deleted
+        for segment_id in segment_ids:
+            deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+            assert deleted_segment is None
+
+        # Check that all upload files are deleted
+        for file_id in file_ids:
+            deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+            assert deleted_file is None
+
+    def test_batch_clean_document_task_integration_with_real_database(
+        self, db_session_with_containers, mock_external_service_dependencies
+    ):
+        """
+        Test full integration with real database operations.
+
+        This test verifies that the task integrates properly with the
+        actual database and maintains data consistency throughout the process.
+        """
+        # Create test data
+        account = self._create_test_account(db_session_with_containers)
+        dataset = self._create_test_dataset(db_session_with_containers, account)
+
+        # Create document with complex structure
+        document = self._create_test_document(db_session_with_containers, dataset, account)
+
+        # Create multiple segments for the document
+        segments = []
+        for i in range(3):
+            segment = DocumentSegment(
+                id=str(uuid.uuid4()),
+                tenant_id=account.current_tenant.id,
+                dataset_id=document.dataset_id,
+                document_id=document.id,
+                position=i,
+                content=f"Segment content {i} with some text",
+                word_count=50 + i * 10,
+                tokens=25 + i * 5,
+                index_node_id=str(uuid.uuid4()),
+                created_by=account.id,
+                status="completed",
+            )
+            segments.append(segment)
+
+        # Create upload file
+        upload_file = self._create_test_upload_file(db_session_with_containers, account)
+
+        # Update document to reference the upload file
+        document.data_source_info = json.dumps({"upload_file_id": upload_file.id})
+
+        # Add all to database
+        for segment in segments:
+            db.session.add(segment)
+        db.session.commit()
+
+        # Verify initial state
+        assert db.session.query(DocumentSegment).filter_by(document_id=document.id).count() == 3
+        assert db.session.query(UploadFile).filter_by(id=upload_file.id).first() is not None
+
+        # Store original IDs for verification
+        document_id = document.id
+        segment_ids = [seg.id for seg in segments]
+        file_id = upload_file.id
+
+        # Execute the task
+        batch_clean_document_task(
+            document_ids=[document_id], dataset_id=dataset.id, doc_form=dataset.doc_form, file_ids=[file_id]
+        )
+
+        # Verify that the task completed successfully
+        # The task should process all segments and clean up all associated resources
+
+        # Verify database cleanup
+        db.session.commit()
+
+        # Check that all segments are deleted
+        for segment_id in segment_ids:
+            deleted_segment = db.session.query(DocumentSegment).filter_by(id=segment_id).first()
+            assert deleted_segment is None
+
+        # Check that upload file is deleted
+        deleted_file = db.session.query(UploadFile).filter_by(id=file_id).first()
+        assert deleted_file is None
+
+        # Verify final database state
+        assert db.session.query(DocumentSegment).filter_by(document_id=document_id).count() == 0
+        assert db.session.query(UploadFile).filter_by(id=file_id).first() is None

web/app/components/app/configuration/index.tsx

@@ -850,84 +850,83 @@ const Configuration: FC = () => {
       <Loading type='area' />
     </div>
   }
-
+  const value = {
+    appId,
+    isAPIKeySet,
+    isTrailFinished: false,
+    mode,
+    modelModeType,
+    promptMode,
+    isAdvancedMode,
+    isAgent,
+    isOpenAI,
+    isFunctionCall,
+    collectionList,
+    setPromptMode,
+    canReturnToSimpleMode,
+    setCanReturnToSimpleMode,
+    chatPromptConfig,
+    completionPromptConfig,
+    currentAdvancedPrompt,
+    setCurrentAdvancedPrompt,
+    conversationHistoriesRole: completionPromptConfig.conversation_histories_role,
+    showHistoryModal,
+    setConversationHistoriesRole,
+    hasSetBlockStatus,
+    conversationId,
+    introduction,
+    setIntroduction,
+    suggestedQuestions,
+    setSuggestedQuestions,
+    setConversationId,
+    controlClearChatMessage,
+    setControlClearChatMessage,
+    prevPromptConfig,
+    setPrevPromptConfig,
+    moreLikeThisConfig,
+    setMoreLikeThisConfig,
+    suggestedQuestionsAfterAnswerConfig,
+    setSuggestedQuestionsAfterAnswerConfig,
+    speechToTextConfig,
+    setSpeechToTextConfig,
+    textToSpeechConfig,
+    setTextToSpeechConfig,
+    citationConfig,
+    setCitationConfig,
+    annotationConfig,
+    setAnnotationConfig,
+    moderationConfig,
+    setModerationConfig,
+    externalDataToolsConfig,
+    setExternalDataToolsConfig,
+    formattingChanged,
+    setFormattingChanged,
+    inputs,
+    setInputs,
+    query,
+    setQuery,
+    completionParams,
+    setCompletionParams,
+    modelConfig,
+    setModelConfig,
+    showSelectDataSet,
+    dataSets,
+    setDataSets,
+    datasetConfigs,
+    datasetConfigsRef,
+    setDatasetConfigs,
+    hasSetContextVar,
+    isShowVisionConfig,
+    visionConfig,
+    setVisionConfig: handleSetVisionConfig,
+    isAllowVideoUpload,
+    isShowDocumentConfig,
+    isShowAudioConfig,
+    rerankSettingModalOpen,
+    setRerankSettingModalOpen,
+  }
   return (
-    <ConfigContext.Provider value={{
-      appId,
-      isAPIKeySet,
-      isTrailFinished: false,
-      mode,
-      modelModeType,
-      promptMode,
-      isAdvancedMode,
-      isAgent,
-      isOpenAI,
-      isFunctionCall,
-      collectionList,
-      setPromptMode,
-      canReturnToSimpleMode,
-      setCanReturnToSimpleMode,
-      chatPromptConfig,
-      completionPromptConfig,
-      currentAdvancedPrompt,
-      setCurrentAdvancedPrompt,
-      conversationHistoriesRole: completionPromptConfig.conversation_histories_role,
-      showHistoryModal,
-      setConversationHistoriesRole,
-      hasSetBlockStatus,
-      conversationId,
-      introduction,
-      setIntroduction,
-      suggestedQuestions,
-      setSuggestedQuestions,
-      setConversationId,
-      controlClearChatMessage,
-      setControlClearChatMessage,
-      prevPromptConfig,
-      setPrevPromptConfig,
-      moreLikeThisConfig,
-      setMoreLikeThisConfig,
-      suggestedQuestionsAfterAnswerConfig,
-      setSuggestedQuestionsAfterAnswerConfig,
-      speechToTextConfig,
-      setSpeechToTextConfig,
-      textToSpeechConfig,
-      setTextToSpeechConfig,
-      citationConfig,
-      setCitationConfig,
-      annotationConfig,
-      setAnnotationConfig,
-      moderationConfig,
-      setModerationConfig,
-      externalDataToolsConfig,
-      setExternalDataToolsConfig,
-      formattingChanged,
-      setFormattingChanged,
-      inputs,
-      setInputs,
-      query,
-      setQuery,
-      completionParams,
-      setCompletionParams,
-      modelConfig,
-      setModelConfig,
-      showSelectDataSet,
-      dataSets,
-      setDataSets,
-      datasetConfigs,
-      datasetConfigsRef,
-      setDatasetConfigs,
-      hasSetContextVar,
-      isShowVisionConfig,
-      visionConfig,
-      setVisionConfig: handleSetVisionConfig,
-      isAllowVideoUpload,
-      isShowDocumentConfig,
-      isShowAudioConfig,
-      rerankSettingModalOpen,
-      setRerankSettingModalOpen,
-    }}
-    >
+    <ConfigContext.Provider value={value}>
       <FeaturesProvider features={featuresData}>
         <MittProvider>
           <div className="flex h-full flex-col">

web/app/install/installForm.tsx

@@ -134,7 +134,7 @@ const InstallForm = () => {
                   <input
                     {...register('email')}
                     placeholder={t('login.emailPlaceholder') || ''}
-                    className={'w-full appearance-none rounded-md border border-transparent bg-components-input-bg-normal py-[7px] pl-2 text-components-input-text-filled caret-primary-600 outline-none placeholder:text-components-input-text-placeholder hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border-components-input-border-active focus:bg-components-input-bg-active focus:shadow-xs'}
+                    className={'system-sm-regular w-full appearance-none rounded-md border border-transparent bg-components-input-bg-normal px-3 py-[7px] text-components-input-text-filled caret-primary-600 outline-none placeholder:text-components-input-text-placeholder hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border-components-input-border-active focus:bg-components-input-bg-active focus:shadow-xs'}
                   />
                   {errors.email && <span className='text-sm text-red-400'>{t(`${errors.email?.message}`)}</span>}
                 </div>
@@ -149,7 +149,7 @@ const InstallForm = () => {
                   <input
                     {...register('name')}
                     placeholder={t('login.namePlaceholder') || ''}
-                    className={'w-full appearance-none rounded-md border border-transparent bg-components-input-bg-normal py-[7px] pl-2 text-components-input-text-filled caret-primary-600 outline-none placeholder:text-components-input-text-placeholder hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border-components-input-border-active focus:bg-components-input-bg-active focus:shadow-xs'}
+                    className={'system-sm-regular w-full appearance-none rounded-md border border-transparent bg-components-input-bg-normal px-3 py-[7px] text-components-input-text-filled caret-primary-600 outline-none placeholder:text-components-input-text-placeholder hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border-components-input-border-active focus:bg-components-input-bg-active focus:shadow-xs'}
                   />
                 </div>
                 {errors.name && <span className='text-sm text-red-400'>{t(`${errors.name.message}`)}</span>}
@@ -164,7 +164,7 @@ const InstallForm = () => {
                     {...register('password')}
                     type={showPassword ? 'text' : 'password'}
                     placeholder={t('login.passwordPlaceholder') || ''}
-                    className={'w-full appearance-none rounded-md border border-transparent bg-components-input-bg-normal py-[7px] pl-2 text-components-input-text-filled caret-primary-600 outline-none placeholder:text-components-input-text-placeholder hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border-components-input-border-active focus:bg-components-input-bg-active focus:shadow-xs'}
+                    className={'system-sm-regular w-full appearance-none rounded-md border border-transparent bg-components-input-bg-normal px-3 py-[7px] text-components-input-text-filled caret-primary-600 outline-none placeholder:text-components-input-text-placeholder hover:border-components-input-border-hover hover:bg-components-input-bg-hover focus:border-components-input-border-active focus:bg-components-input-bg-active focus:shadow-xs'}
                   />
 
                   <div className="absolute inset-y-0 right-0 flex items-center pr-3">
@@ -178,7 +178,7 @@ const InstallForm = () => {
                   </div>
                 </div>
 
-                <div className={classNames('mt-1 text-xs text-text-tertiary', {
+                <div className={classNames('mt-1 text-xs text-text-secondary', {
                   'text-red-400 !text-sm': errors.password,
                 })}>{t('login.error.passwordInvalid')}</div>
               </div>
@@ -189,7 +189,7 @@ const InstallForm = () => {
                 </Button>
               </div>
             </form>
-            <div className="mt-2 block w-full text-xs text-text-tertiary">
+            <div className="mt-2 block w-full text-xs text-text-secondary">
               {t('login.license.tip')}
               &nbsp;
               <Link

web/app/layout.tsx

@@ -53,10 +53,17 @@ const LocaleLayout = async ({
   return (
     <html lang={locale ?? 'en'} className="h-full" suppressHydrationWarning>
       <head>
-        <meta name="theme-color" content="#FFFFFF" />
+        <link rel="manifest" href="/manifest.json" />
+        <meta name="theme-color" content="#1C64F2" />
         <meta name="mobile-web-app-capable" content="yes" />
         <meta name="apple-mobile-web-app-capable" content="yes" />
         <meta name="apple-mobile-web-app-status-bar-style" content="default" />
+        <meta name="apple-mobile-web-app-title" content="Dify" />
+        <link rel="apple-touch-icon" href="/apple-touch-icon.png" />
+        <link rel="icon" type="image/png" sizes="32x32" href="/icon-192x192.png" />
+        <link rel="icon" type="image/png" sizes="16x16" href="/icon-192x192.png" />
+        <meta name="msapplication-TileColor" content="#1C64F2" />
+        <meta name="msapplication-config" content="/browserconfig.xml" />
       </head>
       <body
         className="color-scheme h-full select-auto"

web/i18n/id-ID/workflow.ts

@@ -461,6 +461,12 @@ const translation = {
       contextTooltip: 'Anda dapat mengimpor Pengetahuan sebagai konteks',
       notSetContextInPromptTip: 'Untuk mengaktifkan fitur konteks, silakan isi variabel konteks di PROMPT.',
       context: 'konteks',
+      reasoningFormat: {
+        tagged: 'Tetap pikirkan tag',
+        title: 'Aktifkan pemisahan tag penalaran',
+        separated: 'Pisahkan tag pemikiran',
+        tooltip: 'Ekstrak konten dari tag pikir dan simpan di field reasoning_content.',
+      },
     },
     knowledgeRetrieval: {
       outputVars: {

web/next.config.js

@@ -1,4 +1,71 @@
 const { codeInspectorPlugin } = require('code-inspector-plugin')
+const withPWA = require('next-pwa')({
+  dest: 'public',
+  register: true,
+  skipWaiting: true,
+  disable: process.env.NODE_ENV === 'development',
+  fallbacks: {
+    document: '/_offline.html',
+  },
+  runtimeCaching: [
+    {
+      urlPattern: /^https:\/\/fonts\.googleapis\.com\/.*/i,
+      handler: 'CacheFirst',
+      options: {
+        cacheName: 'google-fonts',
+        expiration: {
+          maxEntries: 4,
+          maxAgeSeconds: 365 * 24 * 60 * 60 // 1 year
+        }
+      }
+    },
+    {
+      urlPattern: /^https:\/\/fonts\.gstatic\.com\/.*/i,
+      handler: 'CacheFirst',
+      options: {
+        cacheName: 'google-fonts-webfonts',
+        expiration: {
+          maxEntries: 4,
+          maxAgeSeconds: 365 * 24 * 60 * 60 // 1 year
+        }
+      }
+    },
+    {
+      urlPattern: /\.(?:png|jpg|jpeg|svg|gif|webp|avif)$/i,
+      handler: 'CacheFirst',
+      options: {
+        cacheName: 'images',
+        expiration: {
+          maxEntries: 64,
+          maxAgeSeconds: 30 * 24 * 60 * 60 // 30 days
+        }
+      }
+    },
+    {
+      urlPattern: /\.(?:js|css)$/i,
+      handler: 'StaleWhileRevalidate',
+      options: {
+        cacheName: 'static-resources',
+        expiration: {
+          maxEntries: 32,
+          maxAgeSeconds: 24 * 60 * 60 // 1 day
+        }
+      }
+    },
+    {
+      urlPattern: /^\/api\/.*/i,
+      handler: 'NetworkFirst',
+      options: {
+        cacheName: 'api-cache',
+        networkTimeoutSeconds: 10,
+        expiration: {
+          maxEntries: 16,
+          maxAgeSeconds: 60 * 60 // 1 hour
+        }
+      }
+    }
+  ]
+})
 const withMDX = require('@next/mdx')({
   extension: /\.mdx?$/,
   options: {
@@ -70,4 +137,4 @@ const nextConfig = {
   output: 'standalone',
 }
 
-module.exports = withBundleAnalyzer(withMDX(nextConfig))
+module.exports = withPWA(withBundleAnalyzer(withMDX(nextConfig)))

web/package.json

@@ -106,6 +106,7 @@
     "mitt": "^3.0.1",
     "negotiator": "^0.6.3",
     "next": "15.5.0",
+    "next-pwa": "^5.6.0",
     "next-themes": "^0.4.3",
     "pinyin-pro": "^3.25.0",
     "qrcode.react": "^4.2.0",
@@ -155,6 +156,8 @@
   },
   "devDependencies": {
     "@antfu/eslint-config": "^5.0.0",
+    "@babel/core": "^7.28.3",
+    "@babel/preset-env": "^7.28.3",
     "@chromatic-com/storybook": "^3.1.0",
     "@eslint-react/eslint-plugin": "^1.15.0",
     "@eslint/eslintrc": "^3.1.0",
@@ -198,6 +201,7 @@
     "@types/sortablejs": "^1.15.1",
     "@types/uuid": "^10.0.0",
     "autoprefixer": "^10.4.20",
+    "babel-loader": "^10.0.0",
     "bing-translate-api": "^4.0.2",
     "code-inspector-plugin": "^0.18.1",
     "cross-env": "^7.0.3",

web/public/_offline.html

@@ -0,0 +1,129 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Dify - Offline</title>
+    <style>
+        * {
+            margin: 0;
+            padding: 0;
+            box-sizing: border-box;
+        }
+        
+        body {
+            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
+            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+            min-height: 100vh;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            color: white;
+            text-align: center;
+            padding: 20px;
+        }
+        
+        .container {
+            max-width: 600px;
+            background: rgba(255, 255, 255, 0.1);
+            backdrop-filter: blur(10px);
+            border-radius: 20px;
+            padding: 40px;
+            box-shadow: 0 25px 50px rgba(0, 0, 0, 0.2);
+        }
+        
+        .icon {
+            width: 100px;
+            height: 100px;
+            margin: 0 auto 30px;
+            background: rgba(255, 255, 255, 0.2);
+            border-radius: 20px;
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            font-size: 48px;
+        }
+        
+        h1 {
+            font-size: 32px;
+            font-weight: 600;
+            margin-bottom: 15px;
+        }
+        
+        p {
+            font-size: 18px;
+            line-height: 1.6;
+            opacity: 0.9;
+            margin-bottom: 30px;
+        }
+        
+        button {
+            background: white;
+            color: #764ba2;
+            border: none;
+            padding: 15px 30px;
+            font-size: 16px;
+            font-weight: 600;
+            border-radius: 50px;
+            cursor: pointer;
+            transition: transform 0.2s, box-shadow 0.2s;
+        }
+        
+        button:hover {
+            transform: translateY(-2px);
+            box-shadow: 0 10px 25px rgba(0, 0, 0, 0.2);
+        }
+        
+        button:active {
+            transform: translateY(0);
+        }
+        
+        @media (max-width: 640px) {
+            .container {
+                padding: 30px;
+            }
+            
+            h1 {
+                font-size: 24px;
+            }
+            
+            p {
+                font-size: 16px;
+            }
+        }
+    </style>
+</head>
+<body>
+    <div class="container">
+        <div class="icon">
+            ⚡
+        </div>
+        <h1>You're Offline</h1>
+        <p>
+            It looks like you've lost your internet connection. 
+            Some features may not be available until you're back online.
+        </p>
+        <button onclick="window.location.reload()">
+            Try Again
+        </button>
+    </div>
+    
+    <script>
+        // Check for connection status changes
+        window.addEventListener('online', function() {
+            window.location.reload();
+        });
+        
+        // Periodically check if online
+        setInterval(function() {
+            fetch(window.location.origin, { method: 'HEAD' })
+                .then(function() {
+                    window.location.reload();
+                })
+                .catch(function() {
+                    // Still offline
+                });
+        }, 5000);
+    </script>
+</body>
+</html>

web/public/browserconfig.xml

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<browserconfig>
+    <msapplication>
+        <tile>
+            <square70x70logo src="/icon-72x72.png"/>
+            <square150x150logo src="/icon-152x152.png"/>
+            <square310x310logo src="/icon-512x512.png"/>
+            <TileColor>#1C64F2</TileColor>
+        </tile>
+    </msapplication>
+</browserconfig>

web/public/fallback-hxi5kegOl0PxtKhvDL_OX.js

@@ -0,0 +1 @@
+(()=>{"use strict";self.fallback=async e=>"document"===e.destination?caches.match("/_offline.html",{ignoreSearch:!0}):Response.error()})();

web/public/manifest.json

@@ -0,0 +1,58 @@
+{
+  "name": "Dify",
+  "short_name": "Dify",
+  "description": "Build Production Ready Agentic AI Solutions",
+  "icons": [
+    {
+      "src": "/icon-192x192.png",
+      "sizes": "192x192",
+      "type": "image/png",
+      "purpose": "any"
+    },
+    {
+      "src": "/icon-192x192.png",
+      "sizes": "192x192",
+      "type": "image/png",
+      "purpose": "maskable"
+    },
+    {
+      "src": "/icon-256x256.png",
+      "sizes": "256x256",
+      "type": "image/png"
+    },
+    {
+      "src": "/icon-384x384.png",
+      "sizes": "384x384",
+      "type": "image/png"
+    },
+    {
+      "src": "/icon-512x512.png",
+      "sizes": "512x512",
+      "type": "image/png"
+    }
+  ],
+  "theme_color": "#1C64F2",
+  "background_color": "#ffffff",
+  "display": "standalone",
+  "scope": "/",
+  "start_url": "/",
+  "orientation": "portrait-primary",
+  "categories": ["productivity", "utilities", "developer"],
+  "lang": "en-US",
+  "dir": "ltr",
+  "prefer_related_applications": false,
+  "shortcuts": [
+    {
+      "name": "Apps",
+      "short_name": "Apps",
+      "url": "/apps",
+      "icons": [{ "src": "/icon-96x96.png", "sizes": "96x96" }]
+    },
+    {
+      "name": "Datasets",
+      "short_name": "Datasets",
+      "url": "/datasets",
+      "icons": [{ "src": "/icon-96x96.png", "sizes": "96x96" }]
+    }
+  ]
+}

web/scripts/generate-icons.js

@@ -0,0 +1,51 @@
+const sharp = require('sharp');
+const fs = require('fs');
+const path = require('path');
+
+const sizes = [
+  { size: 192, name: 'icon-192x192.png' },
+  { size: 256, name: 'icon-256x256.png' },
+  { size: 384, name: 'icon-384x384.png' },
+  { size: 512, name: 'icon-512x512.png' },
+  { size: 96, name: 'icon-96x96.png' },
+  { size: 72, name: 'icon-72x72.png' },
+  { size: 128, name: 'icon-128x128.png' },
+  { size: 144, name: 'icon-144x144.png' },
+  { size: 152, name: 'icon-152x152.png' },
+];
+
+const inputPath = path.join(__dirname, '../public/icon.svg');
+const outputDir = path.join(__dirname, '../public');
+
+// Generate icons
+async function generateIcons() {
+  try {
+    console.log('Generating PWA icons...');
+    
+    for (const { size, name } of sizes) {
+      const outputPath = path.join(outputDir, name);
+      
+      await sharp(inputPath)
+        .resize(size, size)
+        .png()
+        .toFile(outputPath);
+      
+      console.log(`✓ Generated ${name} (${size}x${size})`);
+    }
+    
+    // Generate apple-touch-icon
+    await sharp(inputPath)
+      .resize(180, 180)
+      .png()
+      .toFile(path.join(outputDir, 'apple-touch-icon.png'));
+    
+    console.log('✓ Generated apple-touch-icon.png (180x180)');
+    
+    console.log('\n✅ All icons generated successfully!');
+  } catch (error) {
+    console.error('Error generating icons:', error);
+    process.exit(1);
+  }
+}
+
+generateIcons();