feat(workflow): add configurable workflow file upload limit (#10176)

Co-authored-by: JzoNg <jzongcode@gmail.com>

api/.env.example

@@ -327,6 +327,9 @@ SSRF_DEFAULT_MAX_RETRIES=3
 BATCH_UPLOAD_LIMIT=10
 KEYWORD_DATA_SOURCE_TYPE=database
 
+# Workflow file upload limit
+WORKFLOW_FILE_UPLOAD_LIMIT=10
+
 # CODE EXECUTION CONFIGURATION
 CODE_EXECUTION_ENDPOINT=http://127.0.0.1:8194
 CODE_EXECUTION_API_KEY=dify-sandbox

api/configs/feature/__init__.py

@@ -216,6 +216,11 @@ class FileUploadConfig(BaseSettings):
         default=20,
     )
 
+    WORKFLOW_FILE_UPLOAD_LIMIT: PositiveInt = Field(
+        description="Maximum number of files allowed in a workflow upload operation",
+        default=10,
+    )
+
 
 class HttpConfig(BaseSettings):
     """

api/controllers/common/fields.py

@@ -0,0 +1,24 @@
+from flask_restful import fields
+
+parameters__system_parameters = {
+    "image_file_size_limit": fields.Integer,
+    "video_file_size_limit": fields.Integer,
+    "audio_file_size_limit": fields.Integer,
+    "file_size_limit": fields.Integer,
+    "workflow_file_upload_limit": fields.Integer,
+}
+
+parameters_fields = {
+    "opening_statement": fields.String,
+    "suggested_questions": fields.Raw,
+    "suggested_questions_after_answer": fields.Raw,
+    "speech_to_text": fields.Raw,
+    "text_to_speech": fields.Raw,
+    "retriever_resource": fields.Raw,
+    "annotation_reply": fields.Raw,
+    "more_like_this": fields.Raw,
+    "user_input_form": fields.Raw,
+    "sensitive_word_avoidance": fields.Raw,
+    "file_upload": fields.Raw,
+    "system_parameters": fields.Nested(parameters__system_parameters),
+}

api/controllers/common/helpers.py

@@ -2,11 +2,15 @@ import mimetypes
 import os
 import re
 import urllib.parse
+from collections.abc import Mapping
+from typing import Any
 from uuid import uuid4
 
 import httpx
 from pydantic import BaseModel
 
+from configs import dify_config
+
 
 class FileInfo(BaseModel):
     filename: str
@@ -56,3 +60,38 @@ def guess_file_info_from_response(response: httpx.Response):
         mimetype=mimetype,
         size=int(response.headers.get("Content-Length", -1)),
     )
+
+
+def get_parameters_from_feature_dict(*, features_dict: Mapping[str, Any], user_input_form: list[dict[str, Any]]):
+    return {
+        "opening_statement": features_dict.get("opening_statement"),
+        "suggested_questions": features_dict.get("suggested_questions", []),
+        "suggested_questions_after_answer": features_dict.get("suggested_questions_after_answer", {"enabled": False}),
+        "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
+        "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
+        "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
+        "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
+        "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
+        "user_input_form": user_input_form,
+        "sensitive_word_avoidance": features_dict.get(
+            "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
+        ),
+        "file_upload": features_dict.get(
+            "file_upload",
+            {
+                "image": {
+                    "enabled": False,
+                    "number_limits": 3,
+                    "detail": "high",
+                    "transfer_methods": ["remote_url", "local_file"],
+                }
+            },
+        ),
+        "system_parameters": {
+            "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
+            "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
+            "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
+            "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
+            "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
+        },
+    }

api/controllers/console/explore/parameter.py

@@ -1,6 +1,7 @@
-from flask_restful import fields, marshal_with
+from flask_restful import marshal_with
 
-from configs import dify_config
+from controllers.common import fields
+from controllers.common import helpers as controller_helpers
 from controllers.console import api
 from controllers.console.app.error import AppUnavailableError
 from controllers.console.explore.wraps import InstalledAppResource
@@ -11,43 +12,14 @@ from services.app_service import AppService
 class AppParameterApi(InstalledAppResource):
     """Resource for app variables."""
 
-    variable_fields = {
-        "key": fields.String,
-        "name": fields.String,
-        "description": fields.String,
-        "type": fields.String,
-        "default": fields.String,
-        "max_length": fields.Integer,
-        "options": fields.List(fields.String),
-    }
-
-    system_parameters_fields = {
-        "image_file_size_limit": fields.Integer,
-        "video_file_size_limit": fields.Integer,
-        "audio_file_size_limit": fields.Integer,
-        "file_size_limit": fields.Integer,
-    }
-
-    parameters_fields = {
-        "opening_statement": fields.String,
-        "suggested_questions": fields.Raw,
-        "suggested_questions_after_answer": fields.Raw,
-        "speech_to_text": fields.Raw,
-        "text_to_speech": fields.Raw,
-        "retriever_resource": fields.Raw,
-        "annotation_reply": fields.Raw,
-        "more_like_this": fields.Raw,
-        "user_input_form": fields.Raw,
-        "sensitive_word_avoidance": fields.Raw,
-        "file_upload": fields.Raw,
-        "system_parameters": fields.Nested(system_parameters_fields),
-    }
-
-    @marshal_with(parameters_fields)
+    @marshal_with(fields.parameters_fields)
     def get(self, installed_app: InstalledApp):
         """Retrieve app parameters."""
         app_model = installed_app.app
 
+        if app_model is None:
+            raise AppUnavailableError()
+
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
             workflow = app_model.workflow
             if workflow is None:
@@ -57,43 +29,16 @@ class AppParameterApi(InstalledAppResource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
+            if app_model_config is None:
+                raise AppUnavailableError()
+
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return {
-            "opening_statement": features_dict.get("opening_statement"),
-            "suggested_questions": features_dict.get("suggested_questions", []),
-            "suggested_questions_after_answer": features_dict.get(
-                "suggested_questions_after_answer", {"enabled": False}
-            ),
-            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
-            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
-            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
-            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
-            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
-            "user_input_form": user_input_form,
-            "sensitive_word_avoidance": features_dict.get(
-                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
-            ),
-            "file_upload": features_dict.get(
-                "file_upload",
-                {
-                    "image": {
-                        "enabled": False,
-                        "number_limits": 3,
-                        "detail": "high",
-                        "transfer_methods": ["remote_url", "local_file"],
-                    }
-                },
-            ),
-            "system_parameters": {
-                "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
-                "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
-                "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-                "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
-            },
-        }
+        return controller_helpers.get_parameters_from_feature_dict(
+            features_dict=features_dict, user_input_form=user_input_form
+        )
 
 
 class ExploreAppMetaApi(InstalledAppResource):

api/controllers/console/files/__init__.py

@@ -37,6 +37,7 @@ class FileApi(Resource):
             "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
             "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
             "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
+            "workflow_file_upload_limit": dify_config.WORKFLOW_FILE_UPLOAD_LIMIT,
         }, 200
 
     @setup_required

api/controllers/service_api/app/app.py

@@ -1,6 +1,7 @@
-from flask_restful import Resource, fields, marshal_with
+from flask_restful import Resource, marshal_with
 
-from configs import dify_config
+from controllers.common import fields
+from controllers.common import helpers as controller_helpers
 from controllers.service_api import api
 from controllers.service_api.app.error import AppUnavailableError
 from controllers.service_api.wraps import validate_app_token
@@ -11,40 +12,8 @@ from services.app_service import AppService
 class AppParameterApi(Resource):
     """Resource for app variables."""
 
-    variable_fields = {
-        "key": fields.String,
-        "name": fields.String,
-        "description": fields.String,
-        "type": fields.String,
-        "default": fields.String,
-        "max_length": fields.Integer,
-        "options": fields.List(fields.String),
-    }
-
-    system_parameters_fields = {
-        "image_file_size_limit": fields.Integer,
-        "video_file_size_limit": fields.Integer,
-        "audio_file_size_limit": fields.Integer,
-        "file_size_limit": fields.Integer,
-    }
-
-    parameters_fields = {
-        "opening_statement": fields.String,
-        "suggested_questions": fields.Raw,
-        "suggested_questions_after_answer": fields.Raw,
-        "speech_to_text": fields.Raw,
-        "text_to_speech": fields.Raw,
-        "retriever_resource": fields.Raw,
-        "annotation_reply": fields.Raw,
-        "more_like_this": fields.Raw,
-        "user_input_form": fields.Raw,
-        "sensitive_word_avoidance": fields.Raw,
-        "file_upload": fields.Raw,
-        "system_parameters": fields.Nested(system_parameters_fields),
-    }
-
     @validate_app_token
-    @marshal_with(parameters_fields)
+    @marshal_with(fields.parameters_fields)
     def get(self, app_model: App):
         """Retrieve app parameters."""
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
@@ -56,43 +25,16 @@ class AppParameterApi(Resource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
+            if app_model_config is None:
+                raise AppUnavailableError()
+
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return {
-            "opening_statement": features_dict.get("opening_statement"),
-            "suggested_questions": features_dict.get("suggested_questions", []),
-            "suggested_questions_after_answer": features_dict.get(
-                "suggested_questions_after_answer", {"enabled": False}
-            ),
-            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
-            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
-            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
-            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
-            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
-            "user_input_form": user_input_form,
-            "sensitive_word_avoidance": features_dict.get(
-                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
-            ),
-            "file_upload": features_dict.get(
-                "file_upload",
-                {
-                    "image": {
-                        "enabled": False,
-                        "number_limits": 3,
-                        "detail": "high",
-                        "transfer_methods": ["remote_url", "local_file"],
-                    }
-                },
-            ),
-            "system_parameters": {
-                "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
-                "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
-                "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-                "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
-            },
-        }
+        return controller_helpers.get_parameters_from_feature_dict(
+            features_dict=features_dict, user_input_form=user_input_form
+        )
 
 
 class AppMetaApi(Resource):

api/controllers/web/app.py

@@ -1,6 +1,7 @@
-from flask_restful import fields, marshal_with
+from flask_restful import marshal_with
 
-from configs import dify_config
+from controllers.common import fields
+from controllers.common import helpers as controller_helpers
 from controllers.web import api
 from controllers.web.error import AppUnavailableError
 from controllers.web.wraps import WebApiResource
@@ -11,39 +12,7 @@ from services.app_service import AppService
 class AppParameterApi(WebApiResource):
     """Resource for app variables."""
 
-    variable_fields = {
-        "key": fields.String,
-        "name": fields.String,
-        "description": fields.String,
-        "type": fields.String,
-        "default": fields.String,
-        "max_length": fields.Integer,
-        "options": fields.List(fields.String),
-    }
-
-    system_parameters_fields = {
-        "image_file_size_limit": fields.Integer,
-        "video_file_size_limit": fields.Integer,
-        "audio_file_size_limit": fields.Integer,
-        "file_size_limit": fields.Integer,
-    }
-
-    parameters_fields = {
-        "opening_statement": fields.String,
-        "suggested_questions": fields.Raw,
-        "suggested_questions_after_answer": fields.Raw,
-        "speech_to_text": fields.Raw,
-        "text_to_speech": fields.Raw,
-        "retriever_resource": fields.Raw,
-        "annotation_reply": fields.Raw,
-        "more_like_this": fields.Raw,
-        "user_input_form": fields.Raw,
-        "sensitive_word_avoidance": fields.Raw,
-        "file_upload": fields.Raw,
-        "system_parameters": fields.Nested(system_parameters_fields),
-    }
-
-    @marshal_with(parameters_fields)
+    @marshal_with(fields.parameters_fields)
     def get(self, app_model: App, end_user):
         """Retrieve app parameters."""
         if app_model.mode in {AppMode.ADVANCED_CHAT.value, AppMode.WORKFLOW.value}:
@@ -55,43 +24,16 @@ class AppParameterApi(WebApiResource):
             user_input_form = workflow.user_input_form(to_old_structure=True)
         else:
             app_model_config = app_model.app_model_config
+            if app_model_config is None:
+                raise AppUnavailableError()
+
             features_dict = app_model_config.to_dict()
 
             user_input_form = features_dict.get("user_input_form", [])
 
-        return {
-            "opening_statement": features_dict.get("opening_statement"),
-            "suggested_questions": features_dict.get("suggested_questions", []),
-            "suggested_questions_after_answer": features_dict.get(
-                "suggested_questions_after_answer", {"enabled": False}
-            ),
-            "speech_to_text": features_dict.get("speech_to_text", {"enabled": False}),
-            "text_to_speech": features_dict.get("text_to_speech", {"enabled": False}),
-            "retriever_resource": features_dict.get("retriever_resource", {"enabled": False}),
-            "annotation_reply": features_dict.get("annotation_reply", {"enabled": False}),
-            "more_like_this": features_dict.get("more_like_this", {"enabled": False}),
-            "user_input_form": user_input_form,
-            "sensitive_word_avoidance": features_dict.get(
-                "sensitive_word_avoidance", {"enabled": False, "type": "", "configs": []}
-            ),
-            "file_upload": features_dict.get(
-                "file_upload",
-                {
-                    "image": {
-                        "enabled": False,
-                        "number_limits": 3,
-                        "detail": "high",
-                        "transfer_methods": ["remote_url", "local_file"],
-                    }
-                },
-            ),
-            "system_parameters": {
-                "image_file_size_limit": dify_config.UPLOAD_IMAGE_FILE_SIZE_LIMIT,
-                "video_file_size_limit": dify_config.UPLOAD_VIDEO_FILE_SIZE_LIMIT,
-                "audio_file_size_limit": dify_config.UPLOAD_AUDIO_FILE_SIZE_LIMIT,
-                "file_size_limit": dify_config.UPLOAD_FILE_SIZE_LIMIT,
-            },
-        }
+        return controller_helpers.get_parameters_from_feature_dict(
+            features_dict=features_dict, user_input_form=user_input_form
+        )
 
 
 class AppMeta(WebApiResource):

api/core/app/app_config/features/file_upload/manager.py

@@ -1,8 +1,7 @@
 from collections.abc import Mapping
 from typing import Any
 
-from core.file.models import FileExtraConfig
-from models import FileUploadConfig
+from core.file import FileExtraConfig
 
 
 class FileUploadConfigManager:
@@ -43,6 +42,6 @@ class FileUploadConfigManager:
         if not config.get("file_upload"):
             config["file_upload"] = {}
         else:
-            FileUploadConfig.model_validate(config["file_upload"])
+            FileExtraConfig.model_validate(config["file_upload"])
 
         return config, ["file_upload"]

api/fields/file_fields.py

@@ -8,6 +8,7 @@ upload_config_fields = {
     "image_file_size_limit": fields.Integer,
     "video_file_size_limit": fields.Integer,
     "audio_file_size_limit": fields.Integer,
+    "workflow_file_upload_limit": fields.Integer,
 }
 
 file_fields = {

api/models/__init__.py

@@ -6,7 +6,6 @@ from .model import (
     AppMode,
     Conversation,
     EndUser,
-    FileUploadConfig,
     InstalledApp,
     Message,
     MessageAnnotation,
@@ -50,6 +49,5 @@ __all__ = [
     "Tenant",
     "Conversation",
     "MessageAnnotation",
-    "FileUploadConfig",
     "ToolFile",
 ]

api/models/model.py

@@ -1,7 +1,7 @@
 import json
 import re
 import uuid
-from collections.abc import Mapping, Sequence
+from collections.abc import Mapping
 from datetime import datetime
 from enum import Enum
 from typing import Any, Literal, Optional
@@ -9,7 +9,6 @@ from typing import Any, Literal, Optional
 import sqlalchemy as sa
 from flask import request
 from flask_login import UserMixin
-from pydantic import BaseModel, Field
 from sqlalchemy import Float, func, text
 from sqlalchemy.orm import Mapped, mapped_column
 
@@ -25,14 +24,6 @@ from .account import Account, Tenant
 from .types import StringUUID
 
 
-class FileUploadConfig(BaseModel):
-    enabled: bool = Field(default=False)
-    allowed_file_types: Sequence[FileType] = Field(default_factory=list)
-    allowed_extensions: Sequence[str] = Field(default_factory=list)
-    allowed_upload_methods: Sequence[FileTransferMethod] = Field(default_factory=list)
-    number_limits: int = Field(default=0, gt=0, le=10)
-
-
 class DifySetup(db.Model):
     __tablename__ = "dify_setups"
     __table_args__ = (db.PrimaryKeyConstraint("version", name="dify_setup_pkey"),)
@@ -115,7 +106,7 @@ class App(db.Model):
         return site
 
     @property
-    def app_model_config(self) -> Optional["AppModelConfig"]:
+    def app_model_config(self):
         if self.app_model_config_id:
             return db.session.query(AppModelConfig).filter(AppModelConfig.id == self.app_model_config_id).first()
 

docker/.env.example

@@ -690,6 +690,7 @@ WORKFLOW_MAX_EXECUTION_STEPS=500
 WORKFLOW_MAX_EXECUTION_TIME=1200
 WORKFLOW_CALL_MAX_DEPTH=5
 MAX_VARIABLE_SIZE=204800
+WORKFLOW_FILE_UPLOAD_LIMIT=10
 
 # HTTP request node in workflow configuration
 HTTP_REQUEST_NODE_MAX_BINARY_SIZE=10485760

docker/docker-compose.yaml

@@ -1,4 +1,5 @@
 x-shared-env: &shared-api-worker-env
+  WORKFLOW_FILE_UPLOAD_LIMIT: ${WORKFLOW_FILE_UPLOAD_LIMIT:-10}
   LOG_LEVEL: ${LOG_LEVEL:-INFO}
   LOG_FILE: ${LOG_FILE:-}
   LOG_FILE_MAX_SIZE: ${LOG_FILE_MAX_SIZE:-20}

web/app/components/base/file-uploader/constants.ts

@@ -3,5 +3,6 @@ export const IMG_SIZE_LIMIT = 10 * 1024 * 1024
 export const FILE_SIZE_LIMIT = 15 * 1024 * 1024
 export const AUDIO_SIZE_LIMIT = 50 * 1024 * 1024
 export const VIDEO_SIZE_LIMIT = 100 * 1024 * 1024
+export const MAX_FILE_UPLOAD_LIMIT = 10
 
 export const FILE_URL_REGEX = /^(https?|ftp):\/\//

web/app/components/base/file-uploader/hooks.ts

@@ -18,6 +18,7 @@ import {
   AUDIO_SIZE_LIMIT,
   FILE_SIZE_LIMIT,
   IMG_SIZE_LIMIT,
+  MAX_FILE_UPLOAD_LIMIT,
   VIDEO_SIZE_LIMIT,
 } from '@/app/components/base/file-uploader/constants'
 import { useToastContext } from '@/app/components/base/toast'
@@ -33,12 +34,14 @@ export const useFileSizeLimit = (fileUploadConfig?: FileUploadConfigResponse) =>
   const docSizeLimit = Number(fileUploadConfig?.file_size_limit) * 1024 * 1024 || FILE_SIZE_LIMIT
   const audioSizeLimit = Number(fileUploadConfig?.audio_file_size_limit) * 1024 * 1024 || AUDIO_SIZE_LIMIT
   const videoSizeLimit = Number(fileUploadConfig?.video_file_size_limit) * 1024 * 1024 || VIDEO_SIZE_LIMIT
+  const maxFileUploadLimit = Number(fileUploadConfig?.workflow_file_upload_limit) || MAX_FILE_UPLOAD_LIMIT
 
   return {
     imgSizeLimit,
     docSizeLimit,
     audioSizeLimit,
     videoSizeLimit,
+    maxFileUploadLimit,
   }
 }
 

web/app/components/workflow/nodes/_base/components/file-upload-setting.tsx

@@ -39,7 +39,13 @@ const FileUploadSetting: FC<Props> = ({
     allowed_file_extensions,
   } = payload
   const { data: fileUploadConfigResponse } = useSWR({ url: '/files/upload' }, fetchFileUploadConfig)
-  const { imgSizeLimit, docSizeLimit, audioSizeLimit, videoSizeLimit } = useFileSizeLimit(fileUploadConfigResponse)
+  const {
+    imgSizeLimit,
+    docSizeLimit,
+    audioSizeLimit,
+    videoSizeLimit,
+    maxFileUploadLimit,
+  } = useFileSizeLimit(fileUploadConfigResponse)
 
   const handleSupportFileTypeChange = useCallback((type: SupportUploadFileTypes) => {
     const newPayload = produce(payload, (draft) => {
@@ -156,7 +162,7 @@ const FileUploadSetting: FC<Props> = ({
             <InputNumberWithSlider
               value={max_length}
               min={1}
-              max={10}
+              max={maxFileUploadLimit}
               onChange={handleMaxUploadNumLimitChange}
             />
           </div>

web/models/common.ts

@@ -216,7 +216,7 @@ export type FileUploadConfigResponse = {
   file_size_limit: number // default is 15MB
   audio_file_size_limit?: number // default is 50MB
   video_file_size_limit?: number // default is 100MB
-
+  workflow_file_upload_limit?: number // default is 10
 }
 
 export type InvitationResult = {