feat: support LLM process document file (#10966)

Co-authored-by: -LAN- <laipz8200@outlook.com>

api/core/memory/token_buffer_memory.py

 
 from core.app.app_config.features.file_upload.manager import FileUploadConfigManager
 from core.file import file_manager
-from core.file.models import FileType
 from core.model_manager import ModelInstance
 from core.model_runtime.entities import (
     AssistantPromptMessage,
                     prompt_message_contents: list[PromptMessageContent] = []
                     prompt_message_contents.append(TextPromptMessageContent(data=message.query))
                     for file in file_objs:
-                        if file.type in {FileType.IMAGE, FileType.AUDIO}:
-                            prompt_message = file_manager.to_prompt_message_content(
-                                file,
-                                image_detail_config=detail,
-                            )
-                            prompt_message_contents.append(prompt_message)
+                        prompt_message = file_manager.to_prompt_message_content(
+                            file,
+                            image_detail_config=detail,
+                        )
+                        prompt_message_contents.append(prompt_message)
 
                     prompt_messages.append(UserPromptMessage(content=prompt_message_contents))
 

api/core/model_runtime/entities/message_entities.py

     function: PromptMessageTool
 
 
-class PromptMessageContentType(Enum):
+class PromptMessageContentType(str, Enum):
     """
     Enum class for prompt message content type.
     """

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-001.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-002.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0827.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-8b-exp-0924.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-exp-0827.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash-latest.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-flash.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 1048576

api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-001.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 2097152

api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-002.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 2097152

api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0801.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 2097152

api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-exp-0827.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 2097152

api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro-latest.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 2097152

api/core/model_runtime/model_providers/google/llm/gemini-1.5-pro.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 2097152

api/core/model_runtime/model_providers/google/llm/gemini-exp-1114.yaml

   - vision
   - tool-call
   - stream-tool-call
+  - document
 model_properties:
   mode: chat
   context_size: 32767

api/core/model_runtime/model_providers/google/llm/llm.py

 from core.model_runtime.entities.llm_entities import LLMResult, LLMResultChunk, LLMResultChunkDelta
 from core.model_runtime.entities.message_entities import (
     AssistantPromptMessage,
+    DocumentPromptMessageContent,
     ImagePromptMessageContent,
     PromptMessage,
     PromptMessageContentType,
 from core.model_runtime.errors.validate import CredentialsValidateFailedError
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 
+GOOGLE_AVAILABLE_MIMETYPE = [
+    "application/pdf",
+    "application/x-javascript",
+    "text/javascript",
+    "application/x-python",
+    "text/x-python",
+    "text/plain",
+    "text/html",
+    "text/css",
+    "text/md",
+    "text/csv",
+    "text/xml",
+    "text/rtf",
+]
+
 
 class GoogleLargeLanguageModel(LargeLanguageModel):
     def _invoke(
                                 raise ValueError(f"Failed to fetch image data from url {message_content.data}, {ex}")
                         blob = {"inline_data": {"mime_type": mime_type, "data": base64_data}}
                         glm_content["parts"].append(blob)
+                    elif c.type == PromptMessageContentType.DOCUMENT:
+                        message_content = cast(DocumentPromptMessageContent, c)
+                        if message_content.mime_type not in GOOGLE_AVAILABLE_MIMETYPE:
+                            raise ValueError(f"Unsupported mime type {message_content.mime_type}")
+                        blob = {"inline_data": {"mime_type": message_content.mime_type, "data": message_content.data}}
+                        glm_content["parts"].append(blob)
 
             return glm_content
         elif isinstance(message, AssistantPromptMessage):

api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max-0809.yaml

 features:
   - vision
   - agent-thought
+  - video
 model_properties:
   mode: chat
   context_size: 32000

api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-max.yaml

 features:
   - vision
   - agent-thought
+  - video
 model_properties:
   mode: chat
   context_size: 32000

api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus-0809.yaml

 features:
   - vision
   - agent-thought
+  - video
 model_properties:
   mode: chat
   context_size: 32768

api/core/model_runtime/model_providers/tongyi/llm/qwen-vl-plus.yaml

 features:
   - vision
   - agent-thought
+  - video
 model_properties:
   mode: chat
   context_size: 8000

api/core/model_runtime/model_providers/zhipuai/llm/glm_4v_plus.yaml

   mode: chat
 features:
   - vision
+  - video
 parameter_rules:
   - name: temperature
     use_template: temperature

api/core/workflow/nodes/llm/exc.py

     """Raised when no prompt is found in the LLM configuration."""
 
 
-class NotSupportedPromptTypeError(LLMNodeError):
-    """Raised when the prompt type is not supported."""
+class TemplateTypeNotSupportError(LLMNodeError):
+    def __init__(self, *, type_name: str):
+        super().__init__(f"Prompt type {type_name} is not supported.")
 
 
 class MemoryRolePrefixRequiredError(LLMNodeError):
     """Raised when memory role prefix is required for completion model."""
+
+
+class FileTypeNotSupportError(LLMNodeError):
+    def __init__(self, *, type_name: str):
+        super().__init__(f"{type_name} type is not supported by this model")

api/core/workflow/nodes/llm/node.py

     ModelConfig,
 )
 from .exc import (
+    FileTypeNotSupportError,
     InvalidContextStructureError,
     InvalidVariableTypeError,
     LLMModeRequiredError,
     MemoryRolePrefixRequiredError,
     ModelNotExistError,
     NoPromptFoundError,
-    NotSupportedPromptTypeError,
+    TemplateTypeNotSupportError,
     VariableNotFoundError,
 )
 
                 prompt_content = prompt_messages[0].content.replace("#sys.query#", user_query)
                 prompt_messages[0].content = prompt_content
         else:
-            errmsg = f"Prompt type {type(prompt_template)} is not supported"
-            logger.warning(errmsg)
-            raise NotSupportedPromptTypeError(errmsg)
+            raise TemplateTypeNotSupportError(type_name=str(type(prompt_template)))
 
         if vision_enabled and user_files:
             file_prompts = []
                             and ModelFeature.AUDIO not in model_config.model_schema.features
                         )
                     ):
-                        continue
+                        raise FileTypeNotSupportError(type_name=content_item.type)
                     prompt_message_content.append(content_item)
                 if len(prompt_message_content) == 1 and prompt_message_content[0].type == PromptMessageContentType.TEXT:
                     prompt_message.content = prompt_message_content[0].data

api/tests/unit_tests/core/workflow/nodes/llm/test_node.py

                 )
             },
         ),
-        LLMNodeTestScenario(
-            description="Prompt template with variable selector of File without vision feature",
-            user_query=fake_query,
-            user_files=[],
-            vision_enabled=True,
-            vision_detail=fake_vision_detail,
-            features=[],
-            window_size=fake_window_size,
-            prompt_template=[
-                LLMNodeChatModelMessage(
-                    text="{{#input.image#}}",
-                    role=PromptMessageRole.USER,
-                    edition_type="basic",
-                ),
-            ],
-            expected_messages=mock_history[fake_window_size * -2 :] + [UserPromptMessage(content=fake_query)],
-            file_variables={
-                "input.image": File(
-                    tenant_id="test",
-                    type=FileType.IMAGE,
-                    filename="test1.jpg",
-                    transfer_method=FileTransferMethod.REMOTE_URL,
-                    remote_url=fake_remote_url,
-                )
-            },
-        ),
-        LLMNodeTestScenario(
-            description="Prompt template with variable selector of File with video file and vision feature",
-            user_query=fake_query,
-            user_files=[],
-            vision_enabled=True,
-            vision_detail=fake_vision_detail,
-            features=[ModelFeature.VISION],
-            window_size=fake_window_size,
-            prompt_template=[
-                LLMNodeChatModelMessage(
-                    text="{{#input.image#}}",
-                    role=PromptMessageRole.USER,
-                    edition_type="basic",
-                ),
-            ],
-            expected_messages=mock_history[fake_window_size * -2 :] + [UserPromptMessage(content=fake_query)],
-            file_variables={
-                "input.image": File(
-                    tenant_id="test",
-                    type=FileType.VIDEO,
-                    filename="test1.mp4",
-                    transfer_method=FileTransferMethod.REMOTE_URL,
-                    remote_url=fake_remote_url,
-                    extension="mp4",
-                )
-            },
-        ),
     ]
 
     for scenario in test_scenarios:

web/app/components/app/configuration/config-vision/index.tsx

 // import { Resolution } from '@/types/app'
 import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks'
 import Switch from '@/app/components/base/switch'
-import type { FileUpload } from '@/app/components/base/features/types'
+import { SupportUploadFileTypes } from '@/app/components/workflow/types'
 
 const ConfigVision: FC = () => {
   const { t } = useTranslation()
-  const { isShowVisionConfig } = useContext(ConfigContext)
+  const { isShowVisionConfig, isAllowVideoUpload } = useContext(ConfigContext)
   const file = useFeatures(s => s.features.file)
   const featuresStore = useFeaturesStore()
 
-  const handleChange = useCallback((data: FileUpload) => {
+  const isImageEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.image) ?? false
+
+  const handleChange = useCallback((value: boolean) => {
     const {
       features,
       setFeatures,
     } = featuresStore!.getState()
 
     const newFeatures = produce(features, (draft) => {
-      draft.file = {
-        ...draft.file,
-        enabled: data.enabled,
-        image: {
-          enabled: data.enabled,
-          detail: data.image?.detail,
-          transfer_methods: data.image?.transfer_methods,
-          number_limits: data.image?.number_limits,
-        },
+      if (value) {
+        draft.file!.allowed_file_types = Array.from(new Set([
+          ...(draft.file?.allowed_file_types || []),
+          SupportUploadFileTypes.image,
+          ...(isAllowVideoUpload ? [SupportUploadFileTypes.video] : []),
+        ]))
+      }
+      else {
+        draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter(
+          type => type !== SupportUploadFileTypes.image && (isAllowVideoUpload ? type !== SupportUploadFileTypes.video : true),
+        )
+      }
+
+      if (draft.file) {
+        draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0
+        draft.file.image = {
+          ...(draft.file.image || {}),
+          enabled: value,
+        }
       }
     })
     setFeatures(newFeatures)
-  }, [featuresStore])
+  }, [featuresStore, isAllowVideoUpload])
 
   if (!isShowVisionConfig)
     return null
         <ParamConfig />
         <div className='ml-1 mr-3 w-[1px] h-3.5 bg-divider-subtle'></div>
         <Switch
-          defaultValue={file?.enabled}
-          onChange={value => handleChange({
-            ...(file || {}),
-            enabled: value,
-          })}
+          defaultValue={isImageEnabled}
+          onChange={handleChange}
           size='md'
         />
       </div>

web/app/components/app/configuration/config/config-document.tsx

+'use client'
+import type { FC } from 'react'
+import React, { useCallback } from 'react'
+import { useTranslation } from 'react-i18next'
+import produce from 'immer'
+import { useContext } from 'use-context-selector'
+
+import { Document } from '@/app/components/base/icons/src/vender/features'
+import Tooltip from '@/app/components/base/tooltip'
+import ConfigContext from '@/context/debug-configuration'
+import { SupportUploadFileTypes } from '@/app/components/workflow/types'
+import { useFeatures, useFeaturesStore } from '@/app/components/base/features/hooks'
+import Switch from '@/app/components/base/switch'
+
+const ConfigDocument: FC = () => {
+  const { t } = useTranslation()
+  const file = useFeatures(s => s.features.file)
+  const featuresStore = useFeaturesStore()
+  const { isShowDocumentConfig } = useContext(ConfigContext)
+
+  const isDocumentEnabled = file?.allowed_file_types?.includes(SupportUploadFileTypes.document) ?? false
+
+  const handleChange = useCallback((value: boolean) => {
+    const {
+      features,
+      setFeatures,
+    } = featuresStore!.getState()
+
+    const newFeatures = produce(features, (draft) => {
+      if (value) {
+        draft.file!.allowed_file_types = Array.from(new Set([
+          ...(draft.file?.allowed_file_types || []),
+          SupportUploadFileTypes.document,
+        ]))
+      }
+      else {
+        draft.file!.allowed_file_types = draft.file!.allowed_file_types?.filter(
+          type => type !== SupportUploadFileTypes.document,
+        )
+      }
+      if (draft.file)
+        draft.file.enabled = (draft.file.allowed_file_types?.length ?? 0) > 0
+    })
+    setFeatures(newFeatures)
+  }, [featuresStore])
+
+  if (!isShowDocumentConfig)
+    return null
+
+  return (
+    <div className='mt-2 flex items-center gap-2 p-2 rounded-xl border-t-[0.5px] border-l-[0.5px] bg-background-section-burn'>
+      <div className='shrink-0 p-1'>
+        <div className='p-1 rounded-lg border-[0.5px] border-divider-subtle shadow-xs bg-util-colors-indigo-indigo-600'>
+          <Document className='w-4 h-4 text-text-primary-on-surface' />
+        </div>
+      </div>
+      <div className='grow flex items-center'>
+        <div className='mr-1 text-text-secondary system-sm-semibold'>{t('appDebug.feature.documentUpload.title')}</div>
+        <Tooltip
+          popupContent={
+            <div className='w-[180px]' >
+              {t('appDebug.feature.documentUpload.description')}
+            </div>
+          }
+        />
+      </div>
+      <div className='shrink-0 flex items-center'>
+        <div className='ml-1 mr-3 w-[1px] h-3.5 bg-divider-subtle'></div>
+        <Switch
+          defaultValue={isDocumentEnabled}
+          onChange={handleChange}
+          size='md'
+        />
+      </div>
+    </div>
+  )
+}
+export default React.memo(ConfigDocument)

web/app/components/app/configuration/config/index.tsx

 import DatasetConfig from '../dataset-config'
 import HistoryPanel from '../config-prompt/conversation-history/history-panel'
 import ConfigVision from '../config-vision'
+import ConfigDocument from './config-document'
 import AgentTools from './agent/agent-tools'
 import ConfigContext from '@/context/debug-configuration'
 import ConfigPrompt from '@/app/components/app/configuration/config-prompt'
 
         <ConfigVision />
 
+        <ConfigDocument />
+
         {/* Chat History */}
         {isAdvancedMode && isChatApp && modelModeType === ModelModeType.completion && (
           <HistoryPanel

web/app/components/app/configuration/index.tsx

   }
 
   const isShowVisionConfig = !!currModel?.features?.includes(ModelFeatureEnum.vision)
-
+  const isShowDocumentConfig = !!currModel?.features?.includes(ModelFeatureEnum.document)
+  const isAllowVideoUpload = !!currModel?.features?.includes(ModelFeatureEnum.video)
   // *** web app features ***
   const featuresData: FeaturesData = useMemo(() => {
     return {
           transfer_methods: modelConfig.file_upload?.image?.transfer_methods || ['local_file', 'remote_url'],
         },
         enabled: !!(modelConfig.file_upload?.enabled || modelConfig.file_upload?.image?.enabled),
-        allowed_file_types: modelConfig.file_upload?.allowed_file_types || [SupportUploadFileTypes.image, SupportUploadFileTypes.video],
+        allowed_file_types: modelConfig.file_upload?.allowed_file_types || [],
         allowed_file_extensions: modelConfig.file_upload?.allowed_file_extensions || [...FILE_EXTS[SupportUploadFileTypes.image], ...FILE_EXTS[SupportUploadFileTypes.video]].map(ext => `.${ext}`),
         allowed_file_upload_methods: modelConfig.file_upload?.allowed_file_upload_methods || modelConfig.file_upload?.image?.transfer_methods || ['local_file', 'remote_url'],
         number_limits: modelConfig.file_upload?.number_limits || modelConfig.file_upload?.image?.number_limits || 3,
       isShowVisionConfig,
       visionConfig,
       setVisionConfig: handleSetVisionConfig,
+      isAllowVideoUpload,
+      isShowDocumentConfig,
       rerankSettingModalOpen,
       setRerankSettingModalOpen,
     }}

web/app/components/base/icons/assets/vender/features/document.svg

+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="currentColor">
+<path d="M20 22H4C3.44772 22 3 21.5523 3 21V3C3 2.44772 3.44772 2 4 2H20C20.5523 2 21 2.44772 21 3V21C21 21.5523 20.5523 22 20 22ZM7 6V10H11V6H7ZM7 12V14H17V12H7ZM7 16V18H17V16H7ZM13 7V9H17V7H13Z"></path>
+</svg>

web/app/components/base/icons/src/vender/features/Document.json

+{
+	"icon": {
+		"type": "element",
+		"isRootNode": true,
+		"name": "svg",
+		"attributes": {
+			"xmlns": "http://www.w3.org/2000/svg",
+			"viewBox": "0 0 24 24",
+			"fill": "currentColor"
+		},
+		"children": [
+			{
+				"type": "element",
+				"name": "path",
+				"attributes": {
+					"d": "M20 22H4C3.44772 22 3 21.5523 3 21V3C3 2.44772 3.44772 2 4 2H20C20.5523 2 21 2.44772 21 3V21C21 21.5523 20.5523 22 20 22ZM7 6V10H11V6H7ZM7 12V14H17V12H7ZM7 16V18H17V16H7ZM13 7V9H17V7H13Z"
+				},
+				"children": []
+			}
+		]
+	},
+	"name": "Document"
+}

web/app/components/base/icons/src/vender/features/Document.tsx

+// GENERATE BY script
+// DON NOT EDIT IT MANUALLY
+
+import * as React from 'react'
+import data from './Document.json'
+import IconBase from '@/app/components/base/icons/IconBase'
+import type { IconBaseProps, IconData } from '@/app/components/base/icons/IconBase'
+
+const Icon = React.forwardRef<React.MutableRefObject<SVGElement>, Omit<IconBaseProps, 'data'>>((
+  props,
+  ref,
+) => <IconBase {...props} ref={ref} data={data as IconData} />)
+
+Icon.displayName = 'Document'
+
+export default Icon

web/app/components/base/icons/src/vender/features/index.ts

 export { default as TextToAudio } from './TextToAudio'
 export { default as VirtualAssistant } from './VirtualAssistant'
 export { default as Vision } from './Vision'
+export { default as Document } from './Document'

web/app/components/header/account-setting/model-provider-page/declarations.ts

   multiToolCall = 'multi-tool-call',
   agentThought = 'agent-thought',
   vision = 'vision',
+  video = 'video',
+  document = 'document',
+  audio = 'audio',
 }
 
 export enum ModelFeatureTextEnum {
   multiToolCall = 'Multi Tool Call',
   agentThought = 'Agent Thought',
   vision = 'Vision',
+  video = 'Video',
+  document = 'Document',
+  audio = 'Audio',
 }
 
 export enum ModelStatusEnum {

web/context/debug-configuration.ts

   isShowVisionConfig: boolean
   visionConfig: VisionSettings
   setVisionConfig: (visionConfig: VisionSettings, noNotice?: boolean) => void
+  isAllowVideoUpload: boolean
+  isShowDocumentConfig: boolean
   rerankSettingModalOpen: boolean
   setRerankSettingModalOpen: (rerankSettingModalOpen: boolean) => void
 }
     transfer_methods: [TransferMethod.remote_url],
   },
   setVisionConfig: () => { },
+  isAllowVideoUpload: false,
+  isShowDocumentConfig: false,
   rerankSettingModalOpen: false,
   setRerankSettingModalOpen: () => { },
 })

web/i18n/en-US/app-debug.ts

       enableText: 'Features Enabled',
       manage: 'Manage',
     },
+    documentUpload: {
+      title: 'Document',
+      description: 'Enable Document will allows the model to take in documents and answer questions about them.',
+    },
   },
   codegen: {
     title: 'Code Generator',

web/i18n/zh-Hans/app-debug.ts

       enableText: '功能已开启',
       manage: '管理',
     },
+    documentUpload: {
+      title: '文档',
+      description: '启用文档后，模型可以接收文档并回答关于它们的问题。',
+    },
   },
   codegen: {
     title: '代码生成器',