Feat: Filter the agent form's large model list by type #3221 (#9049)

### What problem does this PR solve?

Feat: Filter the agent form's large model list by type #3221

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/components/large-model-form-field.tsx

+import {
+  DropdownMenu,
+  DropdownMenuContent,
+  DropdownMenuItem,
+  DropdownMenuTrigger,
+} from '@/components/ui/dropdown-menu';
 import {
   FormControl,
   FormField,
   FormLabel,
   FormMessage,
 } from '@/components/ui/form';
-import { useFormContext } from 'react-hook-form';
+import { LlmModelType } from '@/constants/knowledge';
+import { Funnel } from 'lucide-react';
+import { useFormContext, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
+import { z } from 'zod';
 import { NextLLMSelect } from './llm-select/next';
+import { Button } from './ui/button';
+
+const ModelTypes = [
+  {
+    title: 'All Models',
+    value: 'all',
+  },
+  {
+    title: 'Text-only Models',
+    value: LlmModelType.Chat,
+  },
+  {
+    title: 'Multimodal Models',
+    value: LlmModelType.Image2text,
+  },
+];
+
+export const LargeModelFilterFormSchema = {
+  llm_filter: z.string().optional(),
+};
 
 export function LargeModelFormField() {
   const form = useFormContext();
   const { t } = useTranslation();
+  const filter = useWatch({ control: form.control, name: 'llm_filter' });
 
   return (
-    <FormField
-      control={form.control}
-      name="llm_id"
-      render={({ field }) => (
-        <FormItem>
-          <FormLabel tooltip={t('chat.modelTip')}>{t('chat.model')}</FormLabel>
-          <FormControl>
-            <NextLLMSelect {...field} />
-          </FormControl>
-          <FormMessage />
-        </FormItem>
-      )}
-    />
+    <>
+      <FormField
+        control={form.control}
+        name="llm_id"
+        render={({ field }) => (
+          <FormItem>
+            <FormLabel tooltip={t('chat.modelTip')}>
+              {t('chat.model')}
+            </FormLabel>
+            <section className="flex gap-2.5">
+              <FormField
+                control={form.control}
+                name="llm_filter"
+                render={({ field }) => (
+                  <FormItem>
+                    <FormControl>
+                      <DropdownMenu>
+                        <DropdownMenuTrigger>
+                          <Button variant={'ghost'}>
+                            <Funnel />
+                          </Button>
+                        </DropdownMenuTrigger>
+                        <DropdownMenuContent>
+                          {ModelTypes.map((x) => (
+                            <DropdownMenuItem
+                              key={x.value}
+                              onClick={() => {
+                                field.onChange(x.value);
+                              }}
+                            >
+                              {x.title}
+                            </DropdownMenuItem>
+                          ))}
+                        </DropdownMenuContent>
+                      </DropdownMenu>
+                    </FormControl>
+                  </FormItem>
+                )}
+              />
+
+              <FormControl>
+                <NextLLMSelect {...field} filter={filter} />
+              </FormControl>
+            </section>
+
+            <FormMessage />
+          </FormItem>
+        )}
+      />
+    </>
   );
 }

web/src/components/llm-select/next.tsx

   onInitialValue?: (value: string, option: any) => void;
   onChange?: (value: string) => void;
   disabled?: boolean;
+  filter?: string;
 }
 
 const NextInnerLLMSelect = forwardRef<
   React.ElementRef<typeof SelectPrimitive.Trigger>,
   IProps
->(({ value, disabled }, ref) => {
+>(({ value, disabled, filter }, ref) => {
   const [isPopoverOpen, setIsPopoverOpen] = useState(false);
-  const modelOptions = useComposeLlmOptionsByModelTypes([
-    LlmModelType.Chat,
-    LlmModelType.Image2text,
-  ]);
+  const modelTypes =
+    filter === 'all' || filter === undefined
+      ? [LlmModelType.Chat, LlmModelType.Image2text]
+      : [filter as LlmModelType];
+  const modelOptions = useComposeLlmOptionsByModelTypes(modelTypes);
 
   return (
     <Select disabled={disabled} value={value}>
           </SelectTrigger>
         </PopoverTrigger>
         <PopoverContent side={'left'}>
-          <LlmSettingFieldItems></LlmSettingFieldItems>
+          <LlmSettingFieldItems options={modelOptions}></LlmSettingFieldItems>
         </PopoverContent>
       </Popover>
     </Select>

web/src/components/llm-setting-items/next.tsx

 
 interface LlmSettingFieldItemsProps {
   prefix?: string;
+  options?: any[];
 }
 
 export const LlmSettingSchema = {
   maxTokensEnabled: z.boolean(),
 };
 
-export function LlmSettingFieldItems({ prefix }: LlmSettingFieldItemsProps) {
+export function LlmSettingFieldItems({
+  prefix,
+  options,
+}: LlmSettingFieldItemsProps) {
   const form = useFormContext();
   const { t } = useTranslate('chat');
+
   const modelOptions = useComposeLlmOptionsByModelTypes([
     LlmModelType.Chat,
     LlmModelType.Image2text,
             <FormLabel>{t('model')}</FormLabel>
             <FormControl>
               <SelectWithSearch
-                options={modelOptions}
+                options={options || modelOptions}
                 {...field}
               ></SelectWithSearch>
-              {/* <Select onValueChange={field.onChange} {...field}>
-                <SelectTrigger value={field.value}>
-                  <SelectValue />
-                </SelectTrigger>
-                <SelectContent>
-                  {modelOptions.map((x) => (
-                    <SelectGroup key={x.value}>
-                      <SelectLabel>{x.label}</SelectLabel>
-                      {x.options.map((y) => (
-                        <SelectItem
-                          value={y.value}
-                          key={y.value}
-                          disabled={y.disabled}
-                        >
-                          {y.label}
-                        </SelectItem>
-                      ))}
-                    </SelectGroup>
-                  ))}
-                </SelectContent>
-              </Select> */}
             </FormControl>
             <FormMessage />
           </FormItem>

web/src/components/originui/select-with-search.tsx

 'use client';
 
-import { CheckIcon, ChevronDownIcon } from 'lucide-react';
+import { CheckIcon, ChevronDownIcon, XIcon } from 'lucide-react';
 import {
   Fragment,
+  MouseEventHandler,
   ReactNode,
   forwardRef,
   useCallback,
 } from '@/components/ui/popover';
 import { cn } from '@/lib/utils';
 import { RAGFlowSelectOptionType } from '../ui/select';
+import { Separator } from '../ui/separator';
 
 export type SelectWithSearchFlagOptionType = {
   label: ReactNode;
   value?: string;
   onChange?(value: string): void;
   triggerClassName?: string;
+  allowClear?: boolean;
 };
 
 export const SelectWithSearch = forwardRef<
   React.ElementRef<typeof Button>,
   SelectWithSearchFlagProps
->(({ value: val = '', onChange, options = [], triggerClassName }, ref) => {
-  const id = useId();
-  const [open, setOpen] = useState<boolean>(false);
-  const [value, setValue] = useState<string>('');
-
-  const handleSelect = useCallback(
-    (val: string) => {
-      setValue(val);
-      setOpen(false);
-      onChange?.(val);
+>(
+  (
+    {
+      value: val = '',
+      onChange,
+      options = [],
+      triggerClassName,
+      allowClear = false,
     },
-    [onChange],
-  );
+    ref,
+  ) => {
+    const id = useId();
+    const [open, setOpen] = useState<boolean>(false);
+    const [value, setValue] = useState<string>('');
 
-  useEffect(() => {
-    setValue(val);
-  }, [val]);
-  const selectLabel = useMemo(() => {
-    const optionTemp = options[0];
-    if (optionTemp?.options) {
-      return options
-        .map((group) => group?.options?.find((item) => item.value === value))
-        .filter(Boolean)[0]?.label;
-    } else {
-      return options.find((opt) => opt.value === value)?.label || '';
-    }
-  }, [options, value]);
-  return (
-    <Popover open={open} onOpenChange={setOpen}>
-      <PopoverTrigger asChild>
-        <Button
-          id={id}
-          variant="outline"
-          role="combobox"
-          aria-expanded={open}
-          ref={ref}
-          className={cn(
-            'bg-background hover:bg-background border-input w-full  justify-between px-3 font-normal outline-offset-0 outline-none focus-visible:outline-[3px]',
-            triggerClassName,
-          )}
-        >
-          {value ? (
-            <span className="flex min-w-0 options-center gap-2">
-              <span className="text-lg leading-none truncate">
-                {selectLabel}
+    const handleSelect = useCallback(
+      (val: string) => {
+        setValue(val);
+        setOpen(false);
+        onChange?.(val);
+      },
+      [onChange],
+    );
+
+    const handleClear: MouseEventHandler<SVGElement> = useCallback(
+      (e) => {
+        e.stopPropagation();
+        setValue('');
+        onChange?.('');
+      },
+      [onChange],
+    );
+
+    useEffect(() => {
+      setValue(val);
+    }, [val]);
+    const selectLabel = useMemo(() => {
+      const optionTemp = options[0];
+      if (optionTemp?.options) {
+        return options
+          .map((group) => group?.options?.find((item) => item.value === value))
+          .filter(Boolean)[0]?.label;
+      } else {
+        return options.find((opt) => opt.value === value)?.label || '';
+      }
+    }, [options, value]);
+    return (
+      <Popover open={open} onOpenChange={setOpen}>
+        <PopoverTrigger asChild>
+          <Button
+            id={id}
+            variant="outline"
+            role="combobox"
+            aria-expanded={open}
+            ref={ref}
+            className={cn(
+              'bg-background hover:bg-background border-input w-full  justify-between px-3 font-normal outline-offset-0 outline-none focus-visible:outline-[3px] [&_svg]:pointer-events-auto',
+              triggerClassName,
+            )}
+          >
+            {value ? (
+              <span className="flex min-w-0 options-center gap-2">
+                <span className="text-lg leading-none truncate">
+                  {selectLabel}
+                </span>
               </span>
-            </span>
-          ) : (
-            <span className="text-muted-foreground">Select value</span>
-          )}
-          <ChevronDownIcon
-            size={16}
-            className="text-muted-foreground/80 shrink-0"
-            aria-hidden="true"
-          />
-        </Button>
-      </PopoverTrigger>
-      <PopoverContent
-        className="border-input w-full min-w-[var(--radix-popper-anchor-width)] p-0"
-        align="start"
-      >
-        <Command>
-          <CommandInput placeholder="Search ..." />
-          <CommandList>
-            <CommandEmpty>No data found.</CommandEmpty>
-            {options.map((group, idx) => {
-              if (group.options) {
-                return (
-                  <Fragment key={idx}>
-                    <CommandGroup heading={group.label}>
-                      {group.options.map((option) => (
-                        <CommandItem
-                          key={option.value}
-                          value={option.value}
-                          disabled={option.disabled}
-                          onSelect={handleSelect}
-                        >
-                          <span className="text-lg leading-none">
-                            {option.label}
-                          </span>
+            ) : (
+              <span className="text-muted-foreground">Select value</span>
+            )}
+            <div className="flex items-center justify-between">
+              {value && allowClear && (
+                <>
+                  <XIcon
+                    className="h-4 mx-2 cursor-pointer text-muted-foreground"
+                    onClick={handleClear}
+                  />
+                  <Separator
+                    orientation="vertical"
+                    className="flex min-h-6 h-full"
+                  />
+                </>
+              )}
+              <ChevronDownIcon
+                size={16}
+                className="text-muted-foreground/80 shrink-0 ml-2"
+                aria-hidden="true"
+              />
+            </div>
+          </Button>
+        </PopoverTrigger>
+        <PopoverContent
+          className="border-input w-full min-w-[var(--radix-popper-anchor-width)] p-0"
+          align="start"
+        >
+          <Command>
+            <CommandInput placeholder="Search ..." />
+            <CommandList>
+              <CommandEmpty>No data found.</CommandEmpty>
+              {options.map((group, idx) => {
+                if (group.options) {
+                  return (
+                    <Fragment key={idx}>
+                      <CommandGroup heading={group.label}>
+                        {group.options.map((option) => (
+                          <CommandItem
+                            key={option.value}
+                            value={option.value}
+                            disabled={option.disabled}
+                            onSelect={handleSelect}
+                          >
+                            <span className="text-lg leading-none">
+                              {option.label}
+                            </span>
 
-                          {value === option.value && (
-                            <CheckIcon size={16} className="ml-auto" />
-                          )}
-                        </CommandItem>
-                      ))}
-                    </CommandGroup>
-                  </Fragment>
-                );
-              } else {
-                return (
-                  <CommandItem
-                    key={group.value}
-                    value={group.value}
-                    disabled={group.disabled}
-                    onSelect={handleSelect}
-                  >
-                    <span className="text-lg leading-none">{group.label}</span>
+                            {value === option.value && (
+                              <CheckIcon size={16} className="ml-auto" />
+                            )}
+                          </CommandItem>
+                        ))}
+                      </CommandGroup>
+                    </Fragment>
+                  );
+                } else {
+                  return (
+                    <CommandItem
+                      key={group.value}
+                      value={group.value}
+                      disabled={group.disabled}
+                      onSelect={handleSelect}
+                    >
+                      <span className="text-lg leading-none">
+                        {group.label}
+                      </span>
 
-                    {value === group.value && (
-                      <CheckIcon size={16} className="ml-auto" />
-                    )}
-                  </CommandItem>
-                );
-              }
-            })}
-          </CommandList>
-        </Command>
-      </PopoverContent>
-    </Popover>
-  );
-});
+                      {value === group.value && (
+                        <CheckIcon size={16} className="ml-auto" />
+                      )}
+                    </CommandItem>
+                  );
+                }
+              })}
+            </CommandList>
+          </Command>
+        </PopoverContent>
+      </Popover>
+    );
+  },
+);
 
 SelectWithSearch.displayName = 'SelectWithSearch';

web/src/hooks/use-llm-request.ts

+import { LlmModelType } from '@/constants/knowledge';
+import userService from '@/services/user-service';
+import { useQuery } from '@tanstack/react-query';
+
+import {
+  IThirdOAIModelCollection as IThirdAiModelCollection,
+  IThirdOAIModel,
+} from '@/interfaces/database/llm';
+import { buildLlmUuid } from '@/utils/llm-util';
+
+export const useFetchLlmList = (modelType?: LlmModelType) => {
+  const { data } = useQuery<IThirdAiModelCollection>({
+    queryKey: ['llmList'],
+    initialData: {},
+    queryFn: async () => {
+      const { data } = await userService.llm_list({ model_type: modelType });
+
+      return data?.data ?? {};
+    },
+  });
+
+  return data;
+};
+
+type IThirdOAIModelWithUuid = IThirdOAIModel & { uuid: string };
+
+export function useSelectFlatLlmList(modelType?: LlmModelType) {
+  const llmList = useFetchLlmList(modelType);
+
+  return Object.values(llmList).reduce<IThirdOAIModelWithUuid[]>((pre, cur) => {
+    pre.push(...cur.map((x) => ({ ...x, uuid: buildLlmUuid(x) })));
+
+    return pre;
+  }, []);
+}
+
+export function useFindLlmByUuid(modelType?: LlmModelType) {
+  const flatList = useSelectFlatLlmList(modelType);
+
+  return (uuid: string) => {
+    return flatList.find((x) => x.uuid === uuid);
+  };
+}

web/src/pages/agent/form/agent-form/index.tsx

 import { Collapse } from '@/components/collapse';
 import { FormContainer } from '@/components/form-container';
-import { LargeModelFormField } from '@/components/large-model-form-field';
+import {
+  LargeModelFilterFormSchema,
+  LargeModelFormField,
+} from '@/components/large-model-form-field';
 import { LlmSettingSchema } from '@/components/llm-setting-items/next';
 import { MessageHistoryWindowSizeFormField } from '@/components/message-history-window-size-item';
 import {
 } from '@/components/ui/form';
 import { Input, NumberInput } from '@/components/ui/input';
 import { RAGFlowSelect } from '@/components/ui/select';
+import { LlmModelType } from '@/constants/knowledge';
+import { useFindLlmByUuid } from '@/hooks/use-llm-request';
 import { buildOptions } from '@/utils/form';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { memo, useMemo } from 'react';
-import { useForm } from 'react-hook-form';
+import { useForm, useWatch } from 'react-hook-form';
 import { useTranslation } from 'react-i18next';
 import { z } from 'zod';
 import {
   exception_method: z.string().nullable(),
   exception_comment: z.string().optional(),
   exception_goto: z.string().optional(),
+  ...LargeModelFilterFormSchema,
 });
 
 function AgentForm({ node }: INextOperatorForm) {
     resolver: zodResolver(FormSchema),
   });
 
+  const llmId = useWatch({ control: form.control, name: 'llm_id' });
+
+  const findLlmByUuid = useFindLlmByUuid();
+
   useWatchFormChange(node?.id, form);
 
   return (
         <FormContainer>
           {isSubAgent && <DescriptionField></DescriptionField>}
           <LargeModelFormField></LargeModelFormField>
+          {findLlmByUuid(llmId)?.model_type === LlmModelType.Image2text && (
+            <QueryVariable
+              name="visual_files_var"
+              label="Visual Input File"
+              type={VariableType.File}
+            ></QueryVariable>
+          )}
+        </FormContainer>
+
+        <FormContainer>
           <FormField
             control={form.control}
             name={`sys_prompt`}
               </FormItem>
             )}
           />
-          <MessageHistoryWindowSizeFormField></MessageHistoryWindowSizeFormField>
         </FormContainer>
         {isSubAgent || (
           <FormContainer>
         </FormContainer>
         <Collapse title={<div>Advanced Settings</div>}>
           <FormContainer>
-            <QueryVariable
-              name="visual_files_var"
-              label="Visual Input File"
-              type={VariableType.File}
-            ></QueryVariable>
+            <MessageHistoryWindowSizeFormField></MessageHistoryWindowSizeFormField>
             <FormField
               control={form.control}
               name={`max_retries`}

web/src/pages/agent/form/components/query-variable.tsx

             <SelectWithSearch
               options={finalOptions}
               {...field}
+              allowClear
             ></SelectWithSearch>
           </FormControl>
           <FormMessage />

web/src/pages/agent/index.tsx

                 <Upload />
                 {t('flow.export')}
               </AgentDropdownMenuItem>
-              <DropdownMenuSeparator />
-              <AgentDropdownMenuItem
-                onClick={showEmbedModal}
-                disabled={
-                  !isBeginNodeDataQuerySafe ||
-                  userInfo.nickname !== agentDetail.nickname
-                }
-              >
-                <ScreenShare />
-                {t('common.embedIntoSite')}
-              </AgentDropdownMenuItem>
+              {location.hostname !== 'demo.ragflow.io' && (
+                <>
+                  <DropdownMenuSeparator />
+                  <AgentDropdownMenuItem
+                    onClick={showEmbedModal}
+                    disabled={
+                      !isBeginNodeDataQuerySafe ||
+                      userInfo.nickname !== agentDetail.nickname
+                    }
+                  >
+                    <ScreenShare />
+                    {t('common.embedIntoSite')}
+                  </AgentDropdownMenuItem>
+                </>
+              )}
             </DropdownMenuContent>
           </DropdownMenu>
         </div>

web/src/utils/llm-util.ts

+import { IThirdOAIModel } from '@/interfaces/database/llm';
+
 export const getLLMIconName = (fid: string, llm_name: string) => {
   if (fid === 'FastEmbed') {
     return llm_name.split('/').at(0) ?? '';
 export function getRealModelName(llmName: string) {
   return llmName.split('__').at(0) ?? '';
 }
+
+export function buildLlmUuid(llm: IThirdOAIModel) {
+  return `${llm.llm_name}@${llm.fid}`;
+}