Feat: Add TavilyExtract operator #3221 (#8899)

### What problem does this PR solve?

Feat: Add TavilyExtract operator #3221

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/locales/en.ts

@@ -1291,6 +1291,7 @@ This delimiter is used to split the input text into several text pieces echo of
       agent: 'Agent',
       agentDescription:
         'Builds agent components equipped with reasoning, tool usage, and multi-agent collaboration. ',
+      maxRecords: 'Max records',
     },
     llmTools: {
       bad_calculator: {

web/src/locales/zh.ts

@@ -1244,6 +1244,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       query: '查询变量',
       agent: 'Agent',
       agentDescription: '构建具备推理、工具调用和多智能体协同的智能体组件。',
+      maxRecords: '最大记录数',
     },
     footer: {
       profile: 'All rights reserved @ React',

web/src/pages/agent/agent-sidebar.tsx

@@ -1,112 +0,0 @@
-import { SideDown } from '@/assets/icon/Icon';
-import { Card, CardContent } from '@/components/ui/card';
-import {
-  Collapsible,
-  CollapsibleContent,
-  CollapsibleTrigger,
-} from '@/components/ui/collapsible';
-import {
-  Sidebar,
-  SidebarContent,
-  SidebarGroup,
-  SidebarGroupContent,
-  SidebarGroupLabel,
-  SidebarHeader,
-  SidebarMenu,
-} from '@/components/ui/sidebar';
-import { memo, useMemo } from 'react';
-import {
-  AgentOperatorList,
-  Operator,
-  componentMenuList,
-  operatorMap,
-} from './constant';
-import { useHandleDrag } from './hooks';
-import OperatorIcon from './operator-icon';
-
-type OperatorItem = {
-  name: Operator;
-};
-
-function OperatorCard({ name }: OperatorItem) {
-  const { handleDragStart } = useHandleDrag();
-
-  return (
-    <Card
-      draggable
-      onDragStart={handleDragStart(name)}
-      className="bg-colors-background-inverse-weak  border-colors-outline-neutral-standard cursor-pointer"
-    >
-      <CardContent className="p-2 flex items-center gap-2">
-        <OperatorIcon
-          name={name}
-          color={operatorMap[name].color}
-        ></OperatorIcon>
-        {name}
-      </CardContent>
-    </Card>
-  );
-}
-
-type OperatorCollapsibleProps = { operatorList: OperatorItem[]; title: string };
-
-function OperatorCollapsible({
-  operatorList,
-  title,
-}: OperatorCollapsibleProps) {
-  return (
-    <Collapsible defaultOpen className="group/collapsible">
-      <SidebarGroup>
-        <SidebarGroupLabel asChild className="mb-1">
-          <CollapsibleTrigger>
-            <span className="font-bold text-base">{title}</span>
-            <SideDown className="ml-auto" />
-          </CollapsibleTrigger>
-        </SidebarGroupLabel>
-        <CollapsibleContent className="px-2">
-          <SidebarGroupContent>
-            <SidebarMenu className="gap-2">
-              {operatorList.map((item) => (
-                <OperatorCard key={item.name} name={item.name}></OperatorCard>
-              ))}
-            </SidebarMenu>
-          </SidebarGroupContent>
-        </CollapsibleContent>
-      </SidebarGroup>
-    </Collapsible>
-  );
-}
-
-function InnerAgentSidebar() {
-  const agentOperatorList = useMemo(() => {
-    return componentMenuList.filter((x) =>
-      AgentOperatorList.some((y) => y === x.name),
-    );
-  }, []);
-
-  const thirdOperatorList = useMemo(() => {
-    return componentMenuList.filter(
-      (x) => !AgentOperatorList.some((y) => y === x.name),
-    );
-  }, []);
-
-  return (
-    <Sidebar variant={'floating'} className="top-16">
-      <SidebarHeader>
-        <p className="font-bold text-2xl">All nodes</p>
-      </SidebarHeader>
-      <SidebarContent>
-        <OperatorCollapsible
-          title="Agent operator"
-          operatorList={agentOperatorList}
-        ></OperatorCollapsible>
-        <OperatorCollapsible
-          title="Third-party tools"
-          operatorList={thirdOperatorList}
-        ></OperatorCollapsible>
-      </SidebarContent>
-    </Sidebar>
-  );
-}
-
-export const AgentSidebar = memo(InnerAgentSidebar);

web/src/pages/agent/canvas/node/dropdown/next-step-dropdown.tsx

@@ -100,7 +100,7 @@ function AccordionOperators() {
           <OperatorItemList
             operators={[
               Operator.TavilySearch,
-              Operator.Crawler,
+              Operator.TavilyExtract,
               Operator.ExeSQL,
               Operator.Bing,
             ]}

web/src/pages/agent/constant.tsx

@@ -87,6 +87,7 @@ export enum Operator {
   Agent = 'Agent',
   Tool = 'Tool',
   TavilySearch = 'TavilySearch',
+  TavilyExtract = 'TavilyExtract',
   UserFillUp = 'UserFillUp',
   StringTransform = 'StringTransform',
 }
@@ -114,148 +115,6 @@ export const AgentOperatorList = [
   Operator.Agent,
 ];
 
-export const operatorMap: Record<
-  Operator,
-  {
-    backgroundColor?: string;
-    color?: string;
-    width?: number;
-    height?: number;
-    fontSize?: number;
-    iconFontSize?: number;
-    iconWidth?: number;
-    moreIconColor?: string;
-  }
-> = {
-  [Operator.Retrieval]: {
-    backgroundColor: '#cad6e0',
-    color: '#385974',
-  },
-  [Operator.Generate]: {
-    backgroundColor: '#ebd6d6',
-    width: 150,
-    height: 150,
-    fontSize: 20,
-    iconFontSize: 30,
-    color: '#996464',
-  },
-  [Operator.Answer]: {
-    backgroundColor: '#f4816d',
-    color: '#f4816d',
-  },
-  [Operator.Begin]: {
-    backgroundColor: '#4f51d6',
-  },
-  [Operator.Categorize]: {
-    backgroundColor: '#ffebcd',
-    color: '#cc8a26',
-  },
-  [Operator.Message]: {
-    backgroundColor: '#c5ddc7',
-    color: 'green',
-  },
-  [Operator.Relevant]: {
-    backgroundColor: '#9fd94d',
-    color: '#8ef005',
-    width: 70,
-    height: 70,
-    fontSize: 12,
-    iconFontSize: 16,
-  },
-  [Operator.RewriteQuestion]: {
-    backgroundColor: '#f8c7f8',
-    color: '#f32bf3',
-    width: 70,
-    height: 70,
-    fontSize: 12,
-    iconFontSize: 16,
-  },
-  [Operator.KeywordExtract]: {
-    width: 70,
-    height: 70,
-    backgroundColor: '#6E5494',
-    color: '#6E5494',
-    fontSize: 12,
-    iconWidth: 16,
-  },
-  [Operator.DuckDuckGo]: {
-    backgroundColor: '#e7e389',
-    color: '#aea00c',
-  },
-  [Operator.Baidu]: {
-    backgroundColor: '#d9e0f8',
-  },
-  [Operator.Wikipedia]: {
-    backgroundColor: '#dee0e2',
-  },
-  [Operator.PubMed]: {
-    backgroundColor: '#a2ccf0',
-  },
-  [Operator.ArXiv]: {
-    width: 70,
-    height: 70,
-    fontSize: 12,
-    iconWidth: 16,
-    iconFontSize: 16,
-    moreIconColor: 'white',
-    backgroundColor: '#b31b1b',
-    color: 'white',
-  },
-  [Operator.Google]: {
-    backgroundColor: 'pink',
-  },
-  [Operator.Bing]: {
-    backgroundColor: '#c0dcc4',
-  },
-  [Operator.GoogleScholar]: {
-    backgroundColor: '#b4e4f6',
-  },
-  [Operator.DeepL]: {
-    backgroundColor: '#f5e8e6',
-  },
-  [Operator.GitHub]: {
-    backgroundColor: 'purple',
-    color: 'purple',
-  },
-  [Operator.BaiduFanyi]: { backgroundColor: '#e5f2d3' },
-  [Operator.QWeather]: {
-    backgroundColor: '#a4bbf3',
-    color: '#a4bbf3',
-  },
-  [Operator.ExeSQL]: { backgroundColor: '#b9efe8' },
-  [Operator.Switch]: { backgroundColor: '#dbaff6', color: '#dbaff6' },
-  [Operator.WenCai]: { backgroundColor: '#faac5b' },
-  [Operator.AkShare]: { backgroundColor: '#8085f5' },
-  [Operator.YahooFinance]: { backgroundColor: '#b474ff' },
-  [Operator.Jin10]: { backgroundColor: '#a0b9f8' },
-  [Operator.Concentrator]: {
-    backgroundColor: '#32d2a3',
-    color: '#32d2a3',
-    width: 70,
-    height: 70,
-    fontSize: 10,
-    iconFontSize: 16,
-  },
-  [Operator.TuShare]: { backgroundColor: '#f8cfa0' },
-  [Operator.Note]: { backgroundColor: '#f8cfa0' },
-  [Operator.Crawler]: {
-    backgroundColor: '#dee0e2',
-  },
-  [Operator.Invoke]: {
-    backgroundColor: '#dee0e2',
-  },
-  [Operator.Template]: {
-    backgroundColor: '#dee0e2',
-  },
-  [Operator.Email]: { backgroundColor: '#e6f7ff' },
-  [Operator.Iteration]: { backgroundColor: '#e6f7ff' },
-  [Operator.IterationStart]: { backgroundColor: '#e6f7ff' },
-  [Operator.Code]: { backgroundColor: '#4c5458' },
-  [Operator.WaitingDialogue]: { backgroundColor: '#a5d65c' },
-  [Operator.Agent]: { backgroundColor: '#a5d65c' },
-  [Operator.TavilySearch]: { backgroundColor: '#a5d65c' },
-};
-
 export const componentMenuList = [
   {
     name: Operator.Retrieval,
@@ -552,16 +411,14 @@ export const initialQWeatherValues = {
 };
 
 export const initialExeSqlValues = {
-  ...initialLlmBaseValues,
+  sql: '',
   db_type: 'mysql',
   database: '',
   username: '',
   host: '',
   port: 3306,
   password: '',
-  loop: 3,
-  top_n: 30,
-  query: '',
+  max_records: 1024,
 };
 
 export const initialSwitchValues = {
@@ -775,8 +632,34 @@ export const initialTavilyValues = {
       type: 'string',
     },
     json: {
-      value: {},
-      type: 'Object',
+      value: [],
+      type: 'Array<Object>',
+    },
+  },
+};
+
+export enum TavilyExtractDepth {
+  Basic = 'basic',
+  Advanced = 'advanced',
+}
+
+export enum TavilyExtractFormat {
+  Text = 'text',
+  Markdown = 'markdown',
+}
+
+export const initialTavilyExtractValues = {
+  urls: '',
+  extract_depth: TavilyExtractDepth.Basic,
+  format: TavilyExtractFormat.Markdown,
+  outputs: {
+    formalized_content: {
+      value: '',
+      type: 'string',
+    },
+    json: {
+      value: [],
+      type: 'Array<Object>',
     },
   },
 };
@@ -866,6 +749,7 @@ export const RestrictedUpstreamMap = {
   [Operator.WaitingDialogue]: [Operator.Begin],
   [Operator.Agent]: [Operator.Begin],
   [Operator.TavilySearch]: [Operator.Begin],
+  [Operator.TavilyExtract]: [Operator.Begin],
   [Operator.StringTransform]: [Operator.Begin],
   [Operator.UserFillUp]: [Operator.Begin],
 };
@@ -914,6 +798,7 @@ export const NodeMap = {
   [Operator.TavilySearch]: 'ragNode',
   [Operator.UserFillUp]: 'ragNode',
   [Operator.StringTransform]: 'ragNode',
+  [Operator.TavilyExtract]: 'ragNode',
 };
 
 export enum BeginQueryType {

web/src/pages/agent/form-sheet/form-config-map.tsx

@@ -0,0 +1,182 @@
+import { z } from 'zod';
+import { Operator } from '../constant';
+import AgentForm from '../form/agent-form';
+import AkShareForm from '../form/akshare-form';
+import AnswerForm from '../form/answer-form';
+import ArXivForm from '../form/arxiv-form';
+import BaiduFanyiForm from '../form/baidu-fanyi-form';
+import BaiduForm from '../form/baidu-form';
+import BeginForm from '../form/begin-form';
+import BingForm from '../form/bing-form';
+import CategorizeForm from '../form/categorize-form';
+import CodeForm from '../form/code-form';
+import CrawlerForm from '../form/crawler-form';
+import DeepLForm from '../form/deepl-form';
+import DuckDuckGoForm from '../form/duckduckgo-form';
+import EmailForm from '../form/email-form';
+import ExeSQLForm from '../form/exesql-form';
+import GenerateForm from '../form/generate-form';
+import GithubForm from '../form/github-form';
+import GoogleForm from '../form/google-form';
+import GoogleScholarForm from '../form/google-scholar-form';
+import InvokeForm from '../form/invoke-form';
+import IterationForm from '../form/iteration-form';
+import IterationStartForm from '../form/iteration-start-from';
+import Jin10Form from '../form/jin10-form';
+import KeywordExtractForm from '../form/keyword-extract-form';
+import MessageForm from '../form/message-form';
+import PubMedForm from '../form/pubmed-form';
+import QWeatherForm from '../form/qweather-form';
+import RelevantForm from '../form/relevant-form';
+import RetrievalForm from '../form/retrieval-form/next';
+import RewriteQuestionForm from '../form/rewrite-question-form';
+import { StringTransformForm } from '../form/string-transform-form';
+import SwitchForm from '../form/switch-form';
+import TavilyExtractForm from '../form/tavily-extract-form';
+import TavilyForm from '../form/tavily-form';
+import TemplateForm from '../form/template-form';
+import ToolForm from '../form/tool-form';
+import TuShareForm from '../form/tushare-form';
+import UserFillUpForm from '../form/user-fill-up-form';
+import WenCaiForm from '../form/wencai-form';
+import WikipediaForm from '../form/wikipedia-form';
+import YahooFinanceForm from '../form/yahoo-finance-form';
+
+export const FormConfigMap = {
+  [Operator.Begin]: {
+    component: BeginForm,
+  },
+  [Operator.Retrieval]: {
+    component: RetrievalForm,
+  },
+  [Operator.Generate]: {
+    component: GenerateForm,
+  },
+  [Operator.Answer]: {
+    component: AnswerForm,
+  },
+  [Operator.Categorize]: {
+    component: CategorizeForm,
+  },
+  [Operator.Message]: {
+    component: MessageForm,
+  },
+  [Operator.Relevant]: {
+    component: RelevantForm,
+  },
+  [Operator.RewriteQuestion]: {
+    component: RewriteQuestionForm,
+  },
+  [Operator.Code]: {
+    component: CodeForm,
+  },
+  [Operator.WaitingDialogue]: {
+    component: CodeForm,
+  },
+  [Operator.Agent]: {
+    component: AgentForm,
+    defaultValues: {},
+    schema: z.object({}),
+  },
+  [Operator.Baidu]: {
+    component: BaiduForm,
+  },
+  [Operator.DuckDuckGo]: {
+    component: DuckDuckGoForm,
+  },
+  [Operator.KeywordExtract]: {
+    component: KeywordExtractForm,
+  },
+  [Operator.Wikipedia]: {
+    component: WikipediaForm,
+  },
+  [Operator.PubMed]: {
+    component: PubMedForm,
+  },
+  [Operator.ArXiv]: {
+    component: ArXivForm,
+  },
+  [Operator.Google]: {
+    component: GoogleForm,
+  },
+  [Operator.Bing]: {
+    component: BingForm,
+  },
+  [Operator.GoogleScholar]: {
+    component: GoogleScholarForm,
+  },
+  [Operator.DeepL]: {
+    component: DeepLForm,
+    defaultValues: {},
+    schema: z.object({}),
+  },
+  [Operator.GitHub]: {
+    component: GithubForm,
+  },
+  [Operator.BaiduFanyi]: {
+    component: BaiduFanyiForm,
+  },
+  [Operator.QWeather]: {
+    component: QWeatherForm,
+  },
+  [Operator.ExeSQL]: {
+    component: ExeSQLForm,
+  },
+  [Operator.Switch]: {
+    component: SwitchForm,
+  },
+  [Operator.WenCai]: {
+    component: WenCaiForm,
+  },
+  [Operator.AkShare]: {
+    component: AkShareForm,
+  },
+  [Operator.YahooFinance]: {
+    component: YahooFinanceForm,
+  },
+  [Operator.Jin10]: {
+    component: Jin10Form,
+  },
+  [Operator.TuShare]: {
+    component: TuShareForm,
+  },
+  [Operator.Crawler]: {
+    component: CrawlerForm,
+  },
+  [Operator.Invoke]: {
+    component: InvokeForm,
+  },
+  [Operator.Concentrator]: {
+    component: () => <></>,
+  },
+  [Operator.Note]: {
+    component: () => <></>,
+  },
+  [Operator.Template]: {
+    component: TemplateForm,
+  },
+  [Operator.Email]: {
+    component: EmailForm,
+  },
+  [Operator.Iteration]: {
+    component: IterationForm,
+  },
+  [Operator.IterationStart]: {
+    component: IterationStartForm,
+  },
+  [Operator.Tool]: {
+    component: ToolForm,
+  },
+  [Operator.TavilySearch]: {
+    component: TavilyForm,
+  },
+  [Operator.UserFillUp]: {
+    component: UserFillUpForm,
+  },
+  [Operator.StringTransform]: {
+    component: StringTransformForm,
+  },
+  [Operator.TavilyExtract]: {
+    component: TavilyExtractForm,
+  },
+};

web/src/pages/agent/form-sheet/next.tsx

@@ -19,8 +19,8 @@ import { useHandleNodeNameChange } from '../hooks/use-change-node-name';
 import OperatorIcon from '../operator-icon';
 import useGraphStore from '../store';
 import { needsSingleStepDebugging } from '../utils';
+import { FormConfigMap } from './form-config-map';
 import SingleDebugDrawer from './single-debug-drawer';
-import { useFormConfigMap } from './use-form-config-map';
 
 interface IProps {
   node?: RAGFlowNodeType;
@@ -44,8 +44,6 @@ const FormSheet = ({
   const operatorName: Operator = node?.data.label as Operator;
   const clickedToolId = useGraphStore((state) => state.clickedToolId);
 
-  const FormConfigMap = useFormConfigMap();
-
   const currentFormMap = FormConfigMap[operatorName];
 
   const OperatorForm = currentFormMap?.component ?? EmptyContent;

web/src/pages/agent/form-sheet/use-form-config-map.tsx

@@ -1,400 +0,0 @@
-import { LlmSettingSchema } from '@/components/llm-setting-items/next';
-import { CodeTemplateStrMap, ProgrammingLanguage } from '@/constants/agent';
-import { useTranslation } from 'react-i18next';
-import { z } from 'zod';
-import { Operator } from '../constant';
-import AgentForm from '../form/agent-form';
-import AkShareForm from '../form/akshare-form';
-import AnswerForm from '../form/answer-form';
-import ArXivForm from '../form/arxiv-form';
-import BaiduFanyiForm from '../form/baidu-fanyi-form';
-import BaiduForm from '../form/baidu-form';
-import BeginForm from '../form/begin-form';
-import BingForm from '../form/bing-form';
-import CategorizeForm from '../form/categorize-form';
-import CodeForm from '../form/code-form';
-import CrawlerForm from '../form/crawler-form';
-import DeepLForm from '../form/deepl-form';
-import DuckDuckGoForm from '../form/duckduckgo-form';
-import EmailForm from '../form/email-form';
-import ExeSQLForm from '../form/exesql-form';
-import GenerateForm from '../form/generate-form';
-import GithubForm from '../form/github-form';
-import GoogleForm from '../form/google-form';
-import GoogleScholarForm from '../form/google-scholar-form';
-import InvokeForm from '../form/invoke-form';
-import IterationForm from '../form/iteration-form';
-import IterationStartForm from '../form/iteration-start-from';
-import Jin10Form from '../form/jin10-form';
-import KeywordExtractForm from '../form/keyword-extract-form';
-import MessageForm from '../form/message-form';
-import PubMedForm from '../form/pubmed-form';
-import QWeatherForm from '../form/qweather-form';
-import RelevantForm from '../form/relevant-form';
-import RetrievalForm from '../form/retrieval-form/next';
-import RewriteQuestionForm from '../form/rewrite-question-form';
-import { StringTransformForm } from '../form/string-transform-form';
-import SwitchForm from '../form/switch-form';
-import TavilyForm from '../form/tavily-form';
-import TemplateForm from '../form/template-form';
-import ToolForm from '../form/tool-form';
-import TuShareForm from '../form/tushare-form';
-import UserFillUpForm from '../form/user-fill-up-form';
-import WenCaiForm from '../form/wencai-form';
-import WikipediaForm from '../form/wikipedia-form';
-import YahooFinanceForm from '../form/yahoo-finance-form';
-
-export function useFormConfigMap() {
-  const { t } = useTranslation();
-
-  const FormConfigMap = {
-    [Operator.Begin]: {
-      component: BeginForm,
-      defaultValues: {},
-      schema: z.object({
-        enablePrologue: z.boolean().optional(),
-        prologue: z
-          .string()
-          .min(1, {
-            message: t('common.namePlaceholder'),
-          })
-          .trim()
-          .optional(),
-        mode: z.string(),
-        query: z
-          .array(
-            z.object({
-              key: z.string(),
-              type: z.string(),
-              value: z.string(),
-              optional: z.boolean(),
-              name: z.string(),
-              options: z.array(z.union([z.number(), z.string(), z.boolean()])),
-            }),
-          )
-          .optional(),
-      }),
-    },
-    [Operator.Retrieval]: {
-      component: RetrievalForm,
-      defaultValues: { query: [] },
-      schema: z.object({
-        name: z
-          .string()
-          .min(1, {
-            message: t('common.namePlaceholder'),
-          })
-          .trim(),
-        age: z
-          .string()
-          .min(1, {
-            message: t('common.namePlaceholder'),
-          })
-          .trim(),
-        query: z.array(
-          z.object({
-            type: z.string(),
-          }),
-        ),
-      }),
-    },
-    [Operator.Generate]: {
-      component: GenerateForm,
-      defaultValues: {
-        cite: true,
-        prompt: t('flow.promptText'),
-      },
-      schema: z.object({
-        prompt: z.string().min(1, {
-          message: t('flow.promptMessage'),
-        }),
-      }),
-    },
-    [Operator.Answer]: {
-      component: AnswerForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Categorize]: {
-      component: CategorizeForm,
-      defaultValues: {},
-      schema: z.object({
-        parameter: z.string().optional(),
-        ...LlmSettingSchema,
-        message_history_window_size: z.coerce.number(),
-        items: z.array(
-          z
-            .object({
-              name: z.string().min(1, t('flow.nameMessage')).trim(),
-              description: z.string().optional(),
-              // examples: z
-              //   .array(
-              //     z.object({
-              //       value: z.string(),
-              //     }),
-              //   )
-              //   .optional(),
-            })
-            .optional(),
-        ),
-      }),
-    },
-    [Operator.Message]: {
-      component: MessageForm,
-      defaultValues: {},
-      schema: z.object({
-        content: z
-          .array(
-            z.object({
-              value: z.string(),
-            }),
-          )
-          .optional(),
-      }),
-    },
-    [Operator.Relevant]: {
-      component: RelevantForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.RewriteQuestion]: {
-      component: RewriteQuestionForm,
-      defaultValues: {
-        message_history_window_size: 6,
-      },
-      schema: z.object({
-        llm_id: z.string(),
-        message_history_window_size: z.number(),
-        language: z.string(),
-      }),
-    },
-    [Operator.Code]: {
-      component: CodeForm,
-      defaultValues: {
-        lang: ProgrammingLanguage.Python,
-        script: CodeTemplateStrMap[ProgrammingLanguage.Python],
-        arguments: [],
-      },
-      schema: z.object({
-        lang: z.string(),
-        script: z.string(),
-        arguments: z.array(
-          z.object({ name: z.string(), component_id: z.string() }),
-        ),
-        return: z.union([
-          z
-            .array(z.object({ name: z.string(), component_id: z.string() }))
-            .optional(),
-          z.object({ name: z.string(), component_id: z.string() }),
-        ]),
-      }),
-    },
-    [Operator.WaitingDialogue]: {
-      component: CodeForm,
-      defaultValues: {},
-      schema: z.object({
-        arguments: z.array(
-          z.object({ name: z.string(), component_id: z.string() }),
-        ),
-      }),
-    },
-    [Operator.Agent]: {
-      component: AgentForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Baidu]: {
-      component: BaiduForm,
-      defaultValues: { top_n: 10 },
-      schema: z.object({ top_n: z.number() }),
-    },
-    [Operator.DuckDuckGo]: {
-      component: DuckDuckGoForm,
-      defaultValues: {
-        top_n: 10,
-        channel: 'text',
-      },
-      schema: z.object({
-        top_n: z.number(),
-      }),
-    },
-    [Operator.KeywordExtract]: {
-      component: KeywordExtractForm,
-      defaultValues: { top_n: 3 },
-      schema: z.object({
-        llm_id: z.string(),
-        top_n: z.number(),
-      }),
-    },
-    [Operator.Wikipedia]: {
-      component: WikipediaForm,
-      defaultValues: {
-        top_n: 10,
-      },
-      schema: z.object({
-        language: z.string(),
-        top_n: z.number(),
-      }),
-    },
-    [Operator.PubMed]: {
-      component: PubMedForm,
-      defaultValues: { top_n: 10 },
-      schema: z.object({
-        top_n: z.number(),
-        email: z.string(),
-      }),
-    },
-    [Operator.ArXiv]: {
-      component: ArXivForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Google]: {
-      component: GoogleForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Bing]: {
-      component: BingForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.GoogleScholar]: {
-      component: GoogleScholarForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.DeepL]: {
-      component: DeepLForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.GitHub]: {
-      component: GithubForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.BaiduFanyi]: {
-      component: BaiduFanyiForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.QWeather]: {
-      component: QWeatherForm,
-      defaultValues: {},
-      schema: z.object({
-        web_apikey: z.string(),
-        lang: z.string(),
-        type: z.string(),
-        user_type: z.string(),
-        time_period: z.string().optional(),
-      }),
-    },
-    [Operator.ExeSQL]: {
-      component: ExeSQLForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Switch]: {
-      component: SwitchForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.WenCai]: {
-      component: WenCaiForm,
-      defaultValues: {
-        top_n: 20,
-      },
-      schema: z.object({
-        top_n: z.number(),
-        query_type: z.string(),
-      }),
-    },
-    [Operator.AkShare]: {
-      component: AkShareForm,
-      defaultValues: {
-        top_n: 10,
-      },
-      schema: z.object({
-        top_n: z.number(),
-      }),
-    },
-    [Operator.YahooFinance]: {
-      component: YahooFinanceForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Jin10]: {
-      component: Jin10Form,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.TuShare]: {
-      component: TuShareForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Crawler]: {
-      component: CrawlerForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Invoke]: {
-      component: InvokeForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Concentrator]: {
-      component: () => <></>,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Note]: {
-      component: () => <></>,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Template]: {
-      component: TemplateForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Email]: {
-      component: EmailForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Iteration]: {
-      component: IterationForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.IterationStart]: {
-      component: IterationStartForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.Tool]: {
-      component: ToolForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.TavilySearch]: {
-      component: TavilyForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.UserFillUp]: {
-      component: UserFillUpForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-    [Operator.StringTransform]: {
-      component: StringTransformForm,
-      defaultValues: {},
-      schema: z.object({}),
-    },
-  };
-
-  return FormConfigMap;
-}

web/src/pages/agent/form-sheet/use-values.ts

@@ -1,42 +0,0 @@
-import { RAGFlowNodeType } from '@/interfaces/database/flow';
-import { get, isEmpty, isPlainObject, omit } from 'lodash';
-import { useMemo, useRef } from 'react';
-import { Operator } from '../constant';
-import { buildCategorizeListFromObject, convertToObjectArray } from '../utils';
-import { useFormConfigMap } from './use-form-config-map';
-
-export function useValues(node?: RAGFlowNodeType, isDirty?: boolean) {
-  const operatorName: Operator = node?.data.label as Operator;
-  const previousId = useRef<string | undefined>(node?.id);
-
-  const FormConfigMap = useFormConfigMap();
-
-  const currentFormMap = FormConfigMap[operatorName];
-
-  const values = useMemo(() => {
-    const formData = node?.data?.form;
-    if (operatorName === Operator.Categorize) {
-      const items = buildCategorizeListFromObject(
-        get(node, 'data.form.category_description', {}),
-      );
-      if (isPlainObject(formData)) {
-        console.info('xxx');
-        const nextValues = {
-          ...omit(formData, 'category_description'),
-          items,
-        };
-
-        return nextValues;
-      }
-    } else if (operatorName === Operator.Message) {
-      return {
-        ...formData,
-        content: convertToObjectArray(formData.content),
-      };
-    } else {
-      return isEmpty(formData) ? currentFormMap : formData;
-    }
-  }, [currentFormMap, node, operatorName]);
-
-  return values;
-}

web/src/pages/agent/form/agent-form/tool-popover/tool-command.tsx

@@ -18,6 +18,7 @@ const Menus = [
     label: 'Search',
     list: [
       Operator.TavilySearch,
+      Operator.TavilyExtract,
       Operator.Google,
       Operator.Bing,
       Operator.DuckDuckGo,
@@ -41,7 +42,6 @@ const Menus = [
       Operator.GitHub,
       Operator.ExeSQL,
       Operator.Invoke,
-      Operator.Crawler,
       Operator.Code,
       Operator.Retrieval,
     ],

web/src/pages/agent/form/exesql-form/index.tsx

@@ -1,6 +1,5 @@
-import { LargeModelFormField } from '@/components/large-model-form-field';
+import NumberInput from '@/components/originui/number-input';
 import { SelectWithSearch } from '@/components/originui/select-with-search';
-import { TopNFormField } from '@/components/top-n-item';
 import { ButtonLoading } from '@/components/ui/button';
 import {
   Form,
@@ -10,7 +9,7 @@ import {
   FormLabel,
   FormMessage,
 } from '@/components/ui/form';
-import { Input, NumberInput } from '@/components/ui/input';
+import { Input } from '@/components/ui/input';
 import { useTranslate } from '@/hooks/common-hooks';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { memo } from 'react';
@@ -31,7 +30,6 @@ export function ExeSQLFormWidgets({ loading }: { loading: boolean }) {
 
   return (
     <>
-      <LargeModelFormField></LargeModelFormField>
       <FormField
         control={form.control}
         name="db_type"
@@ -94,7 +92,7 @@ export function ExeSQLFormWidgets({ loading }: { loading: boolean }) {
           <FormItem>
             <FormLabel>{t('port')}</FormLabel>
             <FormControl>
-              <NumberInput {...field}></NumberInput>
+              <NumberInput {...field} className="w-full"></NumberInput>
             </FormControl>
             <FormMessage />
           </FormItem>
@@ -113,20 +111,21 @@ export function ExeSQLFormWidgets({ loading }: { loading: boolean }) {
           </FormItem>
         )}
       />
+
       <FormField
         control={form.control}
-        name="loop"
+        name="max_records"
         render={({ field }) => (
           <FormItem>
-            <FormLabel tooltip={t('loopTip')}>{t('loop')}</FormLabel>
+            <FormLabel>{t('maxRecords')}</FormLabel>
             <FormControl>
-              <NumberInput {...field}></NumberInput>
+              <NumberInput {...field} className="w-full"></NumberInput>
             </FormControl>
             <FormMessage />
           </FormItem>
         )}
       />
-      <TopNFormField max={1000}></TopNFormField>
+
       <div className="flex justify-end">
         <ButtonLoading loading={loading} type="submit">
           Test
@@ -151,7 +150,7 @@ function ExeSQLForm({ node }: INextOperatorForm) {
   return (
     <Form {...form}>
       <FormWrapper onSubmit={form.handleSubmit(onSubmit)}>
-        <QueryVariable></QueryVariable>
+        <QueryVariable name="sql"></QueryVariable>
         <ExeSQLFormWidgets loading={loading}></ExeSQLFormWidgets>
       </FormWrapper>
     </Form>

web/src/pages/agent/form/exesql-form/use-submit-form.ts

@@ -3,15 +3,14 @@ import { useCallback } from 'react';
 import { z } from 'zod';
 
 export const ExeSQLFormSchema = {
-  llm_id: z.string().min(1),
+  sql: z.string(),
   db_type: z.string().min(1),
   database: z.string().min(1),
   username: z.string().min(1),
   host: z.string().min(1),
   port: z.number(),
   password: z.string().min(1),
-  loop: z.number(),
-  top_n: z.number(),
+  max_records: z.number(),
 };
 
 export const FormSchema = z.object({

web/src/pages/agent/form/tavily-extract-form/index.tsx

@@ -0,0 +1,115 @@
+import { FormContainer } from '@/components/form-container';
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import { Input } from '@/components/ui/input';
+import { RAGFlowSelect } from '@/components/ui/select';
+import { buildOptions } from '@/utils/form';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { memo } from 'react';
+import { useForm } from 'react-hook-form';
+import { z } from 'zod';
+import {
+  TavilyExtractDepth,
+  TavilyExtractFormat,
+  initialTavilyExtractValues,
+} from '../../constant';
+import { useFormValues } from '../../hooks/use-form-values';
+import { useWatchFormChange } from '../../hooks/use-watch-form-change';
+import { INextOperatorForm } from '../../interface';
+import { buildOutputList } from '../../utils/build-output-list';
+import { FormWrapper } from '../components/form-wrapper';
+import { Output } from '../components/output';
+import { TavilyApiKeyField, TavilyFormSchema } from '../tavily-form';
+
+const outputList = buildOutputList(initialTavilyExtractValues.outputs);
+
+function TavilyExtractForm({ node }: INextOperatorForm) {
+  const values = useFormValues(initialTavilyExtractValues, node);
+
+  const FormSchema = z.object({
+    ...TavilyFormSchema,
+    urls: z.string(),
+    extract_depth: z.enum([
+      TavilyExtractDepth.Advanced,
+      TavilyExtractDepth.Basic,
+    ]),
+    format: z.enum([TavilyExtractFormat.Text, TavilyExtractFormat.Markdown]),
+  });
+
+  const form = useForm<z.infer<typeof FormSchema>>({
+    defaultValues: values,
+    resolver: zodResolver(FormSchema),
+  });
+
+  useWatchFormChange(node?.id, form);
+
+  return (
+    <Form {...form}>
+      <FormWrapper>
+        <FormContainer>
+          <TavilyApiKeyField></TavilyApiKeyField>
+        </FormContainer>
+        <FormContainer>
+          <FormField
+            control={form.control}
+            name="urls"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>URL</FormLabel>
+                <FormControl>
+                  <Input {...field} />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <FormField
+            control={form.control}
+            name="extract_depth"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Extract Depth</FormLabel>
+                <FormControl>
+                  <RAGFlowSelect
+                    placeholder="shadcn"
+                    {...field}
+                    options={buildOptions(TavilyExtractDepth)}
+                  />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+          <FormField
+            control={form.control}
+            name="format"
+            render={({ field }) => (
+              <FormItem>
+                <FormLabel>Format</FormLabel>
+                <FormControl>
+                  <RAGFlowSelect
+                    placeholder="shadcn"
+                    {...field}
+                    options={buildOptions(TavilyExtractFormat)}
+                  />
+                </FormControl>
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+        </FormContainer>
+      </FormWrapper>
+      <div className="p-5">
+        <Output list={outputList}></Output>
+      </div>
+    </Form>
+  );
+}
+
+export default memo(TavilyExtractForm);

web/src/pages/agent/form/tavily-form/index.tsx

@@ -13,7 +13,7 @@ import { Switch } from '@/components/ui/switch';
 import { buildOptions } from '@/utils/form';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { memo, useMemo } from 'react';
-import { useForm } from 'react-hook-form';
+import { useForm, useFormContext } from 'react-hook-form';
 import { z } from 'zod';
 import {
   TavilySearchDepth,
@@ -21,17 +21,41 @@ import {
   initialTavilyValues,
 } from '../../constant';
 import { INextOperatorForm } from '../../interface';
+import { FormWrapper } from '../components/form-wrapper';
 import { Output, OutputType } from '../components/output';
 import { QueryVariable } from '../components/query-variable';
 import { DynamicDomain } from './dynamic-domain';
 import { useValues } from './use-values';
 import { useWatchFormChange } from './use-watch-change';
 
+export function TavilyApiKeyField() {
+  const form = useFormContext();
+  return (
+    <FormField
+      control={form.control}
+      name="api_key"
+      render={({ field }) => (
+        <FormItem>
+          <FormLabel>Api Key</FormLabel>
+          <FormControl>
+            <Input type="password" {...field}></Input>
+          </FormControl>
+          <FormMessage />
+        </FormItem>
+      )}
+    />
+  );
+}
+
+export const TavilyFormSchema = {
+  api_key: z.string(),
+};
+
 function TavilyForm({ node }: INextOperatorForm) {
   const values = useValues(node);
 
   const FormSchema = z.object({
-    api_key: z.string(),
+    ...TavilyFormSchema,
     query: z.string(),
     search_depth: z.enum([TavilySearchDepth.Advanced, TavilySearchDepth.Basic]),
     topic: z.enum([TavilyTopic.News, TavilyTopic.General]),
@@ -64,27 +88,9 @@ function TavilyForm({ node }: INextOperatorForm) {
 
   return (
     <Form {...form}>
-      <form
-        className="space-y-5 px-5 "
-        autoComplete="off"
-        onSubmit={(e) => {
-          e.preventDefault();
-        }}
-      >
+      <FormWrapper>
         <FormContainer>
-          <FormField
-            control={form.control}
-            name="api_key"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>Api Key</FormLabel>
-                <FormControl>
-                  <Input type="password" {...field}></Input>
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
+          <TavilyApiKeyField></TavilyApiKeyField>
         </FormContainer>
         <FormContainer>
           <QueryVariable></QueryVariable>
@@ -221,7 +227,7 @@ function TavilyForm({ node }: INextOperatorForm) {
             label={'Exclude Domains'}
           ></DynamicDomain>
         </FormContainer>
-      </form>
+      </FormWrapper>
       <div className="p-5">
         <Output list={outputList}></Output>
       </div>

web/src/pages/agent/form/tavily-form/use-watch-change.ts

@@ -9,7 +9,7 @@ export function useWatchFormChange(id?: string, form?: UseFormReturn<any>) {
 
   useEffect(() => {
     // Manually triggered form updates are synchronized to the canvas
-    if (id && form?.formState.isDirty) {
+    if (id) {
       values = form?.getValues();
       let nextValues: any = {
         ...values,

web/src/pages/agent/form/tool-form/constant.tsx

@@ -34,4 +34,5 @@ export const ToolFormConfigMap = {
   [Operator.Crawler]: CrawlerForm,
   [Operator.Email]: EmailForm,
   [Operator.TavilySearch]: TavilyForm,
+  [Operator.TavilyExtract]: TavilyForm,
 };

web/src/pages/agent/form/tool-form/tavily-form/index.tsx

@@ -1,25 +1,17 @@
 import { FormContainer } from '@/components/form-container';
-import {
-  Form,
-  FormControl,
-  FormField,
-  FormItem,
-  FormLabel,
-  FormMessage,
-} from '@/components/ui/form';
-import { Input } from '@/components/ui/input';
+import { Form } from '@/components/ui/form';
 import { zodResolver } from '@hookform/resolvers/zod';
 import { useForm } from 'react-hook-form';
 import { z } from 'zod';
+import { FormWrapper } from '../../components/form-wrapper';
+import { TavilyApiKeyField, TavilyFormSchema } from '../../tavily-form';
 import { useValues } from '../use-values';
 import { useWatchFormChange } from '../use-watch-change';
 
 const TavilyForm = () => {
   const values = useValues();
 
-  const FormSchema = z.object({
-    api_key: z.string(),
-  });
+  const FormSchema = z.object(TavilyFormSchema);
 
   const form = useForm<z.infer<typeof FormSchema>>({
     defaultValues: values,
@@ -30,29 +22,11 @@ const TavilyForm = () => {
 
   return (
     <Form {...form}>
-      <form
-        className="space-y-5 px-5 "
-        autoComplete="off"
-        onSubmit={(e) => {
-          e.preventDefault();
-        }}
-      >
+      <FormWrapper>
         <FormContainer>
-          <FormField
-            control={form.control}
-            name="api_key"
-            render={({ field }) => (
-              <FormItem>
-                <FormLabel>Api Key</FormLabel>
-                <FormControl>
-                  <Input type="password" {...field}></Input>
-                </FormControl>
-                <FormMessage />
-              </FormItem>
-            )}
-          />
+          <TavilyApiKeyField></TavilyApiKeyField>
         </FormContainer>
-      </form>
+      </FormWrapper>
     </Form>
   );
 };

web/src/pages/agent/hooks/use-add-node.ts

@@ -41,6 +41,7 @@ import {
   initialRewriteQuestionValues,
   initialStringTransformValues,
   initialSwitchValues,
+  initialTavilyExtractValues,
   initialTavilyValues,
   initialTemplateValues,
   initialTuShareValues,
@@ -94,7 +95,7 @@ export const useInitializeOperatorParams = () => {
       [Operator.GitHub]: initialGithubValues,
       [Operator.BaiduFanyi]: initialBaiduFanyiValues,
       [Operator.QWeather]: initialQWeatherValues,
-      [Operator.ExeSQL]: { ...initialExeSqlValues, llm_id: llmId },
+      [Operator.ExeSQL]: initialExeSqlValues,
       [Operator.Switch]: initialSwitchValues,
       [Operator.WenCai]: initialWenCaiValues,
       [Operator.AkShare]: initialAkShareValues,
@@ -116,6 +117,7 @@ export const useInitializeOperatorParams = () => {
       [Operator.TavilySearch]: initialTavilyValues,
       [Operator.UserFillUp]: initialUserFillUpValues,
       [Operator.StringTransform]: initialStringTransformValues,
+      [Operator.TavilyExtract]: initialTavilyExtractValues,
     };
   }, [llmId]);
 

web/src/pages/agent/hooks/use-agent-tool-initial-values.ts

@@ -16,12 +16,12 @@ export function useAgentToolInitialValues() {
             ...omit(initialValues, 'query'),
             description: '',
           };
-        case Operator.TavilySearch:
+        case (Operator.TavilySearch, Operator.TavilyExtract):
           return {
             api_key: '',
           };
         case Operator.ExeSQL:
-          return omit(initialValues, 'query');
+          return omit(initialValues, 'sql');
         case Operator.Bing:
           return omit(initialValues, 'query');