add datasource empty node

web/app/components/rag-pipeline/conversion.tsx → web/app/components/rag-pipeline/components/conversion.tsx

@@ -1,13 +1,13 @@
 import React, { useCallback, useState } from 'react'
 import { useTranslation } from 'react-i18next'
-import Button from '../base/button'
+import Button from '@/app/components/base/button'
 import PipelineScreenShot from './screenshot'
-import Confirm from '../base/confirm'
+import Confirm from '@/app/components/base/confirm'
 import { useConvertDatasetToPipeline } from '@/service/use-pipeline'
 import { useParams } from 'next/navigation'
 import { useInvalid } from '@/service/use-base'
 import { datasetDetailQueryKeyPrefix } from '@/service/knowledge/use-dataset'
-import Toast from '../base/toast'
+import Toast from '@/app/components/base/toast'
 
 const Conversion = () => {
   const { t } = useTranslation()

web/app/components/rag-pipeline/hooks/use-available-nodes-meta-data.ts

@@ -3,6 +3,7 @@ import { useTranslation } from 'react-i18next'
 import { useGetLanguage } from '@/context/i18n'
 import knowledgeBaseDefault from '@/app/components/workflow/nodes/knowledge-base/default'
 import dataSourceDefault from '@/app/components/workflow/nodes/data-source/default'
+import dataSourceEmptyDefault from '@/app/components/workflow/nodes/data-source-empty/default'
 import { WORKFLOW_COMMON_NODES } from '@/app/components/workflow/constants/node'
 import type { AvailableNodesMetaData } from '@/app/components/workflow/hooks-store/store'
 
@@ -32,6 +33,13 @@ export const useAvailableNodesMetaData = () => {
         isUndeletable: true,
       },
     },
+    {
+      ...dataSourceEmptyDefault,
+      metaData: {
+        ...dataSourceEmptyDefault.metaData,
+        isUndeletable: true,
+      },
+    },
   ], [])
 
   const prefixLink = useMemo(() => {

web/app/components/rag-pipeline/hooks/use-nodes-sync-draft.ts

@@ -10,6 +10,7 @@ import {
 import { API_PREFIX } from '@/config'
 import { syncWorkflowDraft } from '@/service/workflow'
 import { usePipelineRefreshDraft } from '.'
+import { CUSTOM_DATA_SOURCE_EMPTY_NODE } from '@/app/components/workflow/nodes/data-source-empty/constants'
 
 export const useNodesSyncDraft = () => {
   const store = useStoreApi()
@@ -23,7 +24,8 @@ export const useNodesSyncDraft = () => {
       edges,
       transform,
     } = store.getState()
-    const nodes = getNodes()
+    const nodesOriginal = getNodes()
+    const nodes = nodesOriginal.filter(node => node.type !== CUSTOM_DATA_SOURCE_EMPTY_NODE)
     const [x, y, zoom] = transform
     const {
       pipelineId,

web/app/components/rag-pipeline/index.tsx

@@ -13,7 +13,9 @@ import { createRagPipelineSliceSlice } from './store'
 import RagPipelineMain from './components/rag-pipeline-main'
 import { usePipelineInit } from './hooks'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
-import Conversion from './conversion'
+import Conversion from './components/conversion'
+import type { Node } from '@/app/components/workflow/types'
+import { processNodesWithoutDataSource } from './utils'
 
 const RagPipeline = () => {
   const {
@@ -21,10 +23,11 @@ const RagPipeline = () => {
     isLoading,
   } = usePipelineInit()
   const nodesData = useMemo(() => {
+    let result: Node[] = []
     if (data)
-      return initialNodes(data.graph.nodes, data.graph.edges)
+      result = initialNodes(data.graph.nodes, data.graph.edges)
 
-    return []
+    return processNodesWithoutDataSource(result)
   }, [data])
   const edgesData = useMemo(() => {
     if (data)

web/app/components/rag-pipeline/utils/index.ts

@@ -0,0 +1 @@
+export * from './nodes'

web/app/components/rag-pipeline/utils/nodes.ts

@@ -0,0 +1,77 @@
+import type { Node } from '@/app/components/workflow/types'
+import { BlockEnum } from '@/app/components/workflow/types'
+import { generateNewNode } from '@/app/components/workflow/utils'
+import { CUSTOM_DATA_SOURCE_EMPTY_NODE } from '@/app/components/workflow/nodes/data-source-empty/constants'
+import { CUSTOM_NOTE_NODE } from '@/app/components/workflow/note-node/constants'
+import { NoteTheme } from '@/app/components/workflow/note-node/types'
+import type { NoteNodeType } from '@/app/components/workflow/note-node/types'
+import { CUSTOM_NODE } from '@/app/components/workflow/constants'
+
+export const processNodesWithoutDataSource = (nodes: Node[]) => {
+  if (!nodes || nodes.length === 0) return []
+
+  let leftNode
+  let hasNoteBySystem
+  for (let i = 0; i < nodes.length; i++) {
+    const node = nodes[i]
+    if (node.type === CUSTOM_NOTE_NODE && node.data.noteBySystem)
+      hasNoteBySystem = true
+
+    if (node.type !== CUSTOM_NODE)
+      continue
+
+    if (node.data.type === BlockEnum.DataSource)
+      return nodes
+
+    if (!leftNode)
+      leftNode = node
+
+    if (node.position.x < leftNode.position.x)
+      leftNode = node
+  }
+
+  if (leftNode) {
+    const { newNode } = generateNewNode({
+      type: CUSTOM_DATA_SOURCE_EMPTY_NODE,
+      data: {
+        title: '',
+        desc: '',
+        type: BlockEnum.DataSourceEmpty,
+        width: 240,
+      },
+      position: {
+        x: leftNode.position.x - 500,
+        y: leftNode.position.y,
+      },
+    })
+    let newNoteNode
+    if (!hasNoteBySystem) {
+      newNoteNode = generateNewNode({
+        type: CUSTOM_NOTE_NODE,
+        data: {
+          title: '',
+          desc: '',
+          type: '' as any,
+          text: '{"root":{"children":[{"children":[{"detail":0,"format":1,"mode":"normal","style":"font-size: 14px;","text":"Get started with a blank pipeline","type":"text","version":1}],"direction":"ltr","format":"","indent":0,"type":"paragraph","version":1,"textFormat":1,"textStyle":"font-size: 14px;"},{"children":[],"direction":"ltr","format":"","indent":0,"type":"paragraph","version":1,"textFormat":1,"textStyle":""},{"children":[{"detail":0,"format":0,"mode":"normal","style":"","text":"A Knowledge Pipeline starts with Data Source as the starting node and ends with the knowledge base node. The general steps are: import documents from the data source → use extractor to extract document content → split and clean content into structured chunks → store in the knowledge base.","type":"text","version":1}],"direction":"ltr","format":"","indent":0,"type":"paragraph","version":1,"textFormat":0,"textStyle":""},{"children":[],"direction":"ltr","format":"","indent":0,"type":"paragraph","version":1,"textFormat":0,"textStyle":""},{"children":[{"children":[{"detail":0,"format":2,"mode":"normal","style":"","text":"Link to documentation","type":"text","version":1}],"direction":"ltr","format":"","indent":0,"type":"link","version":1,"textFormat":2,"rel":"noreferrer","target":null,"title":null,"url":"https://dify.ai"}],"direction":"ltr","format":"","indent":0,"type":"paragraph","version":1,"textFormat":2,"textStyle":""},{"children":[],"direction":"ltr","format":"","indent":0,"type":"paragraph","version":1,"textFormat":0,"textStyle":""}],"direction":"ltr","format":"","indent":0,"type":"root","version":1,"textFormat":1,"textStyle":"font-size: 14px;"}}',
+          theme: NoteTheme.blue,
+          author: '',
+          showAuthor: true,
+          width: 240,
+          height: 300,
+          noteBySystem: true,
+        } as NoteNodeType,
+        position: {
+          x: leftNode.position.x - 500,
+          y: leftNode.position.y + 100,
+        },
+      }).newNode
+    }
+    return [
+      newNode,
+      ...(newNoteNode ? [newNoteNode] : []),
+      ...nodes,
+    ]
+  }
+
+  return nodes
+}

web/app/components/workflow/block-selector/data-sources.tsx

@@ -2,6 +2,9 @@ import {
   useCallback,
   useRef,
 } from 'react'
+import Link from 'next/link'
+import { useTranslation } from 'react-i18next'
+import { RiArrowRightUpLine } from '@remixicon/react'
 import { BlockEnum } from '../types'
 import type {
   OnSelectBlock,
@@ -12,6 +15,8 @@ import Tools from './tools'
 import { ViewType } from './view-type-select'
 import cn from '@/utils/classnames'
 import type { ListRef } from '@/app/components/workflow/block-selector/market-place-plugin/list'
+import { getMarketplaceUrl } from '@/utils/var'
+import { useGlobalPublicStore } from '@/context/global-public-context'
 
 type AllToolsProps = {
   className?: string
@@ -28,6 +33,7 @@ const DataSources = ({
   onSelect,
   dataSources,
 }: AllToolsProps) => {
+  const { t } = useTranslation()
   const pluginRef = useRef<ListRef>(null)
   const wrapElemRef = useRef<HTMLDivElement>(null)
   const handleSelect = useCallback((_: any, toolDefaultValue: ToolDefaultValue) => {
@@ -40,6 +46,7 @@ const DataSources = ({
       title: toolDefaultValue?.title,
     })
   }, [onSelect])
+  const { enable_marketplace } = useGlobalPublicStore(s => s.systemFeatures)
 
   return (
     <div className={cn(className)}>
@@ -56,6 +63,18 @@ const DataSources = ({
           hasSearchText={!!searchText}
           canNotSelectMultiple
         />
+        {
+          enable_marketplace && (
+            <Link
+              className='system-sm-medium sticky bottom-0 z-10 flex h-8 cursor-pointer items-center rounded-b-lg border-[0.5px] border-t border-components-panel-border bg-components-panel-bg-blur px-4 py-1 text-text-accent-light-mode-only shadow-lg'
+              href={getMarketplaceUrl('')}
+              target='_blank'
+            >
+              <span>{t('plugin.findMoreInMarketplace')}</span>
+              <RiArrowRightUpLine className='ml-0.5 h-3 w-3' />
+            </Link>
+          )
+        }
       </div>
     </div>
   )

web/app/components/workflow/block-selector/hooks.ts

@@ -8,7 +8,7 @@ import {
   ToolTypeEnum,
 } from './types'
 
-export const useTabs = (noBlocks?: boolean, noSources?: boolean) => {
+export const useTabs = (noBlocks?: boolean, noSources?: boolean, noTools?: boolean) => {
   const { t } = useTranslation()
   const tabs = useMemo(() => {
     return [
@@ -32,18 +32,27 @@ export const useTabs = (noBlocks?: boolean, noSources?: boolean) => {
             },
           ]
       ),
-      {
-        key: TabsEnum.Tools,
-        name: t('workflow.tabs.tools'),
-      },
+      ...(
+        noTools
+          ? []
+          : [
+            {
+              key: TabsEnum.Tools,
+              name: t('workflow.tabs.tools'),
+            },
+          ]
+      ),
     ]
-  }, [t, noBlocks, noSources])
+  }, [t, noBlocks, noSources, noTools])
   const initialTab = useMemo(() => {
     if (noBlocks)
-      return noSources ? TabsEnum.Tools : TabsEnum.Sources
+      return noTools ? TabsEnum.Sources : TabsEnum.Tools
+
+    if (noTools)
+      return noBlocks ? TabsEnum.Sources : TabsEnum.Blocks
 
     return TabsEnum.Blocks
-  }, [noBlocks, noSources])
+  }, [noBlocks, noSources, noTools])
   const [activeTab, setActiveTab] = useState(initialTab)
 
   return {

web/app/components/workflow/block-selector/index.tsx

@@ -30,6 +30,9 @@ const NodeSelectorWrapper = (props: NodeSelectorProps) => {
       if (block.metaData.type === BlockEnum.LoopStart)
         return false
 
+      if (block.metaData.type === BlockEnum.DataSourceEmpty)
+        return false
+
       return true
     })
   }, [availableNodesMetaData?.nodes])

web/app/components/workflow/block-selector/main.tsx

@@ -50,6 +50,7 @@ export type NodeSelectorProps = {
   blocks?: NodeDefault[]
   dataSources?: ToolWithProvider[]
   noBlocks?: boolean
+  noTools?: boolean
 }
 const NodeSelector: FC<NodeSelectorProps> = ({
   open: openFromProps,
@@ -68,6 +69,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
   blocks = [],
   dataSources = [],
   noBlocks = false,
+  noTools = false,
 }) => {
   const { t } = useTranslation()
   const [searchText, setSearchText] = useState('')
@@ -98,7 +100,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
     activeTab,
     setActiveTab,
     tabs,
-  } = useTabs(!blocks.length, !dataSources.length)
+  } = useTabs(noBlocks, !dataSources.length, noTools)
 
   const handleActiveTabChange = useCallback((newActiveTab: TabsEnum) => {
     setActiveTab(newActiveTab)
@@ -165,6 +167,17 @@ const NodeSelector: FC<NodeSelectorProps> = ({
                     onClear={() => setSearchText('')}
                   />
                 )}
+                {activeTab === TabsEnum.Sources && (
+                  <Input
+                    showLeftIcon
+                    showClearIcon
+                    autoFocus
+                    value={searchText}
+                    placeholder={searchPlaceholder}
+                    onChange={e => setSearchText(e.target.value)}
+                    onClear={() => setSearchText('')}
+                  />
+                )}
                 {activeTab === TabsEnum.Tools && (
                   <SearchBox
                     search={searchText}
@@ -184,6 +197,7 @@ const NodeSelector: FC<NodeSelectorProps> = ({
             availableBlocksTypes={availableBlocksTypes}
             noBlocks={noBlocks}
             dataSources={dataSources}
+            noTools={noTools}
           />
         </div>
       </PortalToFollowElemContent>

web/app/components/workflow/block-selector/tabs.tsx

@@ -28,6 +28,7 @@ export type TabsProps = {
   }>
   filterElem: React.ReactNode
   noBlocks?: boolean
+  noTools?: boolean
 }
 const Tabs: FC<TabsProps> = ({
   activeTab,
@@ -41,6 +42,7 @@ const Tabs: FC<TabsProps> = ({
   tabs = [],
   filterElem,
   noBlocks,
+  noTools,
 }) => {
   const { data: buildInTools } = useAllBuiltInTools()
   const { data: customTools } = useAllCustomTools()
@@ -96,7 +98,7 @@ const Tabs: FC<TabsProps> = ({
         )
       }
       {
-        activeTab === TabsEnum.Tools && (
+        activeTab === TabsEnum.Tools && !noTools && (
           <AllTools
             searchText={searchText}
             onSelect={onSelect}

web/app/components/workflow/hooks/use-nodes-interactions.ts

@@ -344,6 +344,8 @@ export const useNodesInteractions = () => {
       return
     if (node.type === CUSTOM_LOOP_START_NODE)
       return
+    if (node.data.type === BlockEnum.DataSourceEmpty)
+      return
     handleNodeSelect(node.id)
   }, [handleNodeSelect])
 
@@ -1265,13 +1267,13 @@ export const useNodesInteractions = () => {
     if (nodeId) {
       // If nodeId is provided, copy that specific node
       const nodeToCopy = nodes.find(node => node.id === nodeId && node.data.type !== BlockEnum.Start
-        && node.type !== CUSTOM_ITERATION_START_NODE && node.type !== CUSTOM_LOOP_START_NODE && node.data.type !== BlockEnum.LoopEnd && node.data.type !== BlockEnum.KnowledgeBase)
+        && node.type !== CUSTOM_ITERATION_START_NODE && node.type !== CUSTOM_LOOP_START_NODE && node.data.type !== BlockEnum.LoopEnd && node.data.type !== BlockEnum.KnowledgeBase && node.data.type !== BlockEnum.DataSourceEmpty)
       if (nodeToCopy)
         setClipboardElements([nodeToCopy])
     }
     else {
       // If no nodeId is provided, fall back to the current behavior
-      const bundledNodes = nodes.filter(node => node.data._isBundled && node.data.type !== BlockEnum.Start && node.data.type !== BlockEnum.DataSource && node.data.type !== BlockEnum.KnowledgeBase
+      const bundledNodes = nodes.filter(node => node.data._isBundled && node.data.type !== BlockEnum.Start && node.data.type !== BlockEnum.DataSource && node.data.type !== BlockEnum.KnowledgeBase && node.data.type !== BlockEnum.DataSourceEmpty
         && !node.data.isInIteration && !node.data.isInLoop)
 
       if (bundledNodes.length) {

web/app/components/workflow/index.tsx

@@ -57,6 +57,8 @@ import CustomLoopStartNode from './nodes/loop-start'
 import { CUSTOM_LOOP_START_NODE } from './nodes/loop-start/constants'
 import CustomSimpleNode from './simple-node'
 import { CUSTOM_SIMPLE_NODE } from './simple-node/constants'
+import CustomDataSourceEmptyNode from './nodes/data-source-empty'
+import { CUSTOM_DATA_SOURCE_EMPTY_NODE } from './nodes/data-source-empty/constants'
 import Operator from './operator'
 import Control from './operator/control'
 import CustomEdge from './custom-edge'
@@ -94,6 +96,7 @@ const nodeTypes = {
   [CUSTOM_SIMPLE_NODE]: CustomSimpleNode,
   [CUSTOM_ITERATION_START_NODE]: CustomIterationStartNode,
   [CUSTOM_LOOP_START_NODE]: CustomLoopStartNode,
+  [CUSTOM_DATA_SOURCE_EMPTY_NODE]: CustomDataSourceEmptyNode,
 }
 const edgeTypes = {
   [CUSTOM_EDGE]: CustomEdge,
@@ -190,7 +193,6 @@ export const Workflow: FC<WorkflowProps> = memo(({
     return () => {
       handleSyncWorkflowDraft(true, true)
     }
-    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
   const { handleRefreshWorkflowDraft } = useWorkflowRefreshDraft()
@@ -282,7 +284,6 @@ export const Workflow: FC<WorkflowProps> = memo(({
   const { fetchInspectVars } = useSetWorkflowVarsWithValue()
   useEffect(() => {
     fetchInspectVars()
-    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [])
 
   const store = useStoreApi()

web/app/components/workflow/nodes/data-source-empty/constants.ts

@@ -0,0 +1 @@
+export const CUSTOM_DATA_SOURCE_EMPTY_NODE = 'custom-data-source-empty'

web/app/components/workflow/nodes/data-source-empty/default.ts

@@ -0,0 +1,20 @@
+import type { NodeDefault } from '../../types'
+import type { DataSourceEmptyNodeType } from './types'
+import { genNodeMetaData } from '@/app/components/workflow/utils'
+import { BlockEnum } from '@/app/components/workflow/types'
+
+const metaData = genNodeMetaData({
+  sort: -1,
+  type: BlockEnum.DataSourceEmpty,
+})
+const nodeDefault: NodeDefault<DataSourceEmptyNodeType> = {
+  metaData,
+  defaultValue: {},
+  checkValid() {
+    return {
+      isValid: true,
+    }
+  },
+}
+
+export default nodeDefault

web/app/components/workflow/nodes/data-source-empty/hooks.ts

@@ -0,0 +1,47 @@
+import { useCallback } from 'react'
+import { useStoreApi } from 'reactflow'
+import { produce } from 'immer'
+import type { OnSelectBlock } from '@/app/components/workflow/types'
+import { generateNewNode } from '@/app/components/workflow/utils'
+import { useNodesMetaData } from '@/app/components/workflow/hooks'
+
+export const useReplaceDataSourceNode = (id: string) => {
+  const store = useStoreApi()
+  const { nodesMap: nodesMetaDataMap } = useNodesMetaData()
+
+  const handleReplaceNode = useCallback<OnSelectBlock>((
+    type,
+    toolDefaultValue,
+  ) => {
+    const {
+      getNodes,
+      setNodes,
+    } = store.getState()
+    const nodes = getNodes()
+    const emptyNodeIndex = nodes.findIndex(node => node.id === id)
+
+    if (emptyNodeIndex < 0) return
+    const {
+      defaultValue,
+    } = nodesMetaDataMap![type]
+    const emptyNode = nodes[emptyNodeIndex]
+    const { newNode } = generateNewNode({
+      data: {
+        ...(defaultValue as any),
+        ...(toolDefaultValue || {}),
+      },
+      position: {
+        x: emptyNode.position.x,
+        y: emptyNode.position.y,
+      },
+    })
+    const newNodes = produce(nodes, (draft) => {
+      draft[emptyNodeIndex] = newNode
+    })
+    setNodes(newNodes)
+  }, [])
+
+  return {
+    handleReplaceNode,
+  }
+}

web/app/components/workflow/nodes/data-source-empty/index.tsx

@@ -0,0 +1,70 @@
+import {
+  memo,
+  useCallback,
+} from 'react'
+import { useTranslation } from 'react-i18next'
+import type { NodeProps } from 'reactflow'
+import { RiAddLine } from '@remixicon/react'
+import cn from '@/utils/classnames'
+import Button from '@/app/components/base/button'
+import BlockSelector from '@/app/components/workflow/block-selector'
+import { useReplaceDataSourceNode } from './hooks'
+
+const DataSourceEmptyNode = ({ id }: NodeProps) => {
+  const { t } = useTranslation()
+  const { handleReplaceNode } = useReplaceDataSourceNode(id)
+
+  const renderTrigger = useCallback(() => {
+    return (
+      <Button
+        variant='primary'
+        className='w-full'
+      >
+        <RiAddLine className='mr-1 h-4 w-4' />
+        {t('workflow.nodes.dataSource.add')}
+      </Button>
+    )
+  }, [])
+
+  return (
+    <div
+      className={cn(
+        'relative flex rounded-2xl border',
+        'border-transparent',
+      )}
+    >
+      <div className='absolute inset-[-2px] top-[-22px] z-[-1] rounded-[18px] bg-node-data-source-bg p-0.5 backdrop-blur-[6px]'>
+        <div className='system-2xs-semibold-uppercase flex h-5 items-center px-2.5 text-text-tertiary'>
+          {t('workflow.blocks.datasource')}
+        </div>
+      </div>
+      <div
+        className={cn(
+          'group relative shadow-xs',
+          'rounded-[15px] border border-transparent',
+          'w-[240px] bg-workflow-block-bg',
+        )}
+      >
+        <div className={cn(
+          'flex items-center rounded-t-2xl p-3',
+        )}>
+          <BlockSelector
+            asChild
+            onSelect={handleReplaceNode}
+            trigger={renderTrigger}
+            noBlocks
+            noTools
+            popupClassName='w-[320px]'
+            placement='bottom-start'
+            offset={{
+              mainAxis: 4,
+              crossAxis: 0,
+            }}
+          />
+        </div>
+      </div>
+    </div>
+  )
+}
+
+export default memo(DataSourceEmptyNode)

web/app/components/workflow/nodes/data-source-empty/types.ts

@@ -0,0 +1,3 @@
+import type { CommonNodeType } from '@/app/components/workflow/types'
+
+export type DataSourceEmptyNodeType = CommonNodeType

web/app/components/workflow/types.ts

@@ -47,6 +47,7 @@ export enum BlockEnum {
   LoopStart = 'loop-start',
   LoopEnd = 'loop-end',
   DataSource = 'datasource',
+  DataSourceEmpty = 'datasource-empty',
   KnowledgeBase = 'knowledge-index',
 }
 
@@ -84,6 +85,7 @@ export type CommonNodeType<T = {}> = {
   _waitingRun?: boolean
   _retryIndex?: number
   _dataSourceStartToAdd?: boolean
+  noteBySystem?: boolean
   isInIteration?: boolean
   iteration_id?: string
   selected?: boolean

web/i18n/en-US/workflow.ts

@@ -913,6 +913,7 @@ const translation = {
     dataSource: {
       supportedFileFormats: 'Supported file formats',
       supportedFileFormatsPlaceholder: 'File extension, e.g. doc',
+      add: 'Add data source',
     },
     knowledgeBase: {
       chunkStructure: 'Chunk Structure',

web/i18n/zh-Hans/workflow.ts

@@ -927,6 +927,7 @@ const translation = {
     dataSource: {
       supportedFileFormats: '支持的文件格式',
       supportedFileFormatsPlaceholder: '文件格式，例如：doc',
+      add: '添加数据源',
     },
     knowledgeBase: {
       chunkStructure: '分段结构',