refactor(icons): remove unused Globe01 icon and related files

web/app/components/base/icons/assets/vender/line/mapsAndTravel/globe-01.svg

@@ -1,10 +0,0 @@
-<svg width="16" height="16" viewBox="0 0 16 16" fill="none" xmlns="http://www.w3.org/2000/svg">
-<g id="globe-01" clip-path="url(#clip0_8902_1914)">
-<path id="Icon" d="M1.33325 7.99998H14.6666M1.33325 7.99998C1.33325 11.6819 4.31802 14.6666 7.99992 14.6666M1.33325 7.99998C1.33325 4.31808 4.31802 1.33331 7.99992 1.33331M14.6666 7.99998C14.6666 11.6819 11.6818 14.6666 7.99992 14.6666M14.6666 7.99998C14.6666 4.31808 11.6818 1.33331 7.99992 1.33331M7.99992 1.33331C9.66744 3.15888 10.6151 5.528 10.6666 7.99998C10.6151 10.472 9.66744 12.8411 7.99992 14.6666M7.99992 1.33331C6.3324 3.15888 5.38475 5.528 5.33325 7.99998C5.38475 10.472 6.3324 12.8411 7.99992 14.6666" stroke="#344054" stroke-width="1.25" stroke-linecap="round" stroke-linejoin="round"/>
-</g>
-<defs>
-<clipPath id="clip0_8902_1914">
-<rect width="16" height="16" fill="white"/>
-</clipPath>
-</defs>
-</svg>

web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.json

@@ -1,66 +0,0 @@
-{
-	"icon": {
-		"type": "element",
-		"isRootNode": true,
-		"name": "svg",
-		"attributes": {
-			"width": "16",
-			"height": "16",
-			"viewBox": "0 0 16 16",
-			"fill": "none",
-			"xmlns": "http://www.w3.org/2000/svg"
-		},
-		"children": [
-			{
-				"type": "element",
-				"name": "g",
-				"attributes": {
-					"id": "globe-01",
-					"clip-path": "url(#clip0_8902_1914)"
-				},
-				"children": [
-					{
-						"type": "element",
-						"name": "path",
-						"attributes": {
-							"id": "Icon",
-							"d": "M1.33325 7.99998H14.6666M1.33325 7.99998C1.33325 11.6819 4.31802 14.6666 7.99992 14.6666M1.33325 7.99998C1.33325 4.31808 4.31802 1.33331 7.99992 1.33331M14.6666 7.99998C14.6666 11.6819 11.6818 14.6666 7.99992 14.6666M14.6666 7.99998C14.6666 4.31808 11.6818 1.33331 7.99992 1.33331M7.99992 1.33331C9.66744 3.15888 10.6151 5.528 10.6666 7.99998C10.6151 10.472 9.66744 12.8411 7.99992 14.6666M7.99992 1.33331C6.3324 3.15888 5.38475 5.528 5.33325 7.99998C5.38475 10.472 6.3324 12.8411 7.99992 14.6666",
-							"stroke": "currentColor",
-							"stroke-width": "1.25",
-							"stroke-linecap": "round",
-							"stroke-linejoin": "round"
-						},
-						"children": []
-					}
-				]
-			},
-			{
-				"type": "element",
-				"name": "defs",
-				"attributes": {},
-				"children": [
-					{
-						"type": "element",
-						"name": "clipPath",
-						"attributes": {
-							"id": "clip0_8902_1914"
-						},
-						"children": [
-							{
-								"type": "element",
-								"name": "rect",
-								"attributes": {
-									"width": "16",
-									"height": "16",
-									"fill": "white"
-								},
-								"children": []
-							}
-						]
-					}
-				]
-			}
-		]
-	},
-	"name": "Globe01"
-}

web/app/components/base/icons/src/vender/line/mapsAndTravel/Globe01.tsx

@@ -1,20 +0,0 @@
-// GENERATE BY script
-// DON NOT EDIT IT MANUALLY
-
-import * as React from 'react'
-import data from './Globe01.json'
-import IconBase from '@/app/components/base/icons/IconBase'
-import type { IconData } from '@/app/components/base/icons/IconBase'
-
-const Icon = (
-  {
-    ref,
-    ...props
-  }: React.SVGProps<SVGSVGElement> & {
-    ref?: React.RefObject<React.MutableRefObject<HTMLOrSVGElement>>;
-  },
-) => <IconBase {...props} ref={ref} data={data as IconData} />
-
-Icon.displayName = 'Globe01'
-
-export default Icon

web/app/components/base/icons/src/vender/line/mapsAndTravel/index.ts

@@ -1,2 +1 @@
-export { default as Globe01 } from './Globe01'
 export { default as Route } from './Route'

web/app/components/datasets/documents/create-from-pipeline/index.tsx

@@ -209,10 +209,11 @@ const CreateFormPipeline = () => {
     }
     if (datasourceType === DatasourceType.onlineDrive) {
       const { bucket } = dataSourceStore.getState()
-      const { id, type } = previewOnlineDriveFileRef.current!
+      const { id, type, name } = previewOnlineDriveFileRef.current!
       datasourceInfoList.push({
         bucket,
         id,
+        name,
         type,
         credential_id: currentCredentialId,
       })
@@ -292,6 +293,7 @@ const CreateFormPipeline = () => {
         datasourceInfoList.push({
           bucket,
           id: file?.id,
+          name: file?.name,
           type: file?.type,
           credential_id: currentCredentialId,
         })

web/app/components/datasets/documents/list.tsx

@@ -6,10 +6,10 @@ import { ArrowDownIcon } from '@heroicons/react/24/outline'
 import { pick, uniq } from 'lodash-es'
 import {
   RiEditLine,
+  RiGlobalLine,
 } from '@remixicon/react'
 import { useRouter } from 'next/navigation'
 import { useTranslation } from 'react-i18next'
-import { Globe01 } from '../../base/icons/src/vender/line/mapsAndTravel'
 import ChunkingModeLabel from '../common/chunking-mode-label'
 import FileTypeIcon from '../../base/file-uploader/file-type-icon'
 import s from './style.module.css'
@@ -23,6 +23,7 @@ import { asyncRunSafe } from '@/utils'
 import { formatNumber } from '@/utils/format'
 import NotionIcon from '@/app/components/base/notion-icon'
 import ProgressBar from '@/app/components/base/progress-bar'
+import type { LegacyDataSourceInfo, LocalFileInfo, OnlineDocumentInfo, OnlineDriveInfo } from '@/models/datasets'
 import { ChunkingMode, DataSourceType, DocumentActionType, type SimpleDocumentDetail } from '@/models/datasets'
 import type { CommonResponse } from '@/models/common'
 import useTimestamp from '@/hooks/use-timestamp'
@@ -36,6 +37,7 @@ import useBatchEditDocumentMetadata from '../metadata/hooks/use-batch-edit-docum
 import EditMetadataBatchModal from '@/app/components/datasets/metadata/edit-metadata-batch/modal'
 import StatusItem from './status-item'
 import Operations from './operations'
+import { DatasourceType } from '@/models/pipeline'
 
 export const renderTdValue = (value: string | number | null, isEmptyStyle = false) => {
   return (
@@ -249,6 +251,46 @@ const DocumentList: FC<IDocumentListProps> = ({
     }
   }
 
+  const getFileExtension = useCallback((fileName: string): string => {
+    if (!fileName)
+      return ''
+    const parts = fileName.split('.')
+    if (parts.length <= 1 || (parts[0] === '' && parts.length === 2))
+      return ''
+
+    return parts[parts.length - 1].toLowerCase()
+  }, [])
+
+  const isCreateFromRAGPipeline = useCallback((createFrom: string) => {
+    return createFrom === 'rag-pipeline'
+  }, [])
+
+  /**
+   * Calculate the data source type
+   * DataSourceType: FILE, NOTION, WEB (legacy)
+   * DatasourceType: localFile, onlineDocument, websiteCrawl, onlineDrive (new)
+   */
+  const isLocalFile = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => {
+    if (createFrom === 'rag-pipeline')
+      return dataSourceType === DatasourceType.localFile
+    return dataSourceType === DataSourceType.FILE
+  }, [])
+  const isOnlineDocument = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => {
+    if (createFrom === 'rag-pipeline')
+      return dataSourceType === DatasourceType.onlineDocument
+    return dataSourceType === DataSourceType.NOTION
+  }, [])
+  const isWebsiteCrawl = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => {
+    if (createFrom === 'rag-pipeline')
+      return dataSourceType === DatasourceType.websiteCrawl
+    return dataSourceType === DataSourceType.WEB
+  }, [])
+  const isOnlineDrive = useCallback((dataSourceType: DataSourceType | DatasourceType, createFrom: string) => {
+    if (createFrom === 'rag-pipeline')
+      return dataSourceType === DatasourceType.onlineDrive
+    return false
+  }, [])
+
   return (
     <div className='relative flex h-full w-full flex-col'>
       <div className='relative grow overflow-x-auto'>
@@ -287,7 +329,8 @@ const DocumentList: FC<IDocumentListProps> = ({
           </thead>
           <tbody className="text-text-secondary">
             {localDocs.map((doc, index) => {
-              const isFile = doc.data_source_type === DataSourceType.FILE
+              const isFile = isLocalFile(doc.data_source_type, doc.created_from)
+              const createFromRAGPipeline = isCreateFromRAGPipeline(doc.created_from)
               const fileType = isFile ? doc.data_source_detail_dict?.upload_file?.extension : ''
               return <tr
                 key={doc.id}
@@ -313,10 +356,43 @@ const DocumentList: FC<IDocumentListProps> = ({
                 </td>
                 <td>
                   <div className={'group mr-6 flex max-w-[460px] items-center hover:mr-0'}>
-                    <div className='shrink-0'>
-                      {doc?.data_source_type === DataSourceType.NOTION && <NotionIcon className='mr-1.5 mt-[-3px] inline-flex align-middle' type='page' src={doc.data_source_info.notion_page_icon} />}
-                      {doc?.data_source_type === DataSourceType.FILE && <FileTypeIcon type={extensionToFileType(doc?.data_source_info?.upload_file?.extension ?? fileType)} className='mr-1.5' />}
-                      {doc?.data_source_type === DataSourceType.WEB && <Globe01 className='mr-1.5 mt-[-3px] inline-flex align-middle' />}
+                    <div className='flex shrink-0 items-center'>
+                      {isOnlineDocument(doc.data_source_type, doc.created_from) && (
+                        <NotionIcon
+                          className='mr-1.5'
+                          type='page'
+                          src={
+                            createFromRAGPipeline
+                              ? (doc.data_source_info as OnlineDocumentInfo).page.page_icon
+                              : (doc.data_source_info as LegacyDataSourceInfo).notion_page_icon
+                          }
+                        />
+                      )}
+                      {isLocalFile(doc.data_source_type, doc.created_from) && (
+                        <FileTypeIcon
+                          type={
+                            extensionToFileType(
+                              createFromRAGPipeline
+                                ? (doc?.data_source_info as LocalFileInfo)?.extension
+                                : ((doc?.data_source_info as LegacyDataSourceInfo)?.upload_file?.extension ?? fileType),
+                            )
+                          }
+                          className='mr-1.5'
+                        />
+                      )}
+                      {isOnlineDrive(doc.data_source_type, doc.created_from) && (
+                        <FileTypeIcon
+                          type={
+                            extensionToFileType(
+                              getFileExtension((doc?.data_source_info as unknown as OnlineDriveInfo)?.name),
+                            )
+                          }
+                          className='mr-1.5'
+                        />
+                      )}
+                      {isWebsiteCrawl(doc.data_source_type, doc.created_from) && (
+                        <RiGlobalLine className='mr-1.5 size-4' />
+                      )}
                     </div>
                     <Tooltip
                       popupContent={doc.name}
@@ -353,7 +429,8 @@ const DocumentList: FC<IDocumentListProps> = ({
                 </td>
                 <td>
                   {
-                    (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status) && doc?.data_source_type === DataSourceType.NOTION)
+                    (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status)
+                      && isOnlineDocument(doc.data_source_type, doc.created_from))
                       ? <ProgressBar percent={doc.percent || 0} />
                       : <StatusItem status={doc.display_status} />
                   }

web/app/components/rag-pipeline/components/panel/test-run/preparation/index.tsx

@@ -109,6 +109,7 @@ const Preparation = () => {
       datasourceInfoList.push({
         bucket,
         id: file?.id,
+        name: file?.name,
         type: file?.type,
         credential_id: credentialId,
       })

web/models/datasets.ts

@@ -1,5 +1,5 @@
 import type { DataSourceNotionPage, DataSourceProvider } from './common'
-import type { AppIconType, AppMode, RetrievalConfig } from '@/types/app'
+import type { AppIconType, AppMode, RetrievalConfig, TransferMethod } from '@/types/app'
 import type { Tag } from '@/app/components/base/tag-management/constant'
 import type { IndexingType } from '@/app/components/datasets/create/step-two'
 import type { MetadataFilteringVariableType } from '@/app/components/workflow/nodes/knowledge-retrieval/types'
@@ -308,7 +308,7 @@ export const DisplayStatusList = [
 
 export type DocumentDisplayStatus = typeof DisplayStatusList[number]
 
-export type DataSourceInfo = {
+export type LegacyDataSourceInfo = {
   upload_file: {
     id: string
     name: string
@@ -327,6 +327,47 @@ export type DataSourceInfo = {
   credential_id?: string
 }
 
+export type LocalFileInfo = {
+  extension: string
+  mime_type: string
+  name: string
+  related_id: string
+  size: number
+  transfer_method: TransferMethod
+  url: string
+}
+
+export type WebsiteCrawlInfo = {
+  content: string
+  credential_id: string
+  description: string
+  source_url: string
+  title: string
+}
+
+export type OnlineDocumentInfo = {
+  credential_id: string
+  workspace_id: string
+  page: {
+    last_edited_time: string
+    page_icon: DataSourceNotionPage['page_icon']
+    page_id: string
+    page_name: string
+    parent_id: string
+    type: string
+  },
+}
+
+export type OnlineDriveInfo = {
+  bucket: string
+  credential_id: string
+  id: string
+  name: string
+  type: 'file' | 'folder'
+}
+
+export type DataSourceInfo = LegacyDataSourceInfo | LocalFileInfo | OnlineDocumentInfo | WebsiteCrawlInfo
+
 export type InitialDocumentDetail = {
   id: string
   batch: string
@@ -350,7 +391,6 @@ export type InitialDocumentDetail = {
 export type SimpleDocumentDetail = InitialDocumentDetail & {
   enabled: boolean
   word_count: number
-  is_qa: boolean // TODO waiting for backend to add this field
   error?: string | null
   archived: boolean
   updated_at: number
@@ -363,6 +403,7 @@ export type SimpleDocumentDetail = InitialDocumentDetail & {
     }
   }
   doc_metadata?: MetadataItemWithValue[]
+  created_from: string
 }
 
 export type DocumentListResponse = {