fix: remove notion page selector modal (#25247)

web/app/components/base/notion-page-selector/index.tsx

@@ -1,2 +1 @@
-export { default as NotionPageSelectorModal } from './notion-page-selector-modal'
 export { default as NotionPageSelector } from './base'

web/app/components/base/notion-page-selector/notion-page-selector-modal/index.module.css

@@ -1,28 +0,0 @@
-.modal {
-  width: 600px !important;
-  max-width: 600px !important;
-  padding: 24px 32px !important;
-}
-
-.operate {
-  padding: 0 8px;
-  min-width: 96px;
-  height: 36px;
-  line-height: 36px;
-  text-align: center;
-  background-color: #ffffff;
-  box-shadow: 0px 1px 2px rgba(16, 24, 40, 0.05);
-  border-radius: 8px;
-  border: 0.5px solid #eaecf0;
-  font-size: 14px;
-  font-weight: 500;
-  color: #667085;
-  cursor: pointer;
-}
-
-.operate-save {
-  margin-left: 8px;
-  border-color: #155eef;
-  background-color: #155eef;
-  color: #ffffff;
-}

web/app/components/base/notion-page-selector/notion-page-selector-modal/index.tsx

@@ -1,92 +0,0 @@
-import { useCallback, useMemo, useState } from 'react'
-import { useTranslation } from 'react-i18next'
-import { XMarkIcon } from '@heroicons/react/24/outline'
-import NotionPageSelector from '../base'
-import s from './index.module.css'
-import type { NotionPage } from '@/models/common'
-import cn from '@/utils/classnames'
-import Modal from '@/app/components/base/modal'
-import { noop } from 'lodash-es'
-import { useGetDefaultDataSourceListAuth } from '@/service/use-datasource'
-import NotionConnector from '../../notion-connector'
-import { useModalContextSelector } from '@/context/modal-context'
-
-type NotionPageSelectorModalProps = {
-  isShow: boolean
-  onClose: () => void
-  onSave: (selectedPages: NotionPage[]) => void
-  datasetId: string
-}
-const NotionPageSelectorModal = ({
-  isShow,
-  onClose,
-  onSave,
-  datasetId,
-}: NotionPageSelectorModalProps) => {
-  const { t } = useTranslation()
-  const setShowAccountSettingModal = useModalContextSelector(state => state.setShowAccountSettingModal)
-  const [selectedPages, setSelectedPages] = useState<NotionPage[]>([])
-
-  const { data: dataSourceList } = useGetDefaultDataSourceListAuth()
-
-  const handleClose = useCallback(() => {
-    onClose()
-  }, [onClose])
-
-  const handleSelectPage = useCallback((newSelectedPages: NotionPage[]) => {
-    setSelectedPages(newSelectedPages)
-  }, [])
-
-  const handleSave = useCallback(() => {
-    onSave(selectedPages)
-  }, [onSave])
-
-  const handleOpenSetting = useCallback(() => {
-    setShowAccountSettingModal({ payload: 'data-source' })
-  }, [setShowAccountSettingModal])
-
-  const authedDataSourceList = dataSourceList?.result || []
-
-  const isNotionAuthed = useMemo(() => {
-    if (!authedDataSourceList) return false
-    const notionSource = authedDataSourceList.find(item => item.provider === 'notion_datasource')
-    if (!notionSource) return false
-    return notionSource.credentials_list.length > 0
-  }, [authedDataSourceList])
-
-  const notionCredentialList = useMemo(() => {
-    return authedDataSourceList.find(item => item.provider === 'notion_datasource')?.credentials_list || []
-  }, [authedDataSourceList])
-
-  return (
-    <Modal
-      className={s.modal}
-      isShow={isShow}
-      onClose={noop}
-    >
-      <div className='mb-6 flex h-8 items-center justify-between'>
-        <div className='text-xl font-semibold text-text-primary'>{t('common.dataSource.notion.selector.addPages')}</div>
-        <div
-          className='-mr-2 flex h-8 w-8 cursor-pointer items-center justify-center'
-          onClick={handleClose}>
-          <XMarkIcon className='h-4 w-4' />
-        </div>
-      </div>
-      {!isNotionAuthed && <NotionConnector onSetting={handleOpenSetting} />}
-      {isNotionAuthed && (
-        <NotionPageSelector
-          credentialList={notionCredentialList}
-          onSelect={handleSelectPage}
-          canPreview={false}
-          datasetId={datasetId}
-        />
-      )}
-      <div className='mt-8 flex justify-end'>
-        <div className={s.operate} onClick={handleClose}>{t('common.operation.cancel')}</div>
-        <div className={cn(s.operate, s['operate-save'])} onClick={handleSave}>{t('common.operation.save')}</div>
-      </div>
-    </Modal>
-  )
-}
-
-export default NotionPageSelectorModal

web/app/components/base/progress-bar/index.tsx

@@ -1,20 +0,0 @@
-type ProgressBarProps = {
-  percent: number
-}
-const ProgressBar = ({
-  percent = 0,
-}: ProgressBarProps) => {
-  return (
-    <div className='flex items-center'>
-      <div className='mr-2 w-[100px] rounded-lg bg-gray-100'>
-        <div
-          className='h-1 rounded-lg bg-[#2970FF]'
-          style={{ width: `${percent}%` }}
-        />
-      </div>
-      <div className='text-xs font-medium text-gray-500'>{percent}%</div>
-    </div>
-  )
-}
-
-export default ProgressBar

web/app/components/datasets/documents/index.tsx

@@ -4,7 +4,6 @@ import React, { useCallback, useEffect, useMemo, useState } from 'react'
 import { useTranslation } from 'react-i18next'
 import { useRouter } from 'next/navigation'
 import { useDebounce, useDebounceFn } from 'ahooks'
-import { groupBy } from 'lodash-es'
 import { PlusIcon } from '@heroicons/react/24/solid'
 import { RiDraftLine, RiExternalLinkLine } from '@remixicon/react'
 import AutoDisabledDocument from '../common/document-status-with-action/auto-disabled-document'
@@ -13,13 +12,8 @@ import s from './style.module.css'
 import Loading from '@/app/components/base/loading'
 import Button from '@/app/components/base/button'
 import Input from '@/app/components/base/input'
-import { get } from '@/service/base'
-import { createDocument } from '@/service/datasets'
 import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
-import { NotionPageSelectorModal } from '@/app/components/base/notion-page-selector'
-import type { NotionPage } from '@/models/common'
-import type { CreateDocumentReq } from '@/models/datasets'
-import { DataSourceType, ProcessMode } from '@/models/datasets'
+import { DataSourceType } from '@/models/datasets'
 import IndexFailed from '@/app/components/datasets/common/document-status-with-action/index-failed'
 import { useProviderContext } from '@/context/provider-context'
 import cn from '@/utils/classnames'
@@ -31,7 +25,6 @@ import useEditDocumentMetadata from '../metadata/hooks/use-edit-dataset-metadata
 import DatasetMetadataDrawer from '../metadata/metadata-dataset/dataset-metadata-drawer'
 import StatusWithAction from '../common/document-status-with-action/status-with-action'
 import { useDocLink } from '@/context/i18n'
-import { useFetchDefaultProcessRule } from '@/service/knowledge/use-create-dataset'
 import { SimpleSelect } from '../../base/select'
 import StatusItem from './detail/completed/status-item'
 import type { Item } from '@/app/components/base/select'
@@ -86,8 +79,6 @@ type IDocumentsProps = {
   datasetId: string
 }
 
-export const fetcher = (url: string) => get(url, {}, {})
-
 const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
   const { t } = useTranslation()
   const docLink = useDocLink()
@@ -105,7 +96,6 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
 
   const router = useRouter()
   const dataset = useDatasetDetailContextWithSelector(s => s.dataset)
-  const [notionPageSelectorModalVisible, setNotionPageSelectorModalVisible] = useState(false)
   const [timerCanRun, setTimerCanRun] = useState(true)
   const isDataSourceNotion = dataset?.data_source_type === DataSourceType.NOTION
   const isDataSourceWeb = dataset?.data_source_type === DataSourceType.WEB
@@ -164,14 +154,14 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
     }
   }, [debouncedSearchValue, query.keyword, updateQuery])
 
-  const { data: documentsRes, isFetching: isListLoading } = useDocumentList({
+  const { data: documentsRes, isLoading: isListLoading } = useDocumentList({
     datasetId,
     query: {
       page: currPage + 1,
       limit,
       keyword: debouncedSearchValue,
     },
-    refetchInterval: (isDataSourceNotion && timerCanRun) ? 2500 : 0,
+    refetchInterval: timerCanRun ? 2500 : 0,
   })
 
   const invalidDocumentList = useInvalidDocumentList(datasetId)
@@ -197,10 +187,10 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
     }, 5000)
   }, [])
 
-  const documentsWithProgress = useMemo(() => {
+  useEffect(() => {
     let completedNum = 0
     let percent = 0
-    const documentsData = documentsRes?.data?.map((documentItem) => {
+    documentsRes?.data?.forEach((documentItem) => {
       const { indexing_status, completed_segments, total_segments } = documentItem
       const isEmbedded = indexing_status === 'completed' || indexing_status === 'paused' || indexing_status === 'error'
 
@@ -221,77 +211,20 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
         percent,
       }
     })
-    if (completedNum === documentsRes?.data?.length)
-      setTimerCanRun(false)
-    return {
-      ...documentsRes,
-      data: documentsData,
-    }
+    setTimerCanRun(completedNum !== documentsRes?.data?.length)
   }, [documentsRes])
   const total = documentsRes?.total || 0
 
   const routeToDocCreate = () => {
-    // if dataset is create from pipeline, redirect to create from pipeline page
+    // if dataset is created from pipeline, go to create from pipeline page
     if (dataset?.runtime_mode === 'rag_pipeline') {
       router.push(`/datasets/${datasetId}/documents/create-from-pipeline`)
       return
     }
-    if (isDataSourceNotion) {
-      setNotionPageSelectorModalVisible(true)
-      return
-    }
     router.push(`/datasets/${datasetId}/documents/create`)
   }
 
-  const fetchDefaultProcessRuleMutation = useFetchDefaultProcessRule()
-
-  const handleSaveNotionPageSelected = async (selectedPages: NotionPage[]) => {
-    const workspacesMap = groupBy(selectedPages, 'workspace_id')
-    const workspaces = Object.keys(workspacesMap).map((workspaceId) => {
-      return {
-        workspaceId,
-        pages: workspacesMap[workspaceId],
-      }
-    })
-    const { rules } = await fetchDefaultProcessRuleMutation.mutateAsync('/datasets/process-rule')
-    const params = {
-      data_source: {
-        type: dataset?.data_source_type,
-        info_list: {
-          data_source_type: dataset?.data_source_type,
-          notion_info_list: workspaces.map((workspace) => {
-            return {
-              workspace_id: workspace.workspaceId,
-              pages: workspace.pages.map((page) => {
-                const { page_id, page_name, page_icon, type } = page
-                return {
-                  page_id,
-                  page_name,
-                  page_icon,
-                  type,
-                }
-              }),
-            }
-          }),
-        },
-      },
-      indexing_technique: dataset?.indexing_technique,
-      process_rule: {
-        rules,
-        mode: ProcessMode.general,
-      },
-    } as CreateDocumentReq
-
-    await createDocument({
-      datasetId,
-      body: params,
-    })
-    invalidDocumentList()
-    setTimerCanRun(true)
-    setNotionPageSelectorModalVisible(false)
-  }
-
-  const documentsList = isDataSourceNotion ? documentsWithProgress?.data : documentsRes?.data
+  const documentsList = documentsRes?.data
   const [selectedIds, setSelectedIds] = useState<string[]>([])
 
   // Clear selection when search changes to avoid confusion
@@ -402,32 +335,33 @@ const Documents: FC<IDocumentsProps> = ({ datasetId }) => {
           ? <Loading type='app' />
           // eslint-disable-next-line sonarjs/no-nested-conditional
           : total > 0
-            ? <List
-              embeddingAvailable={embeddingAvailable}
-              documents={documentsList || []}
-              datasetId={datasetId}
-              onUpdate={handleUpdate}
-              selectedIds={selectedIds}
-              onSelectedIdChange={setSelectedIds}
-              statusFilter={statusFilter}
-              onStatusFilterChange={setStatusFilter}
-              pagination={{
-                total,
-                limit,
-                onLimitChange: handleLimitChange,
-                current: currPage,
-                onChange: handlePageChange,
-              }}
-              onManageMetadata={showEditMetadataModal}
-            />
-            : <EmptyElement canAdd={embeddingAvailable} onClick={routeToDocCreate} type={isDataSourceNotion ? 'sync' : 'upload'} />
+            ? (
+              <List
+                embeddingAvailable={embeddingAvailable}
+                documents={documentsList || []}
+                datasetId={datasetId}
+                onUpdate={handleUpdate}
+                selectedIds={selectedIds}
+                onSelectedIdChange={setSelectedIds}
+                statusFilter={statusFilter}
+                pagination={{
+                  total,
+                  limit,
+                  onLimitChange: handleLimitChange,
+                  current: currPage,
+                  onChange: handlePageChange,
+                }}
+                onManageMetadata={showEditMetadataModal}
+              />
+            )
+            : (
+              <EmptyElement
+                canAdd={embeddingAvailable}
+                onClick={routeToDocCreate}
+                type={isDataSourceNotion ? 'sync' : 'upload'}
+              />
+            )
         }
-        <NotionPageSelectorModal
-          isShow={notionPageSelectorModalVisible}
-          onClose={() => setNotionPageSelectorModalVisible(false)}
-          onSave={handleSaveNotionPageSelected}
-          datasetId={dataset?.id || ''}
-        />
       </div>
     </div>
   )

web/app/components/datasets/documents/list.tsx

@@ -1,6 +1,6 @@
 'use client'
 import type { FC } from 'react'
-import React, { useCallback, useEffect, useMemo, useState } from 'react'
+import React, { useCallback, useMemo, useState } from 'react'
 import { useBoolean } from 'ahooks'
 import { ArrowDownIcon } from '@heroicons/react/24/outline'
 import { pick, uniq } from 'lodash-es'
@@ -22,7 +22,6 @@ import type { Item } from '@/app/components/base/select'
 import { asyncRunSafe } from '@/utils'
 import { formatNumber } from '@/utils/format'
 import NotionIcon from '@/app/components/base/notion-icon'
-import ProgressBar from '@/app/components/base/progress-bar'
 import type { LegacyDataSourceInfo, LocalFileInfo, OnlineDocumentInfo, OnlineDriveInfo } from '@/models/datasets'
 import { ChunkingMode, DataSourceType, DocumentActionType, type SimpleDocumentDetail } from '@/models/datasets'
 import type { CommonResponse } from '@/models/common'
@@ -68,7 +67,6 @@ type IDocumentListProps = {
   onUpdate: () => void
   onManageMetadata: () => void
   statusFilter: Item
-  onStatusFilterChange: (filter: string) => void
 }
 
 /**
@@ -92,7 +90,6 @@ const DocumentList: FC<IDocumentListProps> = ({
   const chunkingMode = datasetConfig?.doc_form
   const isGeneralMode = chunkingMode !== ChunkingMode.parentChild
   const isQAMode = chunkingMode === ChunkingMode.qa
-  const [localDocs, setLocalDocs] = useState<LocalDoc[]>(documents)
   const [sortField, setSortField] = useState<'name' | 'word_count' | 'hit_count' | 'created_at' | null>('created_at')
   const [sortOrder, setSortOrder] = useState<'asc' | 'desc'>('desc')
 
@@ -109,7 +106,7 @@ const DocumentList: FC<IDocumentListProps> = ({
     onUpdate,
   })
 
-  useEffect(() => {
+  const localDocs = useMemo(() => {
     let filteredDocs = documents
 
     if (statusFilter.value !== 'all') {
@@ -120,10 +117,8 @@ const DocumentList: FC<IDocumentListProps> = ({
       )
     }
 
-    if (!sortField) {
-      setLocalDocs(filteredDocs)
-      return
-    }
+    if (!sortField)
+      return filteredDocs
 
     const sortedDocs = [...filteredDocs].sort((a, b) => {
       let aValue: any
@@ -160,7 +155,7 @@ const DocumentList: FC<IDocumentListProps> = ({
       }
     })
 
-    setLocalDocs(sortedDocs)
+    return sortedDocs
   }, [documents, sortField, sortOrder, statusFilter])
 
   const handleSort = (field: 'name' | 'word_count' | 'hit_count' | 'created_at') => {
@@ -261,9 +256,9 @@ const DocumentList: FC<IDocumentListProps> = ({
     return parts[parts.length - 1].toLowerCase()
   }, [])
 
-  const isCreateFromRAGPipeline = useMemo(() => {
-    return datasetConfig?.runtime_mode === 'rag_pipeline'
-  }, [datasetConfig?.runtime_mode])
+  const isCreateFromRAGPipeline = useCallback((createdFrom: string) => {
+    return createdFrom === 'rag_pipeline'
+  }, [])
 
   /**
    * Calculate the data source type
@@ -271,25 +266,17 @@ const DocumentList: FC<IDocumentListProps> = ({
    * DatasourceType: localFile, onlineDocument, websiteCrawl, onlineDrive (new)
    */
   const isLocalFile = useCallback((dataSourceType: DataSourceType | DatasourceType) => {
-    if (isCreateFromRAGPipeline)
-      return dataSourceType === DatasourceType.localFile
-    return dataSourceType === DataSourceType.FILE
-  }, [isCreateFromRAGPipeline])
+    return dataSourceType === DatasourceType.localFile || dataSourceType === DataSourceType.FILE
+  }, [])
   const isOnlineDocument = useCallback((dataSourceType: DataSourceType | DatasourceType) => {
-    if (isCreateFromRAGPipeline)
-      return dataSourceType === DatasourceType.onlineDocument
-    return dataSourceType === DataSourceType.NOTION
-  }, [isCreateFromRAGPipeline])
+    return dataSourceType === DatasourceType.onlineDocument || dataSourceType === DataSourceType.NOTION
+  }, [])
   const isWebsiteCrawl = useCallback((dataSourceType: DataSourceType | DatasourceType) => {
-    if (isCreateFromRAGPipeline)
-      return dataSourceType === DatasourceType.websiteCrawl
-    return dataSourceType === DataSourceType.WEB
-  }, [isCreateFromRAGPipeline])
+    return dataSourceType === DatasourceType.websiteCrawl || dataSourceType === DataSourceType.WEB
+  }, [])
   const isOnlineDrive = useCallback((dataSourceType: DataSourceType | DatasourceType) => {
-    if (isCreateFromRAGPipeline)
-      return dataSourceType === DatasourceType.onlineDrive
-    return false
-  }, [isCreateFromRAGPipeline])
+    return dataSourceType === DatasourceType.onlineDrive
+  }, [])
 
   return (
     <div className='relative flex h-full w-full flex-col'>
@@ -361,7 +348,7 @@ const DocumentList: FC<IDocumentListProps> = ({
                           className='mr-1.5'
                           type='page'
                           src={
-                            isCreateFromRAGPipeline
+                            isCreateFromRAGPipeline(doc.created_from)
                               ? (doc.data_source_info as OnlineDocumentInfo).page.page_icon
                               : (doc.data_source_info as LegacyDataSourceInfo).notion_page_icon
                           }
@@ -371,7 +358,7 @@ const DocumentList: FC<IDocumentListProps> = ({
                         <FileTypeIcon
                           type={
                             extensionToFileType(
-                              isCreateFromRAGPipeline
+                              isCreateFromRAGPipeline(doc.created_from)
                                 ? (doc?.data_source_info as LocalFileInfo)?.extension
                                 : ((doc?.data_source_info as LegacyDataSourceInfo)?.upload_file?.extension ?? fileType),
                             )
@@ -427,12 +414,7 @@ const DocumentList: FC<IDocumentListProps> = ({
                   {formatTime(doc.created_at, t('datasetHitTesting.dateTimeFormat') as string)}
                 </td>
                 <td>
-                  {
-                    (['indexing', 'splitting', 'parsing', 'cleaning'].includes(doc.indexing_status)
-                      && isOnlineDocument(doc.data_source_type))
-                      ? <ProgressBar percent={doc.percent || 0} />
-                      : <StatusItem status={doc.display_status} />
-                  }
+                  <StatusItem status={doc.display_status} />
                 </td>
                 <td>
                   <Operations