瀏覽代碼

refactor: refactor preview components

tags/2.0.0-beta.1
twwu 5 月之前
父節點
當前提交
faf6b9ea03
共有 25 個檔案被更改,包括 727 行新增89 行删除
  1. 2
    2
      web/app/components/base/form/form-scenarios/demo/index.tsx
  2. 62
    0
      web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx
  3. 41
    0
      web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx
  4. 44
    0
      web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx
  5. 82
    0
      web/app/components/datasets/documents/create-from-pipeline/hooks.ts
  6. 121
    81
      web/app/components/datasets/documents/create-from-pipeline/index.tsx
  7. 78
    0
      web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx
  8. 52
    0
      web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx
  9. 65
    0
      web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx
  10. 48
    0
      web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx
  11. 3
    0
      web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx
  12. 4
    1
      web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx
  13. 9
    2
      web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx
  14. 18
    1
      web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx
  15. 14
    2
      web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx
  16. 3
    0
      web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx
  17. 3
    0
      web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx
  18. 3
    0
      web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx
  19. 3
    0
      web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx
  20. 4
    0
      web/i18n/en-US/dataset-pipeline.ts
  21. 4
    0
      web/i18n/zh-Hans/dataset-pipeline.ts
  22. 19
    0
      web/service/knowledge/use-dataset.ts
  23. 12
    0
      web/service/use-common.ts
  24. 1
    0
      web/service/use-pipeline.ts
  25. 32
    0
      web/utils/format.ts

+ 2
- 2
web/app/components/base/form/form-scenarios/demo/index.tsx 查看文件

}, },
}) })


const name = useStore(form.store, state => state.values.name)
const name = useStore(form.store, state => state.values.name)


return ( return (
<form <form
) )
} }
<form.AppForm> <form.AppForm>
<form.SubmitButton>Submit</form.SubmitButton>
<form.Actions />
</form.AppForm> </form.AppForm>
</form> </form>
) )

+ 62
- 0
web/app/components/datasets/documents/create-from-pipeline/data-source-options/index.tsx 查看文件

import { useCallback, useEffect } from 'react'
import { useDatasourceOptions } from '../hooks'
import OptionCard from './option-card'
import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge'
import { Notion } from '@/app/components/base/icons/src/public/common'
import { Jina } from '@/app/components/base/icons/src/public/llm'
import { DataSourceType } from '@/models/datasets'
import { DataSourceProvider } from '@/models/common'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import type { Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'

type DataSourceOptionsProps = {
pipelineNodes: Node<DataSourceNodeType>[]
datasourceNodeId: string
onSelect: (option: Datasource) => void
}

const DATA_SOURCE_ICONS = {
[DataSourceType.FILE]: File as React.FC<React.SVGProps<SVGSVGElement>>,
[DataSourceType.NOTION]: Notion as React.FC<React.SVGProps<SVGSVGElement>>,
[DataSourceProvider.fireCrawl]: '🔥',
[DataSourceProvider.jinaReader]: Jina as React.FC<React.SVGProps<SVGSVGElement>>,
[DataSourceProvider.waterCrawl]: Watercrawl as React.FC<React.SVGProps<SVGSVGElement>>,
}

const DataSourceOptions = ({
pipelineNodes,
datasourceNodeId,
onSelect,
}: DataSourceOptionsProps) => {
const { datasources, options } = useDatasourceOptions(pipelineNodes)

const handelSelect = useCallback((value: string) => {
const selectedOption = datasources.find(option => option.nodeId === value)
if (!selectedOption)
return
onSelect(selectedOption)
}, [datasources, onSelect])

useEffect(() => {
if (options.length > 0)
handelSelect(options[0].value)
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [])

return (
<div className='grid w-full grid-cols-4 gap-1'>
{options.map(option => (
<OptionCard
key={option.value}
label={option.label}
selected={datasourceNodeId === option.value}
Icon={DATA_SOURCE_ICONS[option.type as keyof typeof DATA_SOURCE_ICONS]}
onClick={handelSelect.bind(null, option.value)}
/>
))}
</div>
)
}

export default DataSourceOptions

+ 41
- 0
web/app/components/datasets/documents/create-from-pipeline/data-source-options/option-card.tsx 查看文件

import React from 'react'
import cn from '@/utils/classnames'

type OptionCardProps = {
label: string
Icon: React.FC<React.SVGProps<SVGSVGElement>> | string
selected: boolean
onClick?: () => void
}

const OptionCard = ({
label,
Icon,
selected,
onClick,
}: OptionCardProps) => {
return (
<div
className={cn(
'flex items-center gap-2 rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg p-3 shadow-shadow-shadow-3',
selected
? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg shadow-xs ring-[0.5px] ring-inset ring-components-option-card-option-selected-border'
: 'hover:bg-components-option-card-bg-hover hover:border-components-option-card-option-border-hover hover:shadow-xs',
)}
onClick={onClick}
>
<div className='flex size-8 items-center justify-center rounded-lg border-[0.5px] border-components-panel-border bg-background-default-dodge p-1.5'>
{
typeof Icon === 'string'
? <div className='text-[18px] leading-[18px]'>{Icon}</div>
: <Icon className='size-5' />
}
</div>
<div className={cn('system-sm-medium text-text-secondary', selected && 'text-primary')}>
{label}
</div>
</div>
)
}

export default React.memo(OptionCard)

+ 44
- 0
web/app/components/datasets/documents/create-from-pipeline/data-source/actions.tsx 查看文件

import React from 'react'
import Button from '@/app/components/base/button'
import { useTranslation } from 'react-i18next'
import { useParams } from 'next/navigation'
import { RiArrowRightLine } from '@remixicon/react'

type ActionsProps = {
disabled?: boolean
handleNextStep: () => void
}

const Actions = ({
disabled,
handleNextStep,
}: ActionsProps) => {
const { t } = useTranslation()
const { datasetId } = useParams()

return (
<div className='flex justify-end gap-x-2'>
<a
href={`/datasets/${datasetId}/documents`}
>
<Button
variant='ghost'
className='px-3 py-2'
>
{t('common.operation.cancel')}
</Button>
</a>
<Button
disabled={disabled}
variant='primary'
onClick={handleNextStep}
className='gap-x-0.5'
>
<span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span>
<RiArrowRightLine className='size-4' />
</Button>
</div>
)
}

export default React.memo(Actions)

+ 82
- 0
web/app/components/datasets/documents/create-from-pipeline/hooks.ts 查看文件

import { useTranslation } from 'react-i18next' import { useTranslation } from 'react-i18next'
import { AddDocumentsStep } from './types' import { AddDocumentsStep } from './types'
import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import { useMemo } from 'react'
import { BlockEnum, type Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import { DataSourceType } from '@/models/datasets'
import { DataSourceProvider } from '@/models/common'


export const useAddDocumentsSteps = () => { export const useAddDocumentsSteps = () => {
const { t } = useTranslation() const { t } = useTranslation()
] ]
return steps return steps
} }

export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => {
const { t } = useTranslation()
const datasources: Datasource[] = useMemo(() => {
const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource)
return datasourceNodes.map((node) => {
let type: DataSourceType | DataSourceProvider = DataSourceType.FILE
switch (node.data.tool_name) {
case 'file_upload':
type = DataSourceType.FILE
break
case 'search_notion':
type = DataSourceType.NOTION
break
case 'firecrawl':
type = DataSourceProvider.fireCrawl
break
case 'jina_reader':
type = DataSourceProvider.jinaReader
break
case 'water_crawl':
type = DataSourceProvider.waterCrawl
break
}
return {
nodeId: node.id,
type,
variables: node.data.variables,
}
})
}, [pipelineNodes])

const options = useMemo(() => {
const options: DataSourceOption[] = []
datasources.forEach((source) => {
if (source.type === DataSourceType.FILE) {
options.push({
label: t('datasetPipeline.testRun.dataSource.localFiles'),
value: source.nodeId,
type: DataSourceType.FILE,
})
}
if (source.type === DataSourceType.NOTION) {
options.push({
label: 'Notion',
value: source.nodeId,
type: DataSourceType.NOTION,
})
}
if (source.type === DataSourceProvider.fireCrawl) {
options.push({
label: 'Firecrawl',
value: source.nodeId,
type: DataSourceProvider.fireCrawl,
})
}
if (source.type === DataSourceProvider.jinaReader) {
options.push({
label: 'Jina Reader',
value: source.nodeId,
type: DataSourceProvider.jinaReader,
})
}
if (source.type === DataSourceProvider.waterCrawl) {
options.push({
label: 'Water Crawl',
value: source.nodeId,
type: DataSourceProvider.waterCrawl,
})
}
})
return options
}, [datasources, t])

return { datasources, options }
}

+ 121
- 81
web/app/components/datasets/documents/create-from-pipeline/index.tsx 查看文件

'use client' 'use client'
import { useCallback, useMemo, useState } from 'react' import { useCallback, useMemo, useState } from 'react'
// import StepIndicator from './step-indicator'
// import { useTestRunSteps } from './hooks'
// import DataSourceOptions from './data-source-options'
import type { CrawlResultItem, FileItem } from '@/models/datasets'
import DataSourceOptions from './data-source-options'
import type { CrawlResultItem, CustomFile as File, FileItem } from '@/models/datasets'
import { DataSourceType } from '@/models/datasets' import { DataSourceType } from '@/models/datasets'
// import LocalFile from './data-source/local-file'
import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
import produce from 'immer' import produce from 'immer'
import { useProviderContextSelector } from '@/context/provider-context' import { useProviderContextSelector } from '@/context/provider-context'
import { DataSourceProvider, type NotionPage } from '@/models/common' import { DataSourceProvider, type NotionPage } from '@/models/common'
// import Notion from './data-source/notion'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
// import Firecrawl from './data-source/website/firecrawl'
// import JinaReader from './data-source/website/jina-reader'
// import WaterCrawl from './data-source/website/water-crawl'
// import Actions from './data-source/actions'
// import DocumentProcessing from './document-processing'
import { useTranslation } from 'react-i18next'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file'
import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion' import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl' import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl'
import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader' import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader'
import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl' import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl'
import Actions from '@/app/components/rag-pipeline/components/panel/test-run/data-source/actions'
import Actions from './data-source/actions'
import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing' import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing'
import { useTranslation } from 'react-i18next'
import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types'
import LeftHeader from './left-header' import LeftHeader from './left-header'
// import { usePipelineRun } from '../../../hooks'
// import type { Datasource } from './types'
import { usePublishedPipelineInfo } from '@/service/use-pipeline'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import Loading from '@/app/components/base/loading'
import type { Node } from '@/app/components/workflow/types'
import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types'
import FilePreview from './preview/file-preview'
import NotionPagePreview from './preview/notion-page-preview'
import WebsitePreview from './preview/web-preview'


const TestRunPanel = () => { const TestRunPanel = () => {
const { t } = useTranslation() const { t } = useTranslation()
const [notionPages, setNotionPages] = useState<NotionPage[]>([]) const [notionPages, setNotionPages] = useState<NotionPage[]>([])
const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([]) const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([])
const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('')
const [currentFile, setCurrentFile] = useState<File | undefined>()
const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>()
const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>()


const plan = useProviderContextSelector(state => state.plan) const plan = useProviderContextSelector(state => state.plan)
const enableBilling = useProviderContextSelector(state => state.enableBilling) const enableBilling = useProviderContextSelector(state => state.enableBilling)
const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id)


// const steps = useTestRunSteps()
const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '')


const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id))
const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace
setFiles(newList) setFiles(newList)
} }


const updateFileList = (preparedFiles: FileItem[]) => {
const updateFileList = useCallback((preparedFiles: FileItem[]) => {
setFiles(preparedFiles) setFiles(preparedFiles)
}
}, [])


const updateNotionPages = (value: NotionPage[]) => {
const updateNotionPages = useCallback((value: NotionPage[]) => {
setNotionPages(value) setNotionPages(value)
}
}, [])

const updateCurrentFile = useCallback((file: File) => {
setCurrentFile(file)
}, [])

const hideFilePreview = useCallback(() => {
setCurrentFile(undefined)
}, [])

const updateCurrentPage = useCallback((page: NotionPage) => {
setCurrentNotionPage(page)
}, [])

const hideNotionPagePreview = useCallback(() => {
setCurrentNotionPage(undefined)
}, [])

const updateCurrentWebsite = useCallback((website: CrawlResultItem) => {
setCurrentWebsite(website)
}, [])

const hideWebsitePreview = useCallback(() => {
setCurrentWebsite(undefined)
}, [])


const handleNextStep = useCallback(() => { const handleNextStep = useCallback(() => {
setCurrentStep(preStep => preStep + 1) setCurrentStep(preStep => preStep + 1)
setCurrentStep(preStep => preStep - 1) setCurrentStep(preStep => preStep - 1)
}, []) }, [])


// const { handleRun } = usePipelineRun()

const handleProcess = useCallback((data: Record<string, any>) => { const handleProcess = useCallback((data: Record<string, any>) => {
if (!datasource) if (!datasource)
return return
datasourceInfo.jobId = websiteCrawlJobId datasourceInfo.jobId = websiteCrawlJobId
datasourceInfo.result = websitePages datasourceInfo.result = websitePages
} }
// handleRun({
// inputs: data,
// datasource_type,
// datasource_info: datasourceInfo,
// })
// todo: Run Pipeline
console.log('datasource_type', datasource_type)
}, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages]) }, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages])


if (isFetchingPipelineInfo) {
return (
<Loading type='app' />
)
}

return ( return (
<div <div
className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle' className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle'
<div className='grow overflow-y-auto'> <div className='grow overflow-y-auto'>
{ {
currentStep === 1 && ( currentStep === 1 && (
<>
<div className='flex flex-col gap-y-4 px-4 py-2'>
{/* <DataSourceOptions
<div className='flex flex-col gap-y-5 pt-4'>
<DataSourceOptions
datasourceNodeId={datasource?.nodeId || ''} datasourceNodeId={datasource?.nodeId || ''}
onSelect={setDatasource} onSelect={setDatasource}
/> */}
{datasource?.type === DataSourceType.FILE && (
<LocalFile
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
notSupportBatchUpload={notSupportBatchUpload}
/>
)}
{datasource?.type === DataSourceType.NOTION && (
<Notion
nodeId={datasource?.nodeId || ''}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
/>
)}
{datasource?.type === DataSourceProvider.fireCrawl && (
<FireCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
/>
)}
{datasource?.type === DataSourceProvider.jinaReader && (
<JinaReader
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
/>
)}
{datasource?.type === DataSourceProvider.waterCrawl && (
<WaterCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
/>
)}
{isShowVectorSpaceFull && (
<VectorSpaceFull />
)}
</div>
pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]}
/>
{datasource?.type === DataSourceType.FILE && (
<LocalFile
files={fileList}
updateFile={updateFile}
updateFileList={updateFileList}
onPreview={updateCurrentFile}
notSupportBatchUpload={notSupportBatchUpload}
/>
)}
{datasource?.type === DataSourceType.NOTION && (
<Notion
nodeId={datasource?.nodeId || ''}
notionPages={notionPages}
updateNotionPages={updateNotionPages}
canPreview
onPreview={updateCurrentPage}
/>
)}
{datasource?.type === DataSourceProvider.fireCrawl && (
<FireCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>
)}
{datasource?.type === DataSourceProvider.jinaReader && (
<JinaReader
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>
)}
{datasource?.type === DataSourceProvider.waterCrawl && (
<WaterCrawl
nodeId={datasource?.nodeId || ''}
variables={datasource?.variables}
checkedCrawlResult={websitePages}
onCheckedCrawlResultChange={setWebsitePages}
onJobIdChange={setWebsiteCrawlJobId}
onPreview={updateCurrentWebsite}
/>
)}
{isShowVectorSpaceFull && (
<VectorSpaceFull />
)}
<Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} /> <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} />
</>
</div>
) )
} }
{ {
</div> </div>
{/* Preview */} {/* Preview */}
<div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'> <div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'>
{
currentStep === 1 && (
<>
{currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />}
{currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />}
{currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />}
</>
)
}
</div> </div>
</div> </div>
) )

+ 78
- 0
web/app/components/datasets/documents/create-from-pipeline/preview/file-preview.tsx 查看文件

'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import Loading from './loading'
import type { CustomFile as File } from '@/models/datasets'
import { RiCloseLine } from '@remixicon/react'
import { useFilePreview } from '@/service/use-common'
import DocumentFileIcon from '../../../common/document-file-icon'
import { formatNumberAbbreviated } from '@/utils/format'

type FilePreviewProps = {
file: File
hidePreview: () => void
}

const FilePreview = ({
file,
hidePreview,
}: FilePreviewProps) => {
const { t } = useTranslation()
const { data: fileData, isFetching } = useFilePreview(file.id || '')

const getFileName = (currentFile?: File) => {
if (!currentFile)
return ''
const arr = currentFile.name.split('.')
return arr.slice(0, -1).join()
}

const getFileSize = (size: number) => {
if (size / 1024 < 10)
return `${(size / 1024).toFixed(1)} KB`

return `${(size / 1024 / 1024).toFixed(1)} MB`
}

return (
<div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
<div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
<div className='flex grow flex-col gap-y-1'>
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
<div className='title-md-semi-bold text-tex-primary'>{`${getFileName(file)}.${file.extension}`}</div>
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
<DocumentFileIcon
className='size-6 shrink-0'
name={file.name}
extension={file.extension}
/>
<span className='uppercase'>{file.extension}</span>
<span>·</span>
<span>{getFileSize(file.size)}</span>
{fileData && (
<>
<span>·</span>
<span>{`${formatNumberAbbreviated(fileData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
</>
)}
</div>
</div>
<button
type='button'
className='flex h-8 w-8 shrink-0 items-center justify-center'
onClick={hidePreview}
>
<RiCloseLine className='size-[18px]' />
</button>
</div>
<div className='px-6 py-5'>
{isFetching && <Loading />}
{!isFetching && fileData && (
<div className='body-md-regular overflow-hidden text-text-secondary'>{fileData.content}</div>
)}
</div>
</div>
)
}

export default FilePreview

+ 52
- 0
web/app/components/datasets/documents/create-from-pipeline/preview/loading.tsx 查看文件

import React from 'react'
import { SkeletonContainer, SkeletonRectangle } from '@/app/components/base/skeleton'

const Loading = () => {
return (
<div className='flex h-full flex-col gap-y-12 bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg px-6 py-5'>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-3/5' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-[70%]' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-[56%]' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-3/5' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-3/5' />
</SkeletonContainer>
<SkeletonContainer className='w-full gap-0'>
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-full' />
<SkeletonRectangle className='my-1.5 w-1/2' />
</SkeletonContainer>
</div>
)
}

export default React.memo(Loading)

+ 65
- 0
web/app/components/datasets/documents/create-from-pipeline/preview/notion-page-preview.tsx 查看文件

'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import type { NotionPage } from '@/models/common'
import { usePreviewNotionPage } from '@/service/knowledge/use-dataset'
import { RiCloseLine } from '@remixicon/react'
import { formatNumberAbbreviated } from '@/utils/format'
import Loading from './loading'
import { Notion } from '@/app/components/base/icons/src/public/common'

type NotionPagePreviewProps = {
currentPage: NotionPage
hidePreview: () => void
}

const NotionPagePreview = ({
currentPage,
hidePreview,
}: NotionPagePreviewProps) => {
const { t } = useTranslation()

const { data: notionPageData, isFetching } = usePreviewNotionPage({
workspaceID: currentPage.workspace_id,
pageID: currentPage.page_id,
pageType: currentPage.type,
})

return (
<div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
<div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
<div className='flex grow flex-col gap-y-1'>
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
<div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div>
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
<Notion className='size-3.5' />
<span>·</span>
<span>Notion Page</span>
<span>·</span>
{notionPageData && (
<>
<span>·</span>
<span>{`${formatNumberAbbreviated(notionPageData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
</>
)}
</div>
</div>
<button
type='button'
className='flex h-8 w-8 shrink-0 items-center justify-center'
onClick={hidePreview}
>
<RiCloseLine className='size-[18px]' />
</button>
</div>
<div className='px-6 py-5'>
{isFetching && <Loading />}
{!isFetching && notionPageData && (
<div className='body-md-regular overflow-hidden text-text-secondary'>{notionPageData.content}</div>
)}
</div>
</div>
)
}

export default NotionPagePreview

+ 48
- 0
web/app/components/datasets/documents/create-from-pipeline/preview/web-preview.tsx 查看文件

'use client'
import React from 'react'
import { useTranslation } from 'react-i18next'
import type { CrawlResultItem } from '@/models/datasets'
import { RiCloseLine, RiGlobalLine } from '@remixicon/react'
import { formatNumberAbbreviated } from '@/utils/format'

type WebsitePreviewProps = {
payload: CrawlResultItem
hidePreview: () => void
}

const WebsitePreview = ({
payload,
hidePreview,
}: WebsitePreviewProps) => {
const { t } = useTranslation()

return (
<div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'>
<div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'>
<div className='flex grow flex-col gap-y-1'>
<div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div>
<div className='title-md-semi-bold text-tex-primary'>{payload.title}</div>
<div className='system-xs-medium flex gap-x-1 text-text-tertiary'>
<RiGlobalLine className='size-3.5' />
<span className='uppercase' title={payload.source_url}>{payload.source_url}</span>
<span>·</span>
<span>·</span>
<span>{`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span>
</div>
</div>
<button
type='button'
className='flex h-8 w-8 shrink-0 items-center justify-center'
onClick={hidePreview}
>
<RiCloseLine className='size-[18px]' />
</button>
</div>
<div className='px-6 py-5'>
<div className='body-md-regular overflow-hidden text-text-secondary'>{payload.markdown}</div>
</div>
</div>
)
}

export default WebsitePreview

+ 3
- 0
web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/file-uploader.tsx 查看文件

prepareFileList: (files: FileItem[]) => void prepareFileList: (files: FileItem[]) => void
onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void
onFileListUpdate?: (files: FileItem[]) => void onFileListUpdate?: (files: FileItem[]) => void
onPreview?: (file: File) => void
notSupportBatchUpload?: boolean notSupportBatchUpload?: boolean
} }


prepareFileList, prepareFileList,
onFileUpdate, onFileUpdate,
onFileListUpdate, onFileListUpdate,
onPreview,
notSupportBatchUpload, notSupportBatchUpload,
}: IFileUploaderProps) => { }: IFileUploaderProps) => {
const { t } = useTranslation() const { t } = useTranslation()
return ( return (
<div <div
key={`${fileItem.fileID}-${index}`} key={`${fileItem.fileID}-${index}`}
onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)}
className={cn( className={cn(
'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4',
isError && 'border-state-destructive-border bg-state-destructive-hover', isError && 'border-state-destructive-border bg-state-destructive-hover',

+ 4
- 1
web/app/components/rag-pipeline/components/panel/test-run/data-source/local-file/index.tsx 查看文件

import type { FileItem } from '@/models/datasets'
import type { CustomFile as File, FileItem } from '@/models/datasets'
import FileUploader from './file-uploader' import FileUploader from './file-uploader'


type LocalFileProps = { type LocalFileProps = {
files: FileItem[] files: FileItem[]
updateFileList: (files: FileItem[]) => void updateFileList: (files: FileItem[]) => void
updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
onPreview?: (file: File) => void
notSupportBatchUpload: boolean notSupportBatchUpload: boolean
} }


files, files,
updateFileList, updateFileList,
updateFile, updateFile,
onPreview,
notSupportBatchUpload, notSupportBatchUpload,
}: LocalFileProps) => { }: LocalFileProps) => {
return ( return (
prepareFileList={updateFileList} prepareFileList={updateFileList}
onFileListUpdate={updateFileList} onFileListUpdate={updateFileList}
onFileUpdate={updateFile} onFileUpdate={updateFile}
onPreview={onPreview}
notSupportBatchUpload={notSupportBatchUpload} notSupportBatchUpload={notSupportBatchUpload}
/> />
) )

+ 9
- 2
web/app/components/rag-pipeline/components/panel/test-run/data-source/notion/index.tsx 查看文件

nodeId: string nodeId: string
notionPages: NotionPage[] notionPages: NotionPage[]
updateNotionPages: (value: NotionPage[]) => void updateNotionPages: (value: NotionPage[]) => void
canPreview?: boolean
onPreview?: (selectedPage: NotionPage) => void
isInPipeline?: boolean
} }


const Notion = ({ const Notion = ({
nodeId, nodeId,
notionPages, notionPages,
updateNotionPages, updateNotionPages,
canPreview = false,
onPreview,
isInPipeline = false,
}: NotionProps) => { }: NotionProps) => {
return ( return (
<NotionPageSelector <NotionPageSelector
nodeId={nodeId} nodeId={nodeId}
value={notionPages.map(page => page.page_id)} value={notionPages.map(page => page.page_id)}
onSelect={updateNotionPages} onSelect={updateNotionPages}
canPreview={false}
isInPipeline
canPreview={canPreview}
onPreview={onPreview}
isInPipeline={isInPipeline}
/> />
) )
} }

+ 18
- 1
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result-item.tsx 查看文件

import cn from '@/utils/classnames' import cn from '@/utils/classnames'
import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets'
import Checkbox from '@/app/components/base/checkbox' import Checkbox from '@/app/components/base/checkbox'
import Button from '@/app/components/base/button'
import { useTranslation } from 'react-i18next'


type CrawledResultItemProps = { type CrawledResultItemProps = {
payload: CrawlResultItemType payload: CrawlResultItemType
isChecked: boolean isChecked: boolean
onCheckChange: (checked: boolean) => void onCheckChange: (checked: boolean) => void
isPreview: boolean
showPreview: boolean
onPreview: () => void
} }


const CrawledResultItem = ({ const CrawledResultItem = ({
payload, payload,
isChecked, isChecked,
onCheckChange, onCheckChange,
isPreview,
onPreview,
showPreview,
}: CrawledResultItemProps) => { }: CrawledResultItemProps) => {
const { t } = useTranslation()

const handleCheckChange = useCallback(() => { const handleCheckChange = useCallback(() => {
onCheckChange(!isChecked) onCheckChange(!isChecked)
}, [isChecked, onCheckChange]) }, [isChecked, onCheckChange])
return ( return (
<div className={cn('group flex cursor-pointer gap-x-2 rounded-lg p-2 hover:bg-state-base-hover')}>
<div className={cn('flex cursor-pointer gap-x-2 rounded-lg p-2', isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover')}>
<Checkbox <Checkbox
className='shrink-0' className='shrink-0'
checked={isChecked} checked={isChecked}
{payload.source_url} {payload.source_url}
</div> </div>
</div> </div>
{showPreview && <Button
size='small'
onClick={onPreview}
className='system-xs-medium-uppercase right-0 top-0 hidden px-1.5 group-hover:absolute group-hover:block'
>
{t('datasetCreation.stepOne.website.preview')}
</Button>}
</div> </div>
) )
} }

+ 14
- 2
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawled-result.tsx 查看文件

'use client' 'use client'
import React, { useCallback } from 'react'
import React, { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next' import { useTranslation } from 'react-i18next'
import cn from '@/utils/classnames' import cn from '@/utils/classnames'
import type { CrawlResultItem } from '@/models/datasets' import type { CrawlResultItem } from '@/models/datasets'
list: CrawlResultItem[] list: CrawlResultItem[]
checkedList: CrawlResultItem[] checkedList: CrawlResultItem[]
onSelectedChange: (selected: CrawlResultItem[]) => void onSelectedChange: (selected: CrawlResultItem[]) => void
onPreview?: (payload: CrawlResultItem) => void
usedTime: number usedTime: number
} }


checkedList, checkedList,
onSelectedChange, onSelectedChange,
usedTime, usedTime,
onPreview,
}: CrawledResultProps) => { }: CrawledResultProps) => {
const { t } = useTranslation() const { t } = useTranslation()
const [previewIndex, setPreviewIndex] = useState<number>(-1)


const isCheckAll = checkedList.length === list.length const isCheckAll = checkedList.length === list.length


} }
}, [checkedList, onSelectedChange]) }, [checkedList, onSelectedChange])


const handlePreview = useCallback((index: number) => {
if (!onPreview) return
setPreviewIndex(index)
onPreview(list[index])
}, [list, onPreview])

return ( return (
<div className={cn('flex flex-col gap-y-2', className)}> <div className={cn('flex flex-col gap-y-2', className)}>
<div className='system-sm-medium pt-2 text-text-primary'> <div className='system-sm-medium pt-2 text-text-primary'>
/> />
</div> </div>
<div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'> <div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'>
{list.map(item => (
{list.map((item, index) => (
<CrawledResultItem <CrawledResultItem
key={item.source_url} key={item.source_url}
payload={item} payload={item}
isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)} isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)}
onCheckChange={handleItemCheckChange(item)} onCheckChange={handleItemCheckChange(item)}
isPreview={index === previewIndex}
onPreview={handlePreview.bind(null, index)}
showPreview={!!onPreview}
/> />
))} ))}
</div> </div>

+ 3
- 0
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/base/crawler.tsx 查看文件

datasourceProvider: DataSourceProvider datasourceProvider: DataSourceProvider
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
} }


enum Step { enum Step {
datasourceProvider, datasourceProvider,
onCheckedCrawlResultChange, onCheckedCrawlResultChange,
onJobIdChange, onJobIdChange,
onPreview,
}: CrawlerProps) => { }: CrawlerProps) => {
const { t } = useTranslation() const { t } = useTranslation()
const [step, setStep] = useState<Step>(Step.init) const [step, setStep] = useState<Step>(Step.init)
checkedList={checkedCrawlResult} checkedList={checkedCrawlResult}
onSelectedChange={onCheckedCrawlResultChange} onSelectedChange={onCheckedCrawlResultChange}
usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0}
onPreview={onPreview}
/> />
)} )}
</div> </div>

+ 3
- 0
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl/index.tsx 查看文件

checkedCrawlResult: CrawlResultItem[] checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
} }


const FireCrawl = ({ const FireCrawl = ({
checkedCrawlResult, checkedCrawlResult,
onCheckedCrawlResultChange, onCheckedCrawlResultChange,
onJobIdChange, onJobIdChange,
onPreview,
}: FireCrawlProps) => { }: FireCrawlProps) => {
return ( return (
<Crawler <Crawler
datasourceProvider={DataSourceProvider.fireCrawl} datasourceProvider={DataSourceProvider.fireCrawl}
onCheckedCrawlResultChange={onCheckedCrawlResultChange} onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange} onJobIdChange={onJobIdChange}
onPreview={onPreview}
/> />
) )
} }

+ 3
- 0
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader/index.tsx 查看文件

checkedCrawlResult: CrawlResultItem[] checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
} }


const JinaReader = ({ const JinaReader = ({
checkedCrawlResult, checkedCrawlResult,
onCheckedCrawlResultChange, onCheckedCrawlResultChange,
onJobIdChange, onJobIdChange,
onPreview,
}: JinaReaderProps) => { }: JinaReaderProps) => {
return ( return (
<Crawler <Crawler
datasourceProvider={DataSourceProvider.jinaReader} datasourceProvider={DataSourceProvider.jinaReader}
onCheckedCrawlResultChange={onCheckedCrawlResultChange} onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange} onJobIdChange={onJobIdChange}
onPreview={onPreview}
/> />
) )
} }

+ 3
- 0
web/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl/index.tsx 查看文件

checkedCrawlResult: CrawlResultItem[] checkedCrawlResult: CrawlResultItem[]
onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void
onJobIdChange: (jobId: string) => void onJobIdChange: (jobId: string) => void
onPreview?: (payload: CrawlResultItem) => void
} }


const WaterCrawl = ({ const WaterCrawl = ({
checkedCrawlResult, checkedCrawlResult,
onCheckedCrawlResultChange, onCheckedCrawlResultChange,
onJobIdChange, onJobIdChange,
onPreview,
}: WaterCrawlProps) => { }: WaterCrawlProps) => {
return ( return (
<Crawler <Crawler
datasourceProvider={DataSourceProvider.jinaReader} datasourceProvider={DataSourceProvider.jinaReader}
onCheckedCrawlResultChange={onCheckedCrawlResultChange} onCheckedCrawlResultChange={onCheckedCrawlResultChange}
onJobIdChange={onJobIdChange} onJobIdChange={onJobIdChange}
onPreview={onPreview}
/> />
) )
} }

+ 4
- 0
web/i18n/en-US/dataset-pipeline.ts 查看文件

processingDocuments: 'Processing Documents', processingDocuments: 'Processing Documents',
}, },
backToDataSource: 'Data Source', backToDataSource: 'Data Source',
stepOne: {
preview: 'Preview',
},
characters: 'characters',
}, },
} }



+ 4
- 0
web/i18n/zh-Hans/dataset-pipeline.ts 查看文件

processingDocuments: '正在处理文档', processingDocuments: '正在处理文档',
}, },
backToDataSource: '数据源', backToDataSource: '数据源',
stepOne: {
preview: '预览',
},
characters: '字符',
}, },
} }



+ 19
- 0
web/service/knowledge/use-dataset.ts 查看文件

queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`), queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`),
}) })
} }

type NotionPagePreviewRequest = {
workspaceID: string
pageID: string
pageType: string
}

type NotionPagePreviewResponse = {
content: string
}

export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => {
const { workspaceID, pageID, pageType } = params
return useQuery({
queryKey: [NAME_SPACE, 'preview-notion-page'],
queryFn: () => get<NotionPagePreviewResponse>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`),
enabled: !!workspaceID && !!pageID && !!pageType,
})
}

+ 12
- 0
web/service/use-common.ts 查看文件

}), }),
}) })
} }

type FilePreviewResponse = {
content: string
}

export const useFilePreview = (fileID: string) => {
return useQuery<FilePreviewResponse>({
queryKey: [NAME_SPACE, 'file-preview', fileID],
queryFn: () => get<FilePreviewResponse>(`/files/${fileID}/preview`),
enabled: !!fileID,
})
}

+ 1
- 0
web/service/use-pipeline.ts 查看文件

queryFn: () => { queryFn: () => {
return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`) return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`)
}, },
enabled: !!pipelineId,
}) })
} }

+ 32
- 0
web/utils/format.ts 查看文件

a.remove() a.remove()
window.URL.revokeObjectURL(url) window.URL.revokeObjectURL(url)
} }

/**
* Formats a number into a readable string using "k", "M", or "B" suffix.
* @example
* 950 => "950"
* 1200 => "1.2k"
* 1500000 => "1.5M"
* 2000000000 => "2B"
*
* @param {number} num - The number to format
* @returns {string} - The formatted number string
*/
export const formatNumberAbbreviated = (num: number) => {
// If less than 1000, return as-is
if (num < 1000) return num.toString()

// Define thresholds and suffixes
const units = [
{ value: 1e9, symbol: 'B' },
{ value: 1e6, symbol: 'M' },
{ value: 1e3, symbol: 'k' },
]

for (let i = 0; i < units.length; i++) {
if (num >= units[i].value) {
const formatted = (num / units[i].value).toFixed(1)
return formatted.endsWith('.0')
? `${Number.parseInt(formatted)}${units[i].symbol}`
: `${formatted}${units[i].symbol}`
}
}
}

Loading…
取消
儲存