| }, | }, | ||||
| }) | }) | ||||
| const name = useStore(form.store, state => state.values.name) | |||||
| const name = useStore(form.store, state => state.values.name) | |||||
| return ( | return ( | ||||
| <form | <form | ||||
| ) | ) | ||||
| } | } | ||||
| <form.AppForm> | <form.AppForm> | ||||
| <form.SubmitButton>Submit</form.SubmitButton> | |||||
| <form.Actions /> | |||||
| </form.AppForm> | </form.AppForm> | ||||
| </form> | </form> | ||||
| ) | ) |
| import { useCallback, useEffect } from 'react' | |||||
| import { useDatasourceOptions } from '../hooks' | |||||
| import OptionCard from './option-card' | |||||
| import { File, Watercrawl } from '@/app/components/base/icons/src/public/knowledge' | |||||
| import { Notion } from '@/app/components/base/icons/src/public/common' | |||||
| import { Jina } from '@/app/components/base/icons/src/public/llm' | |||||
| import { DataSourceType } from '@/models/datasets' | |||||
| import { DataSourceProvider } from '@/models/common' | |||||
| import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' | |||||
| import type { Node } from '@/app/components/workflow/types' | |||||
| import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' | |||||
| type DataSourceOptionsProps = { | |||||
| pipelineNodes: Node<DataSourceNodeType>[] | |||||
| datasourceNodeId: string | |||||
| onSelect: (option: Datasource) => void | |||||
| } | |||||
| const DATA_SOURCE_ICONS = { | |||||
| [DataSourceType.FILE]: File as React.FC<React.SVGProps<SVGSVGElement>>, | |||||
| [DataSourceType.NOTION]: Notion as React.FC<React.SVGProps<SVGSVGElement>>, | |||||
| [DataSourceProvider.fireCrawl]: '🔥', | |||||
| [DataSourceProvider.jinaReader]: Jina as React.FC<React.SVGProps<SVGSVGElement>>, | |||||
| [DataSourceProvider.waterCrawl]: Watercrawl as React.FC<React.SVGProps<SVGSVGElement>>, | |||||
| } | |||||
| const DataSourceOptions = ({ | |||||
| pipelineNodes, | |||||
| datasourceNodeId, | |||||
| onSelect, | |||||
| }: DataSourceOptionsProps) => { | |||||
| const { datasources, options } = useDatasourceOptions(pipelineNodes) | |||||
| const handelSelect = useCallback((value: string) => { | |||||
| const selectedOption = datasources.find(option => option.nodeId === value) | |||||
| if (!selectedOption) | |||||
| return | |||||
| onSelect(selectedOption) | |||||
| }, [datasources, onSelect]) | |||||
| useEffect(() => { | |||||
| if (options.length > 0) | |||||
| handelSelect(options[0].value) | |||||
| // eslint-disable-next-line react-hooks/exhaustive-deps | |||||
| }, []) | |||||
| return ( | |||||
| <div className='grid w-full grid-cols-4 gap-1'> | |||||
| {options.map(option => ( | |||||
| <OptionCard | |||||
| key={option.value} | |||||
| label={option.label} | |||||
| selected={datasourceNodeId === option.value} | |||||
| Icon={DATA_SOURCE_ICONS[option.type as keyof typeof DATA_SOURCE_ICONS]} | |||||
| onClick={handelSelect.bind(null, option.value)} | |||||
| /> | |||||
| ))} | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default DataSourceOptions |
| import React from 'react' | |||||
| import cn from '@/utils/classnames' | |||||
| type OptionCardProps = { | |||||
| label: string | |||||
| Icon: React.FC<React.SVGProps<SVGSVGElement>> | string | |||||
| selected: boolean | |||||
| onClick?: () => void | |||||
| } | |||||
| const OptionCard = ({ | |||||
| label, | |||||
| Icon, | |||||
| selected, | |||||
| onClick, | |||||
| }: OptionCardProps) => { | |||||
| return ( | |||||
| <div | |||||
| className={cn( | |||||
| 'flex items-center gap-2 rounded-xl border border-components-option-card-option-border bg-components-option-card-option-bg p-3 shadow-shadow-shadow-3', | |||||
| selected | |||||
| ? 'border-components-option-card-option-selected-border bg-components-option-card-option-selected-bg shadow-xs ring-[0.5px] ring-inset ring-components-option-card-option-selected-border' | |||||
| : 'hover:bg-components-option-card-bg-hover hover:border-components-option-card-option-border-hover hover:shadow-xs', | |||||
| )} | |||||
| onClick={onClick} | |||||
| > | |||||
| <div className='flex size-8 items-center justify-center rounded-lg border-[0.5px] border-components-panel-border bg-background-default-dodge p-1.5'> | |||||
| { | |||||
| typeof Icon === 'string' | |||||
| ? <div className='text-[18px] leading-[18px]'>{Icon}</div> | |||||
| : <Icon className='size-5' /> | |||||
| } | |||||
| </div> | |||||
| <div className={cn('system-sm-medium text-text-secondary', selected && 'text-primary')}> | |||||
| {label} | |||||
| </div> | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default React.memo(OptionCard) |
| import React from 'react' | |||||
| import Button from '@/app/components/base/button' | |||||
| import { useTranslation } from 'react-i18next' | |||||
| import { useParams } from 'next/navigation' | |||||
| import { RiArrowRightLine } from '@remixicon/react' | |||||
| type ActionsProps = { | |||||
| disabled?: boolean | |||||
| handleNextStep: () => void | |||||
| } | |||||
| const Actions = ({ | |||||
| disabled, | |||||
| handleNextStep, | |||||
| }: ActionsProps) => { | |||||
| const { t } = useTranslation() | |||||
| const { datasetId } = useParams() | |||||
| return ( | |||||
| <div className='flex justify-end gap-x-2'> | |||||
| <a | |||||
| href={`/datasets/${datasetId}/documents`} | |||||
| > | |||||
| <Button | |||||
| variant='ghost' | |||||
| className='px-3 py-2' | |||||
| > | |||||
| {t('common.operation.cancel')} | |||||
| </Button> | |||||
| </a> | |||||
| <Button | |||||
| disabled={disabled} | |||||
| variant='primary' | |||||
| onClick={handleNextStep} | |||||
| className='gap-x-0.5' | |||||
| > | |||||
| <span className='px-0.5'>{t('datasetCreation.stepOne.button')}</span> | |||||
| <RiArrowRightLine className='size-4' /> | |||||
| </Button> | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default React.memo(Actions) |
| import { useTranslation } from 'react-i18next' | import { useTranslation } from 'react-i18next' | ||||
| import { AddDocumentsStep } from './types' | import { AddDocumentsStep } from './types' | ||||
| import type { DataSourceOption, Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' | |||||
| import { useMemo } from 'react' | |||||
| import { BlockEnum, type Node } from '@/app/components/workflow/types' | |||||
| import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' | |||||
| import { DataSourceType } from '@/models/datasets' | |||||
| import { DataSourceProvider } from '@/models/common' | |||||
| export const useAddDocumentsSteps = () => { | export const useAddDocumentsSteps = () => { | ||||
| const { t } = useTranslation() | const { t } = useTranslation() | ||||
| ] | ] | ||||
| return steps | return steps | ||||
| } | } | ||||
| export const useDatasourceOptions = (pipelineNodes: Node<DataSourceNodeType>[]) => { | |||||
| const { t } = useTranslation() | |||||
| const datasources: Datasource[] = useMemo(() => { | |||||
| const datasourceNodes = pipelineNodes.filter(node => node.data.type === BlockEnum.DataSource) | |||||
| return datasourceNodes.map((node) => { | |||||
| let type: DataSourceType | DataSourceProvider = DataSourceType.FILE | |||||
| switch (node.data.tool_name) { | |||||
| case 'file_upload': | |||||
| type = DataSourceType.FILE | |||||
| break | |||||
| case 'search_notion': | |||||
| type = DataSourceType.NOTION | |||||
| break | |||||
| case 'firecrawl': | |||||
| type = DataSourceProvider.fireCrawl | |||||
| break | |||||
| case 'jina_reader': | |||||
| type = DataSourceProvider.jinaReader | |||||
| break | |||||
| case 'water_crawl': | |||||
| type = DataSourceProvider.waterCrawl | |||||
| break | |||||
| } | |||||
| return { | |||||
| nodeId: node.id, | |||||
| type, | |||||
| variables: node.data.variables, | |||||
| } | |||||
| }) | |||||
| }, [pipelineNodes]) | |||||
| const options = useMemo(() => { | |||||
| const options: DataSourceOption[] = [] | |||||
| datasources.forEach((source) => { | |||||
| if (source.type === DataSourceType.FILE) { | |||||
| options.push({ | |||||
| label: t('datasetPipeline.testRun.dataSource.localFiles'), | |||||
| value: source.nodeId, | |||||
| type: DataSourceType.FILE, | |||||
| }) | |||||
| } | |||||
| if (source.type === DataSourceType.NOTION) { | |||||
| options.push({ | |||||
| label: 'Notion', | |||||
| value: source.nodeId, | |||||
| type: DataSourceType.NOTION, | |||||
| }) | |||||
| } | |||||
| if (source.type === DataSourceProvider.fireCrawl) { | |||||
| options.push({ | |||||
| label: 'Firecrawl', | |||||
| value: source.nodeId, | |||||
| type: DataSourceProvider.fireCrawl, | |||||
| }) | |||||
| } | |||||
| if (source.type === DataSourceProvider.jinaReader) { | |||||
| options.push({ | |||||
| label: 'Jina Reader', | |||||
| value: source.nodeId, | |||||
| type: DataSourceProvider.jinaReader, | |||||
| }) | |||||
| } | |||||
| if (source.type === DataSourceProvider.waterCrawl) { | |||||
| options.push({ | |||||
| label: 'Water Crawl', | |||||
| value: source.nodeId, | |||||
| type: DataSourceProvider.waterCrawl, | |||||
| }) | |||||
| } | |||||
| }) | |||||
| return options | |||||
| }, [datasources, t]) | |||||
| return { datasources, options } | |||||
| } |
| 'use client' | 'use client' | ||||
| import { useCallback, useMemo, useState } from 'react' | import { useCallback, useMemo, useState } from 'react' | ||||
| // import StepIndicator from './step-indicator' | |||||
| // import { useTestRunSteps } from './hooks' | |||||
| // import DataSourceOptions from './data-source-options' | |||||
| import type { CrawlResultItem, FileItem } from '@/models/datasets' | |||||
| import DataSourceOptions from './data-source-options' | |||||
| import type { CrawlResultItem, CustomFile as File, FileItem } from '@/models/datasets' | |||||
| import { DataSourceType } from '@/models/datasets' | import { DataSourceType } from '@/models/datasets' | ||||
| // import LocalFile from './data-source/local-file' | |||||
| import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file' | |||||
| import produce from 'immer' | import produce from 'immer' | ||||
| import { useProviderContextSelector } from '@/context/provider-context' | import { useProviderContextSelector } from '@/context/provider-context' | ||||
| import { DataSourceProvider, type NotionPage } from '@/models/common' | import { DataSourceProvider, type NotionPage } from '@/models/common' | ||||
| // import Notion from './data-source/notion' | |||||
| import VectorSpaceFull from '@/app/components/billing/vector-space-full' | |||||
| // import Firecrawl from './data-source/website/firecrawl' | |||||
| // import JinaReader from './data-source/website/jina-reader' | |||||
| // import WaterCrawl from './data-source/website/water-crawl' | |||||
| // import Actions from './data-source/actions' | |||||
| // import DocumentProcessing from './document-processing' | |||||
| import { useTranslation } from 'react-i18next' | |||||
| import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' | |||||
| import LocalFile from '@/app/components/rag-pipeline/components/panel/test-run/data-source/local-file' | |||||
| import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion' | import Notion from '@/app/components/rag-pipeline/components/panel/test-run/data-source/notion' | ||||
| import VectorSpaceFull from '@/app/components/billing/vector-space-full' | |||||
| import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl' | import FireCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/firecrawl' | ||||
| import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader' | import JinaReader from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/jina-reader' | ||||
| import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl' | import WaterCrawl from '@/app/components/rag-pipeline/components/panel/test-run/data-source/website/water-crawl' | ||||
| import Actions from '@/app/components/rag-pipeline/components/panel/test-run/data-source/actions' | |||||
| import Actions from './data-source/actions' | |||||
| import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing' | import DocumentProcessing from '@/app/components/rag-pipeline/components/panel/test-run/document-processing' | ||||
| import { useTranslation } from 'react-i18next' | |||||
| import type { Datasource } from '@/app/components/rag-pipeline/components/panel/test-run/types' | |||||
| import LeftHeader from './left-header' | import LeftHeader from './left-header' | ||||
| // import { usePipelineRun } from '../../../hooks' | |||||
| // import type { Datasource } from './types' | |||||
| import { usePublishedPipelineInfo } from '@/service/use-pipeline' | |||||
| import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail' | |||||
| import Loading from '@/app/components/base/loading' | |||||
| import type { Node } from '@/app/components/workflow/types' | |||||
| import type { DataSourceNodeType } from '@/app/components/workflow/nodes/data-source/types' | |||||
| import FilePreview from './preview/file-preview' | |||||
| import NotionPagePreview from './preview/notion-page-preview' | |||||
| import WebsitePreview from './preview/web-preview' | |||||
| const TestRunPanel = () => { | const TestRunPanel = () => { | ||||
| const { t } = useTranslation() | const { t } = useTranslation() | ||||
| const [notionPages, setNotionPages] = useState<NotionPage[]>([]) | const [notionPages, setNotionPages] = useState<NotionPage[]>([]) | ||||
| const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([]) | const [websitePages, setWebsitePages] = useState<CrawlResultItem[]>([]) | ||||
| const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') | const [websiteCrawlJobId, setWebsiteCrawlJobId] = useState('') | ||||
| const [currentFile, setCurrentFile] = useState<File | undefined>() | |||||
| const [currentNotionPage, setCurrentNotionPage] = useState<NotionPage | undefined>() | |||||
| const [currentWebsite, setCurrentWebsite] = useState<CrawlResultItem | undefined>() | |||||
| const plan = useProviderContextSelector(state => state.plan) | const plan = useProviderContextSelector(state => state.plan) | ||||
| const enableBilling = useProviderContextSelector(state => state.enableBilling) | const enableBilling = useProviderContextSelector(state => state.enableBilling) | ||||
| const pipelineId = useDatasetDetailContextWithSelector(s => s.dataset?.pipeline_id) | |||||
| // const steps = useTestRunSteps() | |||||
| const { data: pipelineInfo, isFetching: isFetchingPipelineInfo } = usePublishedPipelineInfo(pipelineId || '') | |||||
| const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) | const allFileLoaded = (fileList.length > 0 && fileList.every(file => file.file.id)) | ||||
| const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace | const isVectorSpaceFull = plan.usage.vectorSpace >= plan.total.vectorSpace | ||||
| setFiles(newList) | setFiles(newList) | ||||
| } | } | ||||
| const updateFileList = (preparedFiles: FileItem[]) => { | |||||
| const updateFileList = useCallback((preparedFiles: FileItem[]) => { | |||||
| setFiles(preparedFiles) | setFiles(preparedFiles) | ||||
| } | |||||
| }, []) | |||||
| const updateNotionPages = (value: NotionPage[]) => { | |||||
| const updateNotionPages = useCallback((value: NotionPage[]) => { | |||||
| setNotionPages(value) | setNotionPages(value) | ||||
| } | |||||
| }, []) | |||||
| const updateCurrentFile = useCallback((file: File) => { | |||||
| setCurrentFile(file) | |||||
| }, []) | |||||
| const hideFilePreview = useCallback(() => { | |||||
| setCurrentFile(undefined) | |||||
| }, []) | |||||
| const updateCurrentPage = useCallback((page: NotionPage) => { | |||||
| setCurrentNotionPage(page) | |||||
| }, []) | |||||
| const hideNotionPagePreview = useCallback(() => { | |||||
| setCurrentNotionPage(undefined) | |||||
| }, []) | |||||
| const updateCurrentWebsite = useCallback((website: CrawlResultItem) => { | |||||
| setCurrentWebsite(website) | |||||
| }, []) | |||||
| const hideWebsitePreview = useCallback(() => { | |||||
| setCurrentWebsite(undefined) | |||||
| }, []) | |||||
| const handleNextStep = useCallback(() => { | const handleNextStep = useCallback(() => { | ||||
| setCurrentStep(preStep => preStep + 1) | setCurrentStep(preStep => preStep + 1) | ||||
| setCurrentStep(preStep => preStep - 1) | setCurrentStep(preStep => preStep - 1) | ||||
| }, []) | }, []) | ||||
| // const { handleRun } = usePipelineRun() | |||||
| const handleProcess = useCallback((data: Record<string, any>) => { | const handleProcess = useCallback((data: Record<string, any>) => { | ||||
| if (!datasource) | if (!datasource) | ||||
| return | return | ||||
| datasourceInfo.jobId = websiteCrawlJobId | datasourceInfo.jobId = websiteCrawlJobId | ||||
| datasourceInfo.result = websitePages | datasourceInfo.result = websitePages | ||||
| } | } | ||||
| // handleRun({ | |||||
| // inputs: data, | |||||
| // datasource_type, | |||||
| // datasource_info: datasourceInfo, | |||||
| // }) | |||||
| // todo: Run Pipeline | |||||
| console.log('datasource_type', datasource_type) | |||||
| }, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages]) | }, [datasource, fileList, notionPages, websiteCrawlJobId, websitePages]) | ||||
| if (isFetchingPipelineInfo) { | |||||
| return ( | |||||
| <Loading type='app' /> | |||||
| ) | |||||
| } | |||||
| return ( | return ( | ||||
| <div | <div | ||||
| className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle' | className='relative flex h-[calc(100vh-56px)] min-w-[1512px] rounded-t-2xl border-t border-effects-highlight bg-background-default-subtle' | ||||
| <div className='grow overflow-y-auto'> | <div className='grow overflow-y-auto'> | ||||
| { | { | ||||
| currentStep === 1 && ( | currentStep === 1 && ( | ||||
| <> | |||||
| <div className='flex flex-col gap-y-4 px-4 py-2'> | |||||
| {/* <DataSourceOptions | |||||
| <div className='flex flex-col gap-y-5 pt-4'> | |||||
| <DataSourceOptions | |||||
| datasourceNodeId={datasource?.nodeId || ''} | datasourceNodeId={datasource?.nodeId || ''} | ||||
| onSelect={setDatasource} | onSelect={setDatasource} | ||||
| /> */} | |||||
| {datasource?.type === DataSourceType.FILE && ( | |||||
| <LocalFile | |||||
| files={fileList} | |||||
| updateFile={updateFile} | |||||
| updateFileList={updateFileList} | |||||
| notSupportBatchUpload={notSupportBatchUpload} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceType.NOTION && ( | |||||
| <Notion | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| notionPages={notionPages} | |||||
| updateNotionPages={updateNotionPages} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceProvider.fireCrawl && ( | |||||
| <FireCrawl | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| variables={datasource?.variables} | |||||
| checkedCrawlResult={websitePages} | |||||
| onCheckedCrawlResultChange={setWebsitePages} | |||||
| onJobIdChange={setWebsiteCrawlJobId} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceProvider.jinaReader && ( | |||||
| <JinaReader | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| variables={datasource?.variables} | |||||
| checkedCrawlResult={websitePages} | |||||
| onCheckedCrawlResultChange={setWebsitePages} | |||||
| onJobIdChange={setWebsiteCrawlJobId} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceProvider.waterCrawl && ( | |||||
| <WaterCrawl | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| variables={datasource?.variables} | |||||
| checkedCrawlResult={websitePages} | |||||
| onCheckedCrawlResultChange={setWebsitePages} | |||||
| onJobIdChange={setWebsiteCrawlJobId} | |||||
| /> | |||||
| )} | |||||
| {isShowVectorSpaceFull && ( | |||||
| <VectorSpaceFull /> | |||||
| )} | |||||
| </div> | |||||
| pipelineNodes={(pipelineInfo?.graph.nodes || []) as Node<DataSourceNodeType>[]} | |||||
| /> | |||||
| {datasource?.type === DataSourceType.FILE && ( | |||||
| <LocalFile | |||||
| files={fileList} | |||||
| updateFile={updateFile} | |||||
| updateFileList={updateFileList} | |||||
| onPreview={updateCurrentFile} | |||||
| notSupportBatchUpload={notSupportBatchUpload} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceType.NOTION && ( | |||||
| <Notion | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| notionPages={notionPages} | |||||
| updateNotionPages={updateNotionPages} | |||||
| canPreview | |||||
| onPreview={updateCurrentPage} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceProvider.fireCrawl && ( | |||||
| <FireCrawl | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| variables={datasource?.variables} | |||||
| checkedCrawlResult={websitePages} | |||||
| onCheckedCrawlResultChange={setWebsitePages} | |||||
| onJobIdChange={setWebsiteCrawlJobId} | |||||
| onPreview={updateCurrentWebsite} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceProvider.jinaReader && ( | |||||
| <JinaReader | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| variables={datasource?.variables} | |||||
| checkedCrawlResult={websitePages} | |||||
| onCheckedCrawlResultChange={setWebsitePages} | |||||
| onJobIdChange={setWebsiteCrawlJobId} | |||||
| onPreview={updateCurrentWebsite} | |||||
| /> | |||||
| )} | |||||
| {datasource?.type === DataSourceProvider.waterCrawl && ( | |||||
| <WaterCrawl | |||||
| nodeId={datasource?.nodeId || ''} | |||||
| variables={datasource?.variables} | |||||
| checkedCrawlResult={websitePages} | |||||
| onCheckedCrawlResultChange={setWebsitePages} | |||||
| onJobIdChange={setWebsiteCrawlJobId} | |||||
| onPreview={updateCurrentWebsite} | |||||
| /> | |||||
| )} | |||||
| {isShowVectorSpaceFull && ( | |||||
| <VectorSpaceFull /> | |||||
| )} | |||||
| <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} /> | <Actions disabled={nextBtnDisabled} handleNextStep={handleNextStep} /> | ||||
| </> | |||||
| </div> | |||||
| ) | ) | ||||
| } | } | ||||
| { | { | ||||
| </div> | </div> | ||||
| {/* Preview */} | {/* Preview */} | ||||
| <div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'> | <div className='flex h-full flex-1 shrink-0 flex-col pl-2 pt-2'> | ||||
| { | |||||
| currentStep === 1 && ( | |||||
| <> | |||||
| {currentFile && <FilePreview file={currentFile} hidePreview={hideFilePreview} />} | |||||
| {currentNotionPage && <NotionPagePreview currentPage={currentNotionPage} hidePreview={hideNotionPagePreview} />} | |||||
| {currentWebsite && <WebsitePreview payload={currentWebsite} hidePreview={hideWebsitePreview} />} | |||||
| </> | |||||
| ) | |||||
| } | |||||
| </div> | </div> | ||||
| </div> | </div> | ||||
| ) | ) |
| 'use client' | |||||
| import React from 'react' | |||||
| import { useTranslation } from 'react-i18next' | |||||
| import Loading from './loading' | |||||
| import type { CustomFile as File } from '@/models/datasets' | |||||
| import { RiCloseLine } from '@remixicon/react' | |||||
| import { useFilePreview } from '@/service/use-common' | |||||
| import DocumentFileIcon from '../../../common/document-file-icon' | |||||
| import { formatNumberAbbreviated } from '@/utils/format' | |||||
| type FilePreviewProps = { | |||||
| file: File | |||||
| hidePreview: () => void | |||||
| } | |||||
| const FilePreview = ({ | |||||
| file, | |||||
| hidePreview, | |||||
| }: FilePreviewProps) => { | |||||
| const { t } = useTranslation() | |||||
| const { data: fileData, isFetching } = useFilePreview(file.id || '') | |||||
| const getFileName = (currentFile?: File) => { | |||||
| if (!currentFile) | |||||
| return '' | |||||
| const arr = currentFile.name.split('.') | |||||
| return arr.slice(0, -1).join() | |||||
| } | |||||
| const getFileSize = (size: number) => { | |||||
| if (size / 1024 < 10) | |||||
| return `${(size / 1024).toFixed(1)} KB` | |||||
| return `${(size / 1024 / 1024).toFixed(1)} MB` | |||||
| } | |||||
| return ( | |||||
| <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'> | |||||
| <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'> | |||||
| <div className='flex grow flex-col gap-y-1'> | |||||
| <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div> | |||||
| <div className='title-md-semi-bold text-tex-primary'>{`${getFileName(file)}.${file.extension}`}</div> | |||||
| <div className='system-xs-medium flex gap-x-1 text-text-tertiary'> | |||||
| <DocumentFileIcon | |||||
| className='size-6 shrink-0' | |||||
| name={file.name} | |||||
| extension={file.extension} | |||||
| /> | |||||
| <span className='uppercase'>{file.extension}</span> | |||||
| <span>·</span> | |||||
| <span>{getFileSize(file.size)}</span> | |||||
| {fileData && ( | |||||
| <> | |||||
| <span>·</span> | |||||
| <span>{`${formatNumberAbbreviated(fileData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span> | |||||
| </> | |||||
| )} | |||||
| </div> | |||||
| </div> | |||||
| <button | |||||
| type='button' | |||||
| className='flex h-8 w-8 shrink-0 items-center justify-center' | |||||
| onClick={hidePreview} | |||||
| > | |||||
| <RiCloseLine className='size-[18px]' /> | |||||
| </button> | |||||
| </div> | |||||
| <div className='px-6 py-5'> | |||||
| {isFetching && <Loading />} | |||||
| {!isFetching && fileData && ( | |||||
| <div className='body-md-regular overflow-hidden text-text-secondary'>{fileData.content}</div> | |||||
| )} | |||||
| </div> | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default FilePreview |
| import React from 'react' | |||||
| import { SkeletonContainer, SkeletonRectangle } from '@/app/components/base/skeleton' | |||||
| const Loading = () => { | |||||
| return ( | |||||
| <div className='flex h-full flex-col gap-y-12 bg-gradient-to-b from-components-panel-bg-transparent to-components-panel-bg px-6 py-5'> | |||||
| <SkeletonContainer className='w-full gap-0'> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-3/5' /> | |||||
| </SkeletonContainer> | |||||
| <SkeletonContainer className='w-full gap-0'> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-[70%]' /> | |||||
| </SkeletonContainer> | |||||
| <SkeletonContainer className='w-full gap-0'> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-[56%]' /> | |||||
| </SkeletonContainer> | |||||
| <SkeletonContainer className='w-full gap-0'> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-3/5' /> | |||||
| </SkeletonContainer> | |||||
| <SkeletonContainer className='w-full gap-0'> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-3/5' /> | |||||
| </SkeletonContainer> | |||||
| <SkeletonContainer className='w-full gap-0'> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-full' /> | |||||
| <SkeletonRectangle className='my-1.5 w-1/2' /> | |||||
| </SkeletonContainer> | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default React.memo(Loading) |
| 'use client' | |||||
| import React from 'react' | |||||
| import { useTranslation } from 'react-i18next' | |||||
| import type { NotionPage } from '@/models/common' | |||||
| import { usePreviewNotionPage } from '@/service/knowledge/use-dataset' | |||||
| import { RiCloseLine } from '@remixicon/react' | |||||
| import { formatNumberAbbreviated } from '@/utils/format' | |||||
| import Loading from './loading' | |||||
| import { Notion } from '@/app/components/base/icons/src/public/common' | |||||
| type NotionPagePreviewProps = { | |||||
| currentPage: NotionPage | |||||
| hidePreview: () => void | |||||
| } | |||||
| const NotionPagePreview = ({ | |||||
| currentPage, | |||||
| hidePreview, | |||||
| }: NotionPagePreviewProps) => { | |||||
| const { t } = useTranslation() | |||||
| const { data: notionPageData, isFetching } = usePreviewNotionPage({ | |||||
| workspaceID: currentPage.workspace_id, | |||||
| pageID: currentPage.page_id, | |||||
| pageType: currentPage.type, | |||||
| }) | |||||
| return ( | |||||
| <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'> | |||||
| <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'> | |||||
| <div className='flex grow flex-col gap-y-1'> | |||||
| <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div> | |||||
| <div className='title-md-semi-bold text-tex-primary'>{currentPage?.page_name}</div> | |||||
| <div className='system-xs-medium flex gap-x-1 text-text-tertiary'> | |||||
| <Notion className='size-3.5' /> | |||||
| <span>·</span> | |||||
| <span>Notion Page</span> | |||||
| <span>·</span> | |||||
| {notionPageData && ( | |||||
| <> | |||||
| <span>·</span> | |||||
| <span>{`${formatNumberAbbreviated(notionPageData.content.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span> | |||||
| </> | |||||
| )} | |||||
| </div> | |||||
| </div> | |||||
| <button | |||||
| type='button' | |||||
| className='flex h-8 w-8 shrink-0 items-center justify-center' | |||||
| onClick={hidePreview} | |||||
| > | |||||
| <RiCloseLine className='size-[18px]' /> | |||||
| </button> | |||||
| </div> | |||||
| <div className='px-6 py-5'> | |||||
| {isFetching && <Loading />} | |||||
| {!isFetching && notionPageData && ( | |||||
| <div className='body-md-regular overflow-hidden text-text-secondary'>{notionPageData.content}</div> | |||||
| )} | |||||
| </div> | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default NotionPagePreview |
| 'use client' | |||||
| import React from 'react' | |||||
| import { useTranslation } from 'react-i18next' | |||||
| import type { CrawlResultItem } from '@/models/datasets' | |||||
| import { RiCloseLine, RiGlobalLine } from '@remixicon/react' | |||||
| import { formatNumberAbbreviated } from '@/utils/format' | |||||
| type WebsitePreviewProps = { | |||||
| payload: CrawlResultItem | |||||
| hidePreview: () => void | |||||
| } | |||||
| const WebsitePreview = ({ | |||||
| payload, | |||||
| hidePreview, | |||||
| }: WebsitePreviewProps) => { | |||||
| const { t } = useTranslation() | |||||
| return ( | |||||
| <div className='h-full rounded-t-xl border-l border-t border-components-panel-border bg-background-default-lighter shadow-md shadow-shadow-shadow-5'> | |||||
| <div className='flex gap-x-2 pb-3 pl-6 pr-4 pt-4'> | |||||
| <div className='flex grow flex-col gap-y-1'> | |||||
| <div className='system-2xs-semibold-uppercase'>{t('datasetPipeline.addDocuments.stepOne.preview')}</div> | |||||
| <div className='title-md-semi-bold text-tex-primary'>{payload.title}</div> | |||||
| <div className='system-xs-medium flex gap-x-1 text-text-tertiary'> | |||||
| <RiGlobalLine className='size-3.5' /> | |||||
| <span className='uppercase' title={payload.source_url}>{payload.source_url}</span> | |||||
| <span>·</span> | |||||
| <span>·</span> | |||||
| <span>{`${formatNumberAbbreviated(payload.markdown.length)} ${t('datasetPipeline.addDocuments.characters')}`}</span> | |||||
| </div> | |||||
| </div> | |||||
| <button | |||||
| type='button' | |||||
| className='flex h-8 w-8 shrink-0 items-center justify-center' | |||||
| onClick={hidePreview} | |||||
| > | |||||
| <RiCloseLine className='size-[18px]' /> | |||||
| </button> | |||||
| </div> | |||||
| <div className='px-6 py-5'> | |||||
| <div className='body-md-regular overflow-hidden text-text-secondary'>{payload.markdown}</div> | |||||
| </div> | |||||
| </div> | |||||
| ) | |||||
| } | |||||
| export default WebsitePreview |
| prepareFileList: (files: FileItem[]) => void | prepareFileList: (files: FileItem[]) => void | ||||
| onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void | onFileUpdate: (fileItem: FileItem, progress: number, list: FileItem[]) => void | ||||
| onFileListUpdate?: (files: FileItem[]) => void | onFileListUpdate?: (files: FileItem[]) => void | ||||
| onPreview?: (file: File) => void | |||||
| notSupportBatchUpload?: boolean | notSupportBatchUpload?: boolean | ||||
| } | } | ||||
| prepareFileList, | prepareFileList, | ||||
| onFileUpdate, | onFileUpdate, | ||||
| onFileListUpdate, | onFileListUpdate, | ||||
| onPreview, | |||||
| notSupportBatchUpload, | notSupportBatchUpload, | ||||
| }: IFileUploaderProps) => { | }: IFileUploaderProps) => { | ||||
| const { t } = useTranslation() | const { t } = useTranslation() | ||||
| return ( | return ( | ||||
| <div | <div | ||||
| key={`${fileItem.fileID}-${index}`} | key={`${fileItem.fileID}-${index}`} | ||||
| onClick={() => fileItem.file?.id && onPreview?.(fileItem.file)} | |||||
| className={cn( | className={cn( | ||||
| 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', | 'flex h-12 items-center rounded-lg border border-components-panel-border bg-components-panel-on-panel-item-bg shadow-xs shadow-shadow-shadow-4', | ||||
| isError && 'border-state-destructive-border bg-state-destructive-hover', | isError && 'border-state-destructive-border bg-state-destructive-hover', |
| import type { FileItem } from '@/models/datasets' | |||||
| import type { CustomFile as File, FileItem } from '@/models/datasets' | |||||
| import FileUploader from './file-uploader' | import FileUploader from './file-uploader' | ||||
| type LocalFileProps = { | type LocalFileProps = { | ||||
| files: FileItem[] | files: FileItem[] | ||||
| updateFileList: (files: FileItem[]) => void | updateFileList: (files: FileItem[]) => void | ||||
| updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void | updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void | ||||
| onPreview?: (file: File) => void | |||||
| notSupportBatchUpload: boolean | notSupportBatchUpload: boolean | ||||
| } | } | ||||
| files, | files, | ||||
| updateFileList, | updateFileList, | ||||
| updateFile, | updateFile, | ||||
| onPreview, | |||||
| notSupportBatchUpload, | notSupportBatchUpload, | ||||
| }: LocalFileProps) => { | }: LocalFileProps) => { | ||||
| return ( | return ( | ||||
| prepareFileList={updateFileList} | prepareFileList={updateFileList} | ||||
| onFileListUpdate={updateFileList} | onFileListUpdate={updateFileList} | ||||
| onFileUpdate={updateFile} | onFileUpdate={updateFile} | ||||
| onPreview={onPreview} | |||||
| notSupportBatchUpload={notSupportBatchUpload} | notSupportBatchUpload={notSupportBatchUpload} | ||||
| /> | /> | ||||
| ) | ) |
| nodeId: string | nodeId: string | ||||
| notionPages: NotionPage[] | notionPages: NotionPage[] | ||||
| updateNotionPages: (value: NotionPage[]) => void | updateNotionPages: (value: NotionPage[]) => void | ||||
| canPreview?: boolean | |||||
| onPreview?: (selectedPage: NotionPage) => void | |||||
| isInPipeline?: boolean | |||||
| } | } | ||||
| const Notion = ({ | const Notion = ({ | ||||
| nodeId, | nodeId, | ||||
| notionPages, | notionPages, | ||||
| updateNotionPages, | updateNotionPages, | ||||
| canPreview = false, | |||||
| onPreview, | |||||
| isInPipeline = false, | |||||
| }: NotionProps) => { | }: NotionProps) => { | ||||
| return ( | return ( | ||||
| <NotionPageSelector | <NotionPageSelector | ||||
| nodeId={nodeId} | nodeId={nodeId} | ||||
| value={notionPages.map(page => page.page_id)} | value={notionPages.map(page => page.page_id)} | ||||
| onSelect={updateNotionPages} | onSelect={updateNotionPages} | ||||
| canPreview={false} | |||||
| isInPipeline | |||||
| canPreview={canPreview} | |||||
| onPreview={onPreview} | |||||
| isInPipeline={isInPipeline} | |||||
| /> | /> | ||||
| ) | ) | ||||
| } | } |
| import cn from '@/utils/classnames' | import cn from '@/utils/classnames' | ||||
| import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' | import type { CrawlResultItem as CrawlResultItemType } from '@/models/datasets' | ||||
| import Checkbox from '@/app/components/base/checkbox' | import Checkbox from '@/app/components/base/checkbox' | ||||
| import Button from '@/app/components/base/button' | |||||
| import { useTranslation } from 'react-i18next' | |||||
| type CrawledResultItemProps = { | type CrawledResultItemProps = { | ||||
| payload: CrawlResultItemType | payload: CrawlResultItemType | ||||
| isChecked: boolean | isChecked: boolean | ||||
| onCheckChange: (checked: boolean) => void | onCheckChange: (checked: boolean) => void | ||||
| isPreview: boolean | |||||
| showPreview: boolean | |||||
| onPreview: () => void | |||||
| } | } | ||||
| const CrawledResultItem = ({ | const CrawledResultItem = ({ | ||||
| payload, | payload, | ||||
| isChecked, | isChecked, | ||||
| onCheckChange, | onCheckChange, | ||||
| isPreview, | |||||
| onPreview, | |||||
| showPreview, | |||||
| }: CrawledResultItemProps) => { | }: CrawledResultItemProps) => { | ||||
| const { t } = useTranslation() | |||||
| const handleCheckChange = useCallback(() => { | const handleCheckChange = useCallback(() => { | ||||
| onCheckChange(!isChecked) | onCheckChange(!isChecked) | ||||
| }, [isChecked, onCheckChange]) | }, [isChecked, onCheckChange]) | ||||
| return ( | return ( | ||||
| <div className={cn('group flex cursor-pointer gap-x-2 rounded-lg p-2 hover:bg-state-base-hover')}> | |||||
| <div className={cn('flex cursor-pointer gap-x-2 rounded-lg p-2', isPreview ? 'bg-state-base-active' : 'group hover:bg-state-base-hover')}> | |||||
| <Checkbox | <Checkbox | ||||
| className='shrink-0' | className='shrink-0' | ||||
| checked={isChecked} | checked={isChecked} | ||||
| {payload.source_url} | {payload.source_url} | ||||
| </div> | </div> | ||||
| </div> | </div> | ||||
| {showPreview && <Button | |||||
| size='small' | |||||
| onClick={onPreview} | |||||
| className='system-xs-medium-uppercase right-0 top-0 hidden px-1.5 group-hover:absolute group-hover:block' | |||||
| > | |||||
| {t('datasetCreation.stepOne.website.preview')} | |||||
| </Button>} | |||||
| </div> | </div> | ||||
| ) | ) | ||||
| } | } |
| 'use client' | 'use client' | ||||
| import React, { useCallback } from 'react' | |||||
| import React, { useCallback, useState } from 'react' | |||||
| import { useTranslation } from 'react-i18next' | import { useTranslation } from 'react-i18next' | ||||
| import cn from '@/utils/classnames' | import cn from '@/utils/classnames' | ||||
| import type { CrawlResultItem } from '@/models/datasets' | import type { CrawlResultItem } from '@/models/datasets' | ||||
| list: CrawlResultItem[] | list: CrawlResultItem[] | ||||
| checkedList: CrawlResultItem[] | checkedList: CrawlResultItem[] | ||||
| onSelectedChange: (selected: CrawlResultItem[]) => void | onSelectedChange: (selected: CrawlResultItem[]) => void | ||||
| onPreview?: (payload: CrawlResultItem) => void | |||||
| usedTime: number | usedTime: number | ||||
| } | } | ||||
| checkedList, | checkedList, | ||||
| onSelectedChange, | onSelectedChange, | ||||
| usedTime, | usedTime, | ||||
| onPreview, | |||||
| }: CrawledResultProps) => { | }: CrawledResultProps) => { | ||||
| const { t } = useTranslation() | const { t } = useTranslation() | ||||
| const [previewIndex, setPreviewIndex] = useState<number>(-1) | |||||
| const isCheckAll = checkedList.length === list.length | const isCheckAll = checkedList.length === list.length | ||||
| } | } | ||||
| }, [checkedList, onSelectedChange]) | }, [checkedList, onSelectedChange]) | ||||
| const handlePreview = useCallback((index: number) => { | |||||
| if (!onPreview) return | |||||
| setPreviewIndex(index) | |||||
| onPreview(list[index]) | |||||
| }, [list, onPreview]) | |||||
| return ( | return ( | ||||
| <div className={cn('flex flex-col gap-y-2', className)}> | <div className={cn('flex flex-col gap-y-2', className)}> | ||||
| <div className='system-sm-medium pt-2 text-text-primary'> | <div className='system-sm-medium pt-2 text-text-primary'> | ||||
| /> | /> | ||||
| </div> | </div> | ||||
| <div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'> | <div className='flex flex-col gap-y-px border-t border-divider-subtle bg-background-default-subtle p-2'> | ||||
| {list.map(item => ( | |||||
| {list.map((item, index) => ( | |||||
| <CrawledResultItem | <CrawledResultItem | ||||
| key={item.source_url} | key={item.source_url} | ||||
| payload={item} | payload={item} | ||||
| isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)} | isChecked={checkedList.some(checkedItem => checkedItem.source_url === item.source_url)} | ||||
| onCheckChange={handleItemCheckChange(item)} | onCheckChange={handleItemCheckChange(item)} | ||||
| isPreview={index === previewIndex} | |||||
| onPreview={handlePreview.bind(null, index)} | |||||
| showPreview={!!onPreview} | |||||
| /> | /> | ||||
| ))} | ))} | ||||
| </div> | </div> |
| datasourceProvider: DataSourceProvider | datasourceProvider: DataSourceProvider | ||||
| onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | ||||
| onJobIdChange: (jobId: string) => void | onJobIdChange: (jobId: string) => void | ||||
| onPreview?: (payload: CrawlResultItem) => void | |||||
| } | } | ||||
| enum Step { | enum Step { | ||||
| datasourceProvider, | datasourceProvider, | ||||
| onCheckedCrawlResultChange, | onCheckedCrawlResultChange, | ||||
| onJobIdChange, | onJobIdChange, | ||||
| onPreview, | |||||
| }: CrawlerProps) => { | }: CrawlerProps) => { | ||||
| const { t } = useTranslation() | const { t } = useTranslation() | ||||
| const [step, setStep] = useState<Step>(Step.init) | const [step, setStep] = useState<Step>(Step.init) | ||||
| checkedList={checkedCrawlResult} | checkedList={checkedCrawlResult} | ||||
| onSelectedChange={onCheckedCrawlResultChange} | onSelectedChange={onCheckedCrawlResultChange} | ||||
| usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} | usedTime={Number.parseFloat(crawlResult?.time_consuming as string) || 0} | ||||
| onPreview={onPreview} | |||||
| /> | /> | ||||
| )} | )} | ||||
| </div> | </div> |
| checkedCrawlResult: CrawlResultItem[] | checkedCrawlResult: CrawlResultItem[] | ||||
| onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | ||||
| onJobIdChange: (jobId: string) => void | onJobIdChange: (jobId: string) => void | ||||
| onPreview?: (payload: CrawlResultItem) => void | |||||
| } | } | ||||
| const FireCrawl = ({ | const FireCrawl = ({ | ||||
| checkedCrawlResult, | checkedCrawlResult, | ||||
| onCheckedCrawlResultChange, | onCheckedCrawlResultChange, | ||||
| onJobIdChange, | onJobIdChange, | ||||
| onPreview, | |||||
| }: FireCrawlProps) => { | }: FireCrawlProps) => { | ||||
| return ( | return ( | ||||
| <Crawler | <Crawler | ||||
| datasourceProvider={DataSourceProvider.fireCrawl} | datasourceProvider={DataSourceProvider.fireCrawl} | ||||
| onCheckedCrawlResultChange={onCheckedCrawlResultChange} | onCheckedCrawlResultChange={onCheckedCrawlResultChange} | ||||
| onJobIdChange={onJobIdChange} | onJobIdChange={onJobIdChange} | ||||
| onPreview={onPreview} | |||||
| /> | /> | ||||
| ) | ) | ||||
| } | } |
| checkedCrawlResult: CrawlResultItem[] | checkedCrawlResult: CrawlResultItem[] | ||||
| onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | ||||
| onJobIdChange: (jobId: string) => void | onJobIdChange: (jobId: string) => void | ||||
| onPreview?: (payload: CrawlResultItem) => void | |||||
| } | } | ||||
| const JinaReader = ({ | const JinaReader = ({ | ||||
| checkedCrawlResult, | checkedCrawlResult, | ||||
| onCheckedCrawlResultChange, | onCheckedCrawlResultChange, | ||||
| onJobIdChange, | onJobIdChange, | ||||
| onPreview, | |||||
| }: JinaReaderProps) => { | }: JinaReaderProps) => { | ||||
| return ( | return ( | ||||
| <Crawler | <Crawler | ||||
| datasourceProvider={DataSourceProvider.jinaReader} | datasourceProvider={DataSourceProvider.jinaReader} | ||||
| onCheckedCrawlResultChange={onCheckedCrawlResultChange} | onCheckedCrawlResultChange={onCheckedCrawlResultChange} | ||||
| onJobIdChange={onJobIdChange} | onJobIdChange={onJobIdChange} | ||||
| onPreview={onPreview} | |||||
| /> | /> | ||||
| ) | ) | ||||
| } | } |
| checkedCrawlResult: CrawlResultItem[] | checkedCrawlResult: CrawlResultItem[] | ||||
| onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | onCheckedCrawlResultChange: (payload: CrawlResultItem[]) => void | ||||
| onJobIdChange: (jobId: string) => void | onJobIdChange: (jobId: string) => void | ||||
| onPreview?: (payload: CrawlResultItem) => void | |||||
| } | } | ||||
| const WaterCrawl = ({ | const WaterCrawl = ({ | ||||
| checkedCrawlResult, | checkedCrawlResult, | ||||
| onCheckedCrawlResultChange, | onCheckedCrawlResultChange, | ||||
| onJobIdChange, | onJobIdChange, | ||||
| onPreview, | |||||
| }: WaterCrawlProps) => { | }: WaterCrawlProps) => { | ||||
| return ( | return ( | ||||
| <Crawler | <Crawler | ||||
| datasourceProvider={DataSourceProvider.jinaReader} | datasourceProvider={DataSourceProvider.jinaReader} | ||||
| onCheckedCrawlResultChange={onCheckedCrawlResultChange} | onCheckedCrawlResultChange={onCheckedCrawlResultChange} | ||||
| onJobIdChange={onJobIdChange} | onJobIdChange={onJobIdChange} | ||||
| onPreview={onPreview} | |||||
| /> | /> | ||||
| ) | ) | ||||
| } | } |
| processingDocuments: 'Processing Documents', | processingDocuments: 'Processing Documents', | ||||
| }, | }, | ||||
| backToDataSource: 'Data Source', | backToDataSource: 'Data Source', | ||||
| stepOne: { | |||||
| preview: 'Preview', | |||||
| }, | |||||
| characters: 'characters', | |||||
| }, | }, | ||||
| } | } | ||||
| processingDocuments: '正在处理文档', | processingDocuments: '正在处理文档', | ||||
| }, | }, | ||||
| backToDataSource: '数据源', | backToDataSource: '数据源', | ||||
| stepOne: { | |||||
| preview: '预览', | |||||
| }, | |||||
| characters: '字符', | |||||
| }, | }, | ||||
| } | } | ||||
| queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`), | queryFn: () => get<RelatedAppResponse>(`/datasets/${datasetId}/related-apps`), | ||||
| }) | }) | ||||
| } | } | ||||
| type NotionPagePreviewRequest = { | |||||
| workspaceID: string | |||||
| pageID: string | |||||
| pageType: string | |||||
| } | |||||
| type NotionPagePreviewResponse = { | |||||
| content: string | |||||
| } | |||||
| export const usePreviewNotionPage = (params: NotionPagePreviewRequest) => { | |||||
| const { workspaceID, pageID, pageType } = params | |||||
| return useQuery({ | |||||
| queryKey: [NAME_SPACE, 'preview-notion-page'], | |||||
| queryFn: () => get<NotionPagePreviewResponse>(`notion/workspaces/${workspaceID}/pages/${pageID}/${pageType}/preview`), | |||||
| enabled: !!workspaceID && !!pageID && !!pageType, | |||||
| }) | |||||
| } |
| }), | }), | ||||
| }) | }) | ||||
| } | } | ||||
| type FilePreviewResponse = { | |||||
| content: string | |||||
| } | |||||
| export const useFilePreview = (fileID: string) => { | |||||
| return useQuery<FilePreviewResponse>({ | |||||
| queryKey: [NAME_SPACE, 'file-preview', fileID], | |||||
| queryFn: () => get<FilePreviewResponse>(`/files/${fileID}/preview`), | |||||
| enabled: !!fileID, | |||||
| }) | |||||
| } |
| queryFn: () => { | queryFn: () => { | ||||
| return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`) | return get<PublishedPipelineInfoResponse>(`/rag/pipelines/${pipelineId}/workflows/publish`) | ||||
| }, | }, | ||||
| enabled: !!pipelineId, | |||||
| }) | }) | ||||
| } | } |
| a.remove() | a.remove() | ||||
| window.URL.revokeObjectURL(url) | window.URL.revokeObjectURL(url) | ||||
| } | } | ||||
| /** | |||||
| * Formats a number into a readable string using "k", "M", or "B" suffix. | |||||
| * @example | |||||
| * 950 => "950" | |||||
| * 1200 => "1.2k" | |||||
| * 1500000 => "1.5M" | |||||
| * 2000000000 => "2B" | |||||
| * | |||||
| * @param {number} num - The number to format | |||||
| * @returns {string} - The formatted number string | |||||
| */ | |||||
| export const formatNumberAbbreviated = (num: number) => { | |||||
| // If less than 1000, return as-is | |||||
| if (num < 1000) return num.toString() | |||||
| // Define thresholds and suffixes | |||||
| const units = [ | |||||
| { value: 1e9, symbol: 'B' }, | |||||
| { value: 1e6, symbol: 'M' }, | |||||
| { value: 1e3, symbol: 'k' }, | |||||
| ] | |||||
| for (let i = 0; i < units.length; i++) { | |||||
| if (num >= units[i].value) { | |||||
| const formatted = (num / units[i].value).toFixed(1) | |||||
| return formatted.endsWith('.0') | |||||
| ? `${Number.parseInt(formatted)}${units[i].symbol}` | |||||
| : `${formatted}${units[i].symbol}` | |||||
| } | |||||
| } | |||||
| } |