fix: Change the data in the dataset page to be obtained using the interface (#8726)

### What problem does this PR solve?

Change the data in the dataset page to be obtained using the interface,
and change the import to obtain all data every 15 seconds to obtain the
data of the current page every 5 seconds when parsing the existing file.
[#3221](https://github.com/infiniflow/ragflow/issues/3221)

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)

web/src/components/list-filter-bar/filter-popover.tsx

@@ -24,6 +24,7 @@ export type CheckboxFormMultipleProps = {
   filters?: FilterCollection[];
   value?: FilterValue;
   onChange?: FilterChange;
+  onOpenChange?: (open: boolean) => void;
   setOpen(open: boolean): void;
 };
 
@@ -148,12 +149,19 @@ export function FilterPopover({
   children,
   value,
   onChange,
+  onOpenChange,
   filters,
 }: PropsWithChildren & Omit<CheckboxFormMultipleProps, 'setOpen'>) {
   const [open, setOpen] = useState(false);
-
+  const onOpenChangeFun = useCallback(
+    (e: boolean) => {
+      onOpenChange?.(e);
+      setOpen(e);
+    },
+    [onOpenChange],
+  );
   return (
-    <Popover open={open} onOpenChange={setOpen}>
+    <Popover open={open} onOpenChange={onOpenChangeFun}>
       <PopoverTrigger asChild>{children}</PopoverTrigger>
       <PopoverContent className="p-0">
         <CheckboxFormMultiple

web/src/components/list-filter-bar/index.tsx

@@ -52,6 +52,7 @@ export default function ListFilterBar({
   leftPanel,
   value,
   onChange,
+  onOpenChange,
   filters,
   className,
   icon,
@@ -79,7 +80,12 @@ export default function ListFilterBar({
       </div>
       <div className="flex gap-5 items-center">
         {showFilter && (
-          <FilterPopover value={value} onChange={onChange} filters={filters}>
+          <FilterPopover
+            value={value}
+            onChange={onChange}
+            filters={filters}
+            onOpenChange={onOpenChange}
+          >
             <FilterButton count={filterCount}></FilterButton>
           </FilterPopover>
         )}

web/src/hooks/use-document-request.ts

@@ -1,5 +1,8 @@
 import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit';
-import { IDocumentInfo } from '@/interfaces/database/document';
+import {
+  IDocumentInfo,
+  IDocumentInfoFilter,
+} from '@/interfaces/database/document';
 import {
   IChangeParserConfigRequestBody,
   IDocumentMetaRequestBody,
@@ -10,7 +13,7 @@ import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { useDebounce } from 'ahooks';
 import { message } from 'antd';
 import { get } from 'lodash';
-import { useCallback } from 'react';
+import { useCallback, useMemo, useState } from 'react';
 import { useParams } from 'umi';
 import {
   useGetPaginationWithRouter,
@@ -30,7 +33,7 @@ export const enum DocumentApiAction {
   SaveDocumentName = 'saveDocumentName',
   SetDocumentParser = 'setDocumentParser',
   SetDocumentMeta = 'setDocumentMeta',
-  FetchAllDocumentList = 'fetchAllDocumentList',
+  FetchDocumentFilter = 'fetchDocumentFilter',
   CreateDocument = 'createDocument',
 }
 
@@ -81,6 +84,10 @@ export const useFetchDocumentList = () => {
   const { id } = useParams();
   const debouncedSearchString = useDebounce(searchString, { wait: 500 });
   const { filterValue, handleFilterSubmit } = useHandleFilterSubmit();
+  const [docs, setDocs] = useState<IDocumentInfo[]>([]);
+  const isLoop = useMemo(() => {
+    return docs.some((doc) => doc.run === '1');
+  }, [docs]);
 
   const { data, isFetching: loading } = useQuery<{
     docs: IDocumentInfo[];
@@ -93,7 +100,7 @@ export const useFetchDocumentList = () => {
       filterValue,
     ],
     initialData: { docs: [], total: 0 },
-    // refetchInterval: 15000,
+    refetchInterval: isLoop ? 5000 : false,
     enabled: !!knowledgeId || !!id,
     queryFn: async () => {
       const ret = await listDocument(
@@ -104,7 +111,7 @@ export const useFetchDocumentList = () => {
           page: pagination.current,
         },
         {
-          types: filterValue.type,
+          suffix: filterValue.type,
           run_status: filterValue.run,
         },
       );
@@ -118,7 +125,9 @@ export const useFetchDocumentList = () => {
       };
     },
   });
-
+  useMemo(() => {
+    setDocs(data.docs);
+  }, [data.docs]);
   const onInputChange: React.ChangeEventHandler<HTMLInputElement> = useCallback(
     (e) => {
       setPagination({ page: 1 });
@@ -139,34 +148,48 @@ export const useFetchDocumentList = () => {
   };
 };
 
-export function useFetchAllDocumentList() {
+// get document filter
+export const useGetDocumentFilter = (): {
+  filter: IDocumentInfoFilter;
+  onOpenChange: (open: boolean) => void;
+} => {
+  const { knowledgeId } = useGetKnowledgeSearchParams();
+  const { searchString } = useHandleSearchChange();
   const { id } = useParams();
-  const { data, isFetching: loading } = useQuery<{
-    docs: IDocumentInfo[];
-    total: number;
-  }>({
-    queryKey: [DocumentApiAction.FetchAllDocumentList],
-    initialData: { docs: [], total: 0 },
-    refetchInterval: 15000,
-    enabled: !!id,
+  const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+  const [open, setOpen] = useState<number>(0);
+  const { data } = useQuery({
+    queryKey: [
+      DocumentApiAction.FetchDocumentFilter,
+      debouncedSearchString,
+      knowledgeId,
+      open,
+    ],
     queryFn: async () => {
-      const ret = await listDocument({
-        kb_id: id,
+      const { data } = await kbService.documentFilter({
+        kb_id: knowledgeId || id,
+        keywords: debouncedSearchString,
       });
-      if (ret.data.code === 0) {
-        return ret.data.data;
+      if (data.code === 0) {
+        return data.data;
       }
-
-      return {
-        docs: [],
-        total: 0,
-      };
     },
   });
-
-  return { data, loading };
-}
-
+  const handleOnpenChange = (e: boolean) => {
+    if (e) {
+      const currentOpen = open + 1;
+      setOpen(currentOpen);
+    }
+  };
+  return {
+    filter: data?.filter || {
+      run_status: {},
+      suffix: {},
+    },
+    onOpenChange: handleOnpenChange,
+  };
+};
+// update document status
 export const useSetDocumentStatus = () => {
   const queryClient = useQueryClient();
 
@@ -200,6 +223,7 @@ export const useSetDocumentStatus = () => {
   return { setDocumentStatus: mutateAsync, data, loading };
 };
 
+// This hook is used to run a document by its IDs
 export const useRunDocument = () => {
   const queryClient = useQueryClient();
 

web/src/interfaces/database/document.ts

@@ -47,3 +47,8 @@ interface GraphRag {
   resolution?: boolean;
   use_graphrag?: boolean;
 }
+
+export type IDocumentInfoFilter = {
+  run_status: Record<number, number>;
+  suffix: Record<string, number>;
+};

web/src/interfaces/request/knowledge.ts

@@ -21,6 +21,6 @@ export interface IFetchKnowledgeListRequestParams {
 }
 
 export interface IFetchDocumentListRequestBody {
-  types?: string[];
+  suffix?: string[];
   run_status?: string[];
 }

web/src/locales/en.ts

@@ -407,6 +407,11 @@ This auto-tagging feature enhances retrieval by adding another layer of domain-s
       mind: 'Mind map',
       question: 'Question',
       questionTip: `If there are given questions, the embedding of the chunk will be based on them.`,
+      chunkResult: 'Chunk Result',
+      chunkResultTip: `View the chunked segments used for embedding and retrieval.`,
+      enable: 'Enable',
+      disable: 'Disable',
+      delete: 'Delete',
     },
     chat: {
       newConversation: 'New conversation',

web/src/locales/zh-traditional.ts

@@ -396,6 +396,11 @@ export default {
       mind: '心智圖',
       question: '問題',
       questionTip: `如果存在給定的問題，則區塊的嵌入將基於它們。`,
+      chunkResult: '切片結果',
+      chunkResultTip: `查看用於嵌入和召回的切片段落`,
+      enable: '啟用',
+      disable: '禁用',
+      delete: '删除',
     },
     chat: {
       newConversation: '新會話',

web/src/locales/zh.ts

@@ -414,6 +414,11 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       mind: '思维导图',
       question: '问题',
       questionTip: `如果有给定的问题，则块的嵌入将基于它们。`,
+      chunkResult: '切片结果',
+      chunkResultTip: `查看用于嵌入和召回的切片段落。`,
+      enable: '启用',
+      disable: '禁用',
+      delete: '删除',
     },
     chat: {
       newConversation: '新会话',

web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.less

@@ -24,11 +24,13 @@
 
 .chunkCard {
   width: 100%;
+  padding: 18px 10px;
 }
 
 .cardSelected {
   background-color: @selectedBackgroundColor;
 }
+
 .cardSelectedDark {
   background-color: #ffffff2f;
 }

web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-card/index.tsx

@@ -1,11 +1,18 @@
 import Image from '@/components/image';
+import { useTheme } from '@/components/theme-provider';
+import { Card } from '@/components/ui/card';
+import { Checkbox } from '@/components/ui/checkbox';
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from '@/components/ui/popover';
+import { Switch } from '@/components/ui/switch';
 import { IChunk } from '@/interfaces/database/knowledge';
-import { Card, Checkbox, CheckboxProps, Flex, Popover, Switch } from 'antd';
+import { CheckedState } from '@radix-ui/react-checkbox';
 import classNames from 'classnames';
 import DOMPurify from 'dompurify';
 import { useEffect, useState } from 'react';
-
-import { useTheme } from '@/components/theme-provider';
 import { ChunkTextMode } from '../../constant';
 import styles from './index.less';
 
@@ -39,8 +46,8 @@ const ChunkCard = ({
     switchChunk(available === 0 ? 1 : 0, [item.chunk_id]);
   };
 
-  const handleCheck: CheckboxProps['onChange'] = (e) => {
-    handleCheckboxClick(item.chunk_id, e.target.checked);
+  const handleCheck = (e: CheckedState) => {
+    handleCheckboxClick(item.chunk_id, e === 'indeterminate' ? false : e);
   };
 
   const handleContentDoubleClick = () => {
@@ -54,7 +61,7 @@ const ChunkCard = ({
   useEffect(() => {
     setEnabled(available === 1);
   }, [available]);
-
+  const [open, setOpen] = useState<boolean>(false);
   return (
     <Card
       className={classNames(styles.chunkCard, {
@@ -62,19 +69,34 @@ const ChunkCard = ({
           selected,
       })}
     >
-      <Flex gap={'middle'} justify={'space-between'}>
-        <Checkbox onChange={handleCheck} checked={checked}></Checkbox>
+      <div className="flex items-start justify-between gap-2">
+        <Checkbox onCheckedChange={handleCheck} checked={checked}></Checkbox>
         {item.image_id && (
-          <Popover
-            placement="right"
-            content={
-              <Image id={item.image_id} className={styles.imagePreview}></Image>
-            }
-          >
-            <Image id={item.image_id} className={styles.image}></Image>
+          <Popover open={open}>
+            <PopoverTrigger
+              asChild
+              onMouseEnter={() => setOpen(true)}
+              onMouseLeave={() => setOpen(false)}
+            >
+              <div>
+                <Image id={item.image_id} className={styles.image}></Image>
+              </div>
+            </PopoverTrigger>
+            <PopoverContent
+              className="p-0"
+              align={'start'}
+              side={'right'}
+              sideOffset={-20}
+            >
+              <div>
+                <Image
+                  id={item.image_id}
+                  className={styles.imagePreview}
+                ></Image>
+              </div>
+            </PopoverContent>
           </Popover>
         )}
-
         <section
           onDoubleClick={handleContentDoubleClick}
           onClick={handleContentClick}
@@ -89,11 +111,15 @@ const ChunkCard = ({
             })}
           ></div>
         </section>
-
         <div>
-          <Switch checked={enabled} onChange={onChange} />
+          <Switch
+            checked={enabled}
+            onCheckedChange={onChange}
+            aria-readonly
+            className="!m-0"
+          />
         </div>
-      </Flex>
+      </div>
     </Card>
   );
 };

web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/components/chunk-result-bar/checkbox-sets.tsx

@@ -2,6 +2,7 @@ import { Checkbox } from '@/components/ui/checkbox';
 import { Label } from '@/components/ui/label';
 import { Ban, CircleCheck, Trash2 } from 'lucide-react';
 import { useCallback } from 'react';
+import { useTranslation } from 'react-i18next';
 
 type ICheckboxSetProps = {
   selectAllChunk: (e: any) => void;
@@ -11,6 +12,7 @@ type ICheckboxSetProps = {
 };
 export default (props: ICheckboxSetProps) => {
   const { selectAllChunk, removeChunk, switchChunk, checked } = props;
+  const { t } = useTranslation();
   const handleSelectAllCheck = useCallback(
     (e: any) => {
       console.log('eee=', e);
@@ -33,35 +35,35 @@ export default (props: ICheckboxSetProps) => {
 
   return (
     <div className="flex gap-[40px] p-4">
-      <div className="flex items-center gap-3 cursor-pointer">
+      <div className="flex items-center gap-3 cursor-pointer text-muted-foreground hover:text-white">
         <Checkbox
           id="all_chunks_checkbox"
           onCheckedChange={handleSelectAllCheck}
           checked={checked}
-          className=" data-[state=checked]:bg-[#1668dc] data-[state=checked]:border-[#1668dc] data-[state=checked]:text-white"
+          className=" data-[state=checked]:bg-white data-[state=checked]:border-white data-[state=checked]:text-black  border-muted-foreground text-muted-foreground hover:text-black hover:border-white "
         />
-        <Label htmlFor="all_chunks_checkbox">All Chunks</Label>
+        <Label htmlFor="all_chunks_checkbox">{t('chunk.selectAll')}</Label>
       </div>
       <div
-        className="flex items-center cursor-pointer"
+        className="flex items-center cursor-pointer text-muted-foreground hover:text-white"
         onClick={handleEnabledClick}
       >
         <CircleCheck size={16} />
-        <span className="block ml-1">Enable</span>
+        <span className="block ml-1">{t('chunk.enable')}</span>
       </div>
       <div
-        className="flex items-center cursor-pointer"
+        className="flex items-center cursor-pointer text-muted-foreground hover:text-white"
         onClick={handleDisabledClick}
       >
         <Ban size={16} />
-        <span className="block ml-1">Disable</span>
+        <span className="block ml-1">{t('chunk.disable')}</span>
       </div>
       <div
-        className="flex items-center text-red-500 cursor-pointer"
+        className="flex items-center cursor-pointer text-red-400 hover:text-red-500"
         onClick={handleDeleteClick}
       >
         <Trash2 size={16} />
-        <span className="block ml-1">Delete</span>
+        <span className="block ml-1">{t('chunk.delete')}</span>
       </div>
     </div>
   );

web/src/pages/chunk/parsed-result/add-knowledge/components/knowledge-chunk/index.tsx

@@ -183,9 +183,9 @@ const Chunk = () => {
             <Spin spinning={loading} className={styles.spin} size="large">
               <div className="h-[100px] flex flex-col justify-end pb-[5px]">
                 <div>
-                  <h2 className="text-[24px]">Chunk Result</h2>
+                  <h2 className="text-[24px]">{t('chunk.chunkResult')}</h2>
                   <div className="text-[14px] text-[#979AAB]">
-                    View the chunked segments used for embedding and retrieval.
+                    {t('chunk.chunkResultTip')}
                   </div>
                 </div>
               </div>

web/src/pages/dataset/dataset/index.tsx

@@ -12,9 +12,7 @@ import {
 } from '@/components/ui/dropdown-menu';
 import { useRowSelection } from '@/hooks/logic-hooks/use-row-selection';
 import { useFetchDocumentList } from '@/hooks/use-document-request';
-import { IDocumentInfo } from '@/interfaces/database/document';
 import { Upload } from 'lucide-react';
-import { useMemo, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { DatasetTable } from './dataset-table';
 import { useBulkOperateDataset } from './use-bulk-operate-dataset';
@@ -42,16 +40,7 @@ export default function Dataset() {
     handleFilterSubmit,
     loading,
   } = useFetchDocumentList();
-  const { filters, documents: filteredDocuments } = useSelectDatasetFilters();
-  const [datasetInfo, setDatasetInfo] = useState<IDocumentInfo[]>(documents);
-
-  useMemo(() => {
-    setDatasetInfo(documents);
-  }, [documents]);
-
-  useMemo(() => {
-    setDatasetInfo(filteredDocuments);
-  }, [filteredDocuments]);
+  const { filters, onOpenChange } = useSelectDatasetFilters();
 
   const {
     createLoading,
@@ -69,7 +58,6 @@ export default function Dataset() {
     rowSelection,
     setRowSelection,
   });
-
   return (
     <section className="p-5">
       <ListFilterBar
@@ -78,13 +66,13 @@ export default function Dataset() {
         searchString={searchString}
         value={filterValue}
         onChange={handleFilterSubmit}
+        onOpenChange={onOpenChange}
         filters={filters}
         leftPanel={
           <div className="items-start">
-            <div className="pb-1">Dataset</div>
+            <div className="pb-1">{t('knowledgeDetails.dataset')}</div>
             <div className="text-text-sub-title-invert text-sm">
-              Please wait for your files to finish parsing before starting an
-              AI-powered chat.
+              {t('knowledgeDetails.datasetDescription')}
             </div>
           </div>
         }
@@ -111,7 +99,7 @@ export default function Dataset() {
         <BulkOperateBar list={list} count={selectedCount}></BulkOperateBar>
       )}
       <DatasetTable
-        documents={datasetInfo}
+        documents={documents}
         pagination={pagination}
         setPagination={setPagination}
         rowSelection={rowSelection}

web/src/pages/dataset/dataset/use-select-filters.ts

@@ -1,30 +1,36 @@
-import { useFetchAllDocumentList } from '@/hooks/use-document-request';
-import { groupListByType } from '@/utils/dataset-util';
+import { FilterCollection } from '@/components/list-filter-bar/interface';
+import { useTranslate } from '@/hooks/common-hooks';
+import { useGetDocumentFilter } from '@/hooks/use-document-request';
 import { useMemo } from 'react';
-import { useTranslation } from 'react-i18next';
 
 export function useSelectDatasetFilters() {
-  const {
-    data: { docs: documents },
-  } = useFetchAllDocumentList();
-  const { t } = useTranslation();
+  const { t } = useTranslate('knowledgeDetails');
+  const { filter, onOpenChange } = useGetDocumentFilter();
 
   const fileTypes = useMemo(() => {
-    return groupListByType(documents, 'type', 'type');
-  }, [documents]);
-
+    if (filter.suffix) {
+      return Object.keys(filter.suffix).map((x) => ({
+        id: x,
+        label: x.toUpperCase(),
+        count: filter.suffix[x],
+      }));
+    }
+  }, [filter.suffix]);
   const fileStatus = useMemo(() => {
-    return groupListByType(documents, 'run', 'run').map((x) => ({
-      ...x,
-      label: t(`knowledgeDetails.runningStatus${x.label}`),
-    }));
-  }, [documents, t]);
-  const filters = useMemo(() => {
+    if (filter.run_status) {
+      return Object.keys(filter.run_status).map((x) => ({
+        id: x,
+        label: t(`runningStatus${x}`),
+        count: filter.run_status[x as unknown as number],
+      }));
+    }
+  }, [filter.run_status, t]);
+  const filters: FilterCollection[] = useMemo(() => {
     return [
       { field: 'type', label: 'File Type', list: fileTypes },
       { field: 'run', label: 'Status', list: fileStatus },
-    ];
+    ] as FilterCollection[];
   }, [fileStatus, fileTypes]);
 
-  return { fileTypes, fileStatus, filters, documents };
+  return { filters, onOpenChange };
 }

web/src/pages/dataset/testing/index.tsx

@@ -35,9 +35,8 @@ export default function RetrievalTesting() {
     <div className="p-5">
       <section className="flex justify-between items-center">
         <TopTitle
-          title={'Configuration'}
-          description={`  Update your knowledge base configuration here, particularly the chunk
-                  method.`}
+          title={'Retrieval testing'}
+          description={`Conduct a retrieval test to check if RAGFlow can recover the intended content for the LLM.`}
         ></TopTitle>
         {/* <Button>Save as Preset</Button> */}
       </section>

web/src/services/knowledge-service.ts

@@ -155,6 +155,10 @@ const methods = {
     url: listTagByKnowledgeIds,
     method: 'get',
   },
+  documentFilter: {
+    url: api.get_dataset_filter,
+    method: 'post',
+  },
 };
 
 const kbService = registerServer<keyof typeof methods>(methods, request);
@@ -188,4 +192,7 @@ export const listDocument = (
   body?: IFetchDocumentListRequestBody,
 ) => request.post(api.get_document_list, { data: body || {}, params });
 
+export const documentFilter = (kb_id: string) =>
+  request.post(api.get_dataset_filter, { kb_id });
+
 export default kbService;

web/src/utils/api.ts

@@ -78,6 +78,7 @@ export default {
   upload_and_parse: `${api_host}/document/upload_and_parse`,
   parse: `${api_host}/document/parse`,
   setMeta: `${api_host}/document/set_meta`,
+  get_dataset_filter: `${api_host}/document/filter`,
 
   // chat
   setDialog: `${api_host}/dialog/set`,