Feat: Filter document by running status and file type. #3221 (#7340)

### What problem does this PR solve?
Feat: Filter document by running status and file type. #3221

### Type of change


- [x] New Feature (non-breaking change which adds functionality)

web/src/components/chunk-method-dialog/index.tsx

@@ -281,7 +281,10 @@ export function ChunkMethodDialog({
                 )}
               />
             )}
-            <DatasetConfigurationContainer show={showOne || showMaxTokenNumber}>
+            <DatasetConfigurationContainer
+              show={showOne || showMaxTokenNumber}
+              className="space-y-3"
+            >
               {showOne && <LayoutRecognizeFormField></LayoutRecognizeFormField>}
               {showMaxTokenNumber && (
                 <>
@@ -298,6 +301,7 @@ export function ChunkMethodDialog({
             </DatasetConfigurationContainer>
             <DatasetConfigurationContainer
               show={showAutoKeywords(selectedTag) || showExcelToHtml}
+              className="space-y-3"
             >
               {showAutoKeywords(selectedTag) && (
                 <>

web/src/components/list-filter-bar/filter-popover.tsx

@@ -0,0 +1,163 @@
+import {
+  Popover,
+  PopoverContent,
+  PopoverTrigger,
+} from '@/components/ui/popover';
+import { zodResolver } from '@hookform/resolvers/zod';
+import { PropsWithChildren, useCallback, useEffect } from 'react';
+import { useForm } from 'react-hook-form';
+import { ZodArray, ZodString, z } from 'zod';
+
+import { Button } from '@/components/ui/button';
+import { Checkbox } from '@/components/ui/checkbox';
+import {
+  Form,
+  FormControl,
+  FormField,
+  FormItem,
+  FormLabel,
+  FormMessage,
+} from '@/components/ui/form';
+import { FilterChange, FilterCollection, FilterValue } from './interface';
+
+export type CheckboxFormMultipleProps = {
+  filters?: FilterCollection[];
+  value?: FilterValue;
+  onChange?: FilterChange;
+};
+
+function CheckboxFormMultiple({
+  filters = [],
+  value,
+  onChange,
+}: CheckboxFormMultipleProps) {
+  const fieldsDict = filters?.reduce<Record<string, Array<any>>>((pre, cur) => {
+    pre[cur.field] = [];
+    return pre;
+  }, {});
+
+  const FormSchema = z.object(
+    filters.reduce<Record<string, ZodArray<ZodString, 'many'>>>((pre, cur) => {
+      pre[cur.field] = z.array(z.string());
+
+      // .refine((value) => value.some((item) => item), {
+      //   message: 'You have to select at least one item.',
+      // });
+      return pre;
+    }, {}),
+  );
+
+  const form = useForm<z.infer<typeof FormSchema>>({
+    resolver: zodResolver(FormSchema),
+    defaultValues: fieldsDict,
+  });
+
+  function onSubmit(data: z.infer<typeof FormSchema>) {
+    console.log('🚀 ~ onSubmit ~ data:', data);
+    // setOwnerIds(data.items);
+    onChange?.(data);
+  }
+
+  const onReset = useCallback(() => {
+    onChange?.(fieldsDict);
+  }, [fieldsDict, onChange]);
+
+  useEffect(() => {
+    form.reset(value);
+  }, [form, value]);
+
+  return (
+    <Form {...form}>
+      <form
+        onSubmit={form.handleSubmit(onSubmit)}
+        className="space-y-8"
+        onReset={() => form.reset()}
+      >
+        {filters.map((x) => (
+          <FormField
+            key={x.field}
+            control={form.control}
+            name={x.field}
+            render={() => (
+              <FormItem>
+                <div className="mb-4">
+                  <FormLabel className="text-base">{x.label}</FormLabel>
+                </div>
+                {x.list.map((item) => (
+                  <FormField
+                    key={item.id}
+                    control={form.control}
+                    name={x.field}
+                    render={({ field }) => {
+                      return (
+                        <div className="flex items-center justify-between">
+                          <FormItem
+                            key={item.id}
+                            className="flex flex-row  space-x-3 space-y-0 items-center"
+                          >
+                            <FormControl>
+                              <Checkbox
+                                checked={field.value?.includes(item.id)}
+                                onCheckedChange={(checked) => {
+                                  return checked
+                                    ? field.onChange([...field.value, item.id])
+                                    : field.onChange(
+                                        field.value?.filter(
+                                          (value) => value !== item.id,
+                                        ),
+                                      );
+                                }}
+                              />
+                            </FormControl>
+                            <FormLabel className="text-lg">
+                              {item.label}
+                            </FormLabel>
+                          </FormItem>
+                          <span className=" text-sm">{item.count}</span>
+                        </div>
+                      );
+                    }}
+                  />
+                ))}
+                <FormMessage />
+              </FormItem>
+            )}
+          />
+        ))}
+        <div className="flex justify-between">
+          <Button
+            type="button"
+            variant={'outline'}
+            size={'sm'}
+            onClick={onReset}
+          >
+            Clear
+          </Button>
+          <Button type="submit" size={'sm'}>
+            Submit
+          </Button>
+        </div>
+      </form>
+    </Form>
+  );
+}
+
+export function FilterPopover({
+  children,
+  value,
+  onChange,
+  filters,
+}: PropsWithChildren & CheckboxFormMultipleProps) {
+  return (
+    <Popover>
+      <PopoverTrigger asChild>{children}</PopoverTrigger>
+      <PopoverContent>
+        <CheckboxFormMultiple
+          onChange={onChange}
+          value={value}
+          filters={filters}
+        ></CheckboxFormMultiple>
+      </PopoverContent>
+    </Popover>
+  );
+}

web/src/components/list-filter-bar.tsx → web/src/components/list-filter-bar/index.tsx

@@ -1,19 +1,18 @@
 import { ChevronDown } from 'lucide-react';
 import React, {
   ChangeEventHandler,
-  FunctionComponent,
   PropsWithChildren,
   ReactNode,
+  useMemo,
 } from 'react';
-import { Button, ButtonProps } from './ui/button';
-import { SearchInput } from './ui/input';
+import { Button, ButtonProps } from '../ui/button';
+import { SearchInput } from '../ui/input';
+import { CheckboxFormMultipleProps, FilterPopover } from './filter-popover';
 
 interface IProps {
   title?: string;
-  FilterPopover?: FunctionComponent<any>;
   searchString?: string;
   onSearchChange?: ChangeEventHandler<HTMLInputElement>;
-  count?: number;
   showFilter?: boolean;
   leftPanel?: ReactNode;
 }
@@ -32,25 +31,31 @@ const FilterButton = React.forwardRef<
 export default function ListFilterBar({
   title,
   children,
-  FilterPopover,
   searchString,
   onSearchChange,
-  count,
   showFilter = true,
   leftPanel,
-}: PropsWithChildren<IProps>) {
+  value,
+  onChange,
+  filters,
+}: PropsWithChildren<IProps & CheckboxFormMultipleProps>) {
+  const filterCount = useMemo(() => {
+    return typeof value === 'object' && value !== null
+      ? Object.values(value).reduce((pre, cur) => {
+          return pre + cur.length;
+        }, 0)
+      : 0;
+  }, [value]);
+
   return (
     <div className="flex justify-between mb-6 items-center">
       <span className="text-3xl font-bold ">{leftPanel || title}</span>
       <div className="flex gap-4 items-center">
-        {showFilter &&
-          (FilterPopover ? (
-            <FilterPopover>
-              <FilterButton count={count}></FilterButton>
-            </FilterPopover>
-          ) : (
-            <FilterButton></FilterButton>
-          ))}
+        {showFilter && (
+          <FilterPopover value={value} onChange={onChange} filters={filters}>
+            <FilterButton count={filterCount}></FilterButton>
+          </FilterPopover>
+        )}
 
         <SearchInput
           value={searchString}

web/src/components/list-filter-bar/interface.ts

@@ -0,0 +1,15 @@
+export type FilterType = {
+  id: string;
+  label: string;
+  count: number;
+};
+
+export type FilterCollection = {
+  field: string;
+  label: string;
+  list: FilterType[];
+};
+
+export type FilterValue = Record<string, Array<string>>;
+
+export type FilterChange = (value: FilterValue) => void;

web/src/components/list-filter-bar/use-handle-filter-submit.ts

@@ -0,0 +1,12 @@
+import { useCallback, useState } from 'react';
+import { FilterChange, FilterValue } from './interface';
+
+export function useHandleFilterSubmit() {
+  const [filterValue, setFilterValue] = useState<FilterValue>({});
+
+  const handleFilterSubmit: FilterChange = useCallback((value) => {
+    setFilterValue(value);
+  }, []);
+
+  return { filterValue, setFilterValue, handleFilterSubmit };
+}

web/src/components/parse-configuration/raptor-form-fields.tsx

@@ -76,7 +76,7 @@ const RaptorFormFields = () => {
         )}
       />
       {useRaptor && (
-        <>
+        <div className="space-y-3">
           <FormField
             control={form.control}
             name={'parser_config.raptor.prompt'}
@@ -137,7 +137,7 @@ const RaptorFormFields = () => {
               </FormItem>
             )}
           />
-        </>
+        </div>
       )}
     </>
   );

web/src/hooks/document-hooks.ts

@@ -7,7 +7,7 @@ import {
 } from '@/interfaces/request/document';
 import i18n from '@/locales/config';
 import chatService from '@/services/chat-service';
-import kbService from '@/services/knowledge-service';
+import kbService, { listDocument } from '@/services/knowledge-service';
 import api, { api_host } from '@/utils/api';
 import { buildChunkHighlights } from '@/utils/document-util';
 import { post } from '@/utils/request';
@@ -73,7 +73,7 @@ export const useFetchNextDocumentList = () => {
     refetchInterval: 15000,
     enabled: !!knowledgeId || !!id,
     queryFn: async () => {
-      const ret = await kbService.get_document_list({
+      const ret = await listDocument({
         kb_id: knowledgeId || id,
         keywords: searchString,
         page_size: pagination.pageSize,

web/src/hooks/use-document-request.ts

@@ -1,10 +1,11 @@
+import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit';
 import { IDocumentInfo } from '@/interfaces/database/document';
 import {
   IChangeParserConfigRequestBody,
   IDocumentMetaRequestBody,
 } from '@/interfaces/request/document';
 import i18n from '@/locales/config';
-import kbService from '@/services/knowledge-service';
+import kbService, { listDocument } from '@/services/knowledge-service';
 import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
 import { useDebounce } from 'ahooks';
 import { message } from 'antd';
@@ -26,6 +27,7 @@ export const enum DocumentApiAction {
   SaveDocumentName = 'saveDocumentName',
   SetDocumentParser = 'setDocumentParser',
   SetDocumentMeta = 'setDocumentMeta',
+  FetchAllDocumentList = 'fetchAllDocumentList',
 }
 
 export const useUploadNextDocument = () => {
@@ -74,6 +76,7 @@ export const useFetchDocumentList = () => {
   const { pagination, setPagination } = useGetPaginationWithRouter();
   const { id } = useParams();
   const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+  const { filterValue, handleFilterSubmit } = useHandleFilterSubmit();
 
   const { data, isFetching: loading } = useQuery<{
     docs: IDocumentInfo[];
@@ -83,17 +86,24 @@ export const useFetchDocumentList = () => {
       DocumentApiAction.FetchDocumentList,
       debouncedSearchString,
       pagination,
+      filterValue,
     ],
     initialData: { docs: [], total: 0 },
     refetchInterval: 15000,
     enabled: !!knowledgeId || !!id,
     queryFn: async () => {
-      const ret = await kbService.get_document_list({
-        kb_id: knowledgeId || id,
-        keywords: debouncedSearchString,
-        page_size: pagination.pageSize,
-        page: pagination.current,
-      });
+      const ret = await listDocument(
+        {
+          kb_id: knowledgeId || id,
+          keywords: debouncedSearchString,
+          page_size: pagination.pageSize,
+          page: pagination.current,
+        },
+        {
+          types: filterValue.type,
+          run_status: filterValue.run,
+        },
+      );
       if (ret.data.code === 0) {
         return ret.data.data;
       }
@@ -120,9 +130,39 @@ export const useFetchDocumentList = () => {
     pagination: { ...pagination, total: data?.total },
     handleInputChange: onInputChange,
     setPagination,
+    filterValue,
+    handleFilterSubmit,
   };
 };
 
+export function useFetchAllDocumentList() {
+  const { id } = useParams();
+  const { data, isFetching: loading } = useQuery<{
+    docs: IDocumentInfo[];
+    total: number;
+  }>({
+    queryKey: [DocumentApiAction.FetchAllDocumentList],
+    initialData: { docs: [], total: 0 },
+    refetchInterval: 15000,
+    enabled: !!id,
+    queryFn: async () => {
+      const ret = await listDocument({
+        kb_id: id,
+      });
+      if (ret.data.code === 0) {
+        return ret.data.data;
+      }
+
+      return {
+        docs: [],
+        total: 0,
+      };
+    },
+  });
+
+  return { data, loading };
+}
+
 export const useSetDocumentStatus = () => {
   const queryClient = useQueryClient();
 

web/src/hooks/use-knowledge-request.ts

@@ -1,3 +1,4 @@
+import { useHandleFilterSubmit } from '@/components/list-filter-bar/use-handle-filter-submit';
 import {
   IKnowledge,
   IKnowledgeResult,
@@ -72,8 +73,8 @@ export const useTestRetrieval = () => {
 export const useFetchNextKnowledgeListByPage = () => {
   const { searchString, handleInputChange } = useHandleSearchChange();
   const { pagination, setPagination } = useGetPaginationWithRouter();
-  const [ownerIds, setOwnerIds] = useState<string[]>([]);
   const debouncedSearchString = useDebounce(searchString, { wait: 500 });
+  const { filterValue, handleFilterSubmit } = useHandleFilterSubmit();
 
   const { data, isFetching: loading } = useQuery<IKnowledgeResult>({
     queryKey: [
@@ -81,7 +82,7 @@ export const useFetchNextKnowledgeListByPage = () => {
       {
         debouncedSearchString,
         ...pagination,
-        ownerIds,
+        filterValue,
       },
     ],
     initialData: {
@@ -97,7 +98,7 @@ export const useFetchNextKnowledgeListByPage = () => {
           page: pagination.current,
         },
         {
-          owner_ids: ownerIds,
+          owner_ids: filterValue.owner,
         },
       );
 
@@ -113,11 +114,6 @@ export const useFetchNextKnowledgeListByPage = () => {
     [handleInputChange],
   );
 
-  const handleOwnerIdsChange = useCallback((ids: string[]) => {
-    // setPagination({ page: 1 }); // TODO: 这里导致重复请求
-    setOwnerIds(ids);
-  }, []);
-
   return {
     ...data,
     searchString,
@@ -125,8 +121,8 @@ export const useFetchNextKnowledgeListByPage = () => {
     pagination: { ...pagination, total: data?.total },
     setPagination,
     loading,
-    setOwnerIds: handleOwnerIdsChange,
-    ownerIds,
+    filterValue,
+    handleFilterSubmit,
   };
 };
 

web/src/interfaces/request/knowledge.ts

@@ -14,7 +14,13 @@ export interface IFetchKnowledgeListRequestBody {
 }
 
 export interface IFetchKnowledgeListRequestParams {
+  kb_id?: string;
   keywords?: string;
   page?: number;
   page_size?: number;
 }
+
+export interface IFetchDocumentListRequestBody {
+  types?: string[];
+  run_status?: string[];
+}

web/src/pages/dataset/dataset/dataset-table.tsx

@@ -35,14 +35,16 @@ import { useDatasetTableColumns } from './use-dataset-table-columns';
 import { useRenameDocument } from './use-rename-document';
 import { useSaveMeta } from './use-save-meta';
 
-export function DatasetTable() {
-  const {
-    // searchString,
-    documents,
-    pagination,
-    // handleInputChange,
-    setPagination,
-  } = useFetchDocumentList();
+export type DatasetTableProps = Pick<
+  ReturnType<typeof useFetchDocumentList>,
+  'documents' | 'setPagination' | 'pagination'
+>;
+
+export function DatasetTable({
+  documents,
+  pagination,
+  setPagination,
+}: DatasetTableProps) {
   const [sorting, setSorting] = React.useState<SortingState>([]);
   const [columnFilters, setColumnFilters] = React.useState<ColumnFiltersState>(
     [],

web/src/pages/dataset/dataset/index.tsx

@@ -2,9 +2,11 @@ import { BulkOperateBar } from '@/components/bulk-operate-bar';
 import { FileUploadDialog } from '@/components/file-upload-dialog';
 import ListFilterBar from '@/components/list-filter-bar';
 import { Button } from '@/components/ui/button';
+import { useFetchDocumentList } from '@/hooks/use-document-request';
 import { Upload } from 'lucide-react';
 import { DatasetTable } from './dataset-table';
 import { useBulkOperateDataset } from './use-bulk-operate-dataset';
+import { useSelectDatasetFilters } from './use-select-filters';
 import { useHandleUploadDocument } from './use-upload-document';
 
 export default function Dataset() {
@@ -16,10 +18,27 @@ export default function Dataset() {
     documentUploadLoading,
   } = useHandleUploadDocument();
   const { list } = useBulkOperateDataset();
+  const {
+    searchString,
+    documents,
+    pagination,
+    handleInputChange,
+    setPagination,
+    filterValue,
+    handleFilterSubmit,
+  } = useFetchDocumentList();
+  const { filters } = useSelectDatasetFilters();
 
   return (
     <section className="p-8">
-      <ListFilterBar title="Dataset">
+      <ListFilterBar
+        title="Dataset"
+        onSearchChange={handleInputChange}
+        searchString={searchString}
+        value={filterValue}
+        onChange={handleFilterSubmit}
+        filters={filters}
+      >
         <Button
           variant={'tertiary'}
           size={'sm'}
@@ -30,7 +49,11 @@ export default function Dataset() {
         </Button>
       </ListFilterBar>
       <BulkOperateBar list={list}></BulkOperateBar>
-      <DatasetTable></DatasetTable>
+      <DatasetTable
+        documents={documents}
+        pagination={pagination}
+        setPagination={setPagination}
+      ></DatasetTable>
       {documentUploadVisible && (
         <FileUploadDialog
           hideModal={hideDocumentUploadModal}

web/src/pages/dataset/dataset/use-select-filters.ts

@@ -0,0 +1,31 @@
+import { useFetchAllDocumentList } from '@/hooks/use-document-request';
+import { groupListByType } from '@/utils/dataset-util';
+import { useMemo } from 'react';
+import { useTranslation } from 'react-i18next';
+
+export function useSelectDatasetFilters() {
+  const {
+    data: { docs: documents },
+  } = useFetchAllDocumentList();
+  const { t } = useTranslation();
+
+  const fileTypes = useMemo(() => {
+    return groupListByType(documents, 'type', 'type');
+  }, [documents]);
+
+  const fileStatus = useMemo(() => {
+    return groupListByType(documents, 'run', 'run').map((x) => ({
+      ...x,
+      label: t(`knowledgeDetails.runningStatus${x.label}`),
+    }));
+  }, [documents, t]);
+
+  const filters = useMemo(() => {
+    return [
+      { field: 'type', label: 'File Type', list: fileTypes },
+      { field: 'run', label: 'Status', list: fileStatus },
+    ];
+  }, [fileStatus, fileTypes]);
+
+  return { fileTypes, fileStatus, filters };
+}

web/src/pages/datasets/index.tsx

@@ -4,14 +4,14 @@ import { Button } from '@/components/ui/button';
 import { useFetchNextKnowledgeListByPage } from '@/hooks/use-knowledge-request';
 import { pick } from 'lodash';
 import { Plus } from 'lucide-react';
-import { PropsWithChildren, useCallback } from 'react';
+import { useCallback } from 'react';
 import { useTranslation } from 'react-i18next';
 import { DatasetCard } from './dataset-card';
 import { DatasetCreatingDialog } from './dataset-creating-dialog';
-import { DatasetsFilterPopover } from './datasets-filter-popover';
 import { DatasetsPagination } from './datasets-pagination';
 import { useSaveKnowledge } from './hooks';
 import { useRenameDataset } from './use-rename-dataset';
+import { useSelectOwners } from './use-select-owners';
 
 export default function Datasets() {
   const { t } = useTranslation();
@@ -30,10 +30,12 @@ export default function Datasets() {
     setPagination,
     handleInputChange,
     searchString,
-    setOwnerIds,
-    ownerIds,
+    filterValue,
+    handleFilterSubmit,
   } = useFetchNextKnowledgeListByPage();
 
+  const owners = useSelectOwners();
+
   const {
     datasetRenameLoading,
     initialDatasetName,
@@ -54,14 +56,11 @@ export default function Datasets() {
     <section className="p-8 text-foreground">
       <ListFilterBar
         title="Datasets"
-        count={ownerIds.length}
-        FilterPopover={({ children }: PropsWithChildren) => (
-          <DatasetsFilterPopover setOwnerIds={setOwnerIds} ownerIds={ownerIds}>
-            {children}
-          </DatasetsFilterPopover>
-        )}
         searchString={searchString}
         onSearchChange={handleInputChange}
+        value={filterValue}
+        filters={owners}
+        onChange={handleFilterSubmit}
       >
         <Button variant={'tertiary'} size={'sm'} onClick={showModal}>
           <Plus className="mr-2 h-4 w-4" />

web/src/pages/datasets/use-select-owners.ts

@@ -1,28 +1,18 @@
+import { FilterCollection } from '@/components/list-filter-bar/interface';
 import { useFetchKnowledgeList } from '@/hooks/knowledge-hooks';
+import { groupListByType } from '@/utils/dataset-util';
 import { useMemo } from 'react';
 
-export type OwnerFilterType = {
-  id: string;
-  label: string;
-  count: number;
-};
-
 export function useSelectOwners() {
   const { list } = useFetchKnowledgeList();
 
   const owners = useMemo(() => {
-    const ownerList: OwnerFilterType[] = [];
-    list.forEach((x) => {
-      const item = ownerList.find((y) => y.id === x.tenant_id);
-      if (!item) {
-        ownerList.push({ id: x.tenant_id, label: x.nickname, count: 1 });
-      } else {
-        item.count += 1;
-      }
-    });
-
-    return ownerList;
+    return groupListByType(list, 'tenant_id', 'nickname');
   }, [list]);
 
-  return owners;
+  const filters: FilterCollection[] = [
+    { field: 'owner', list: owners, label: 'Owner' },
+  ];
+
+  return filters;
 }

web/src/services/knowledge-service.ts

@@ -1,5 +1,6 @@
 import { IRenameTag } from '@/interfaces/database/knowledge';
 import {
+  IFetchDocumentListRequestBody,
   IFetchKnowledgeListRequestBody,
   IFetchKnowledgeListRequestParams,
 } from '@/interfaces/request/knowledge';
@@ -182,4 +183,9 @@ export const listDataset = (
   body?: IFetchKnowledgeListRequestBody,
 ) => request.post(api.kb_list, { data: body || {}, params });
 
+export const listDocument = (
+  params?: IFetchKnowledgeListRequestParams,
+  body?: IFetchDocumentListRequestBody,
+) => request.post(api.get_document_list, { data: body || {}, params });
+
 export default kbService;

web/src/utils/dataset-util.ts

@@ -7,3 +7,27 @@ export function isKnowledgeGraphParser(parserId: DocumentParserType) {
 export function isNaiveParser(parserId: DocumentParserType) {
   return parserId === DocumentParserType.Naive;
 }
+
+export type FilterType = {
+  id: string;
+  label: string;
+  count: number;
+};
+
+export function groupListByType<T extends Record<string, any>>(
+  list: T[],
+  idField: string,
+  labelField: string,
+) {
+  const fileTypeList: FilterType[] = [];
+  list.forEach((x) => {
+    const item = fileTypeList.find((y) => y.id === x[idField]);
+    if (!item) {
+      fileTypeList.push({ id: x[idField], label: x[labelField], count: 1 });
+    } else {
+      item.count += 1;
+    }
+  });
+
+  return fileTypeList;
+}