Sfoglia il codice sorgente

fix: test chunk by @tanstack/react-query #1306 (#1719)

### What problem does this PR solve?

fix: test chunk by @tanstack/react-query #1306

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.9.0
balibabu 1 anno fa
parent
commit
549d67e281
Nessun account collegato all'indirizzo email del committer

+ 5
- 2
web/src/hooks/file-manager-hooks.ts Vedi File

import React, { useCallback } from 'react'; import React, { useCallback } from 'react';
import { useTranslation } from 'react-i18next'; import { useTranslation } from 'react-i18next';
import { useSearchParams } from 'umi'; import { useSearchParams } from 'umi';
import { useGetNextPagination, useHandleSearchChange } from './logic-hooks';
import {
useGetPaginationWithRouter,
useHandleSearchChange,
} from './logic-hooks';
import { useSetPaginationParams } from './route-hook'; import { useSetPaginationParams } from './route-hook';


export const useGetFolderId = () => { export const useGetFolderId = () => {


export const useFetchFileList = (): ResponseType<any> & IListResult => { export const useFetchFileList = (): ResponseType<any> & IListResult => {
const { searchString, handleInputChange } = useHandleSearchChange(); const { searchString, handleInputChange } = useHandleSearchChange();
const { pagination, setPagination } = useGetNextPagination();
const { pagination, setPagination } = useGetPaginationWithRouter();
const id = useGetFolderId(); const id = useGetFolderId();


const { data, isFetching: loading } = useQuery({ const { data, isFetching: loading } = useQuery({

+ 58
- 26
web/src/hooks/knowledge-hooks.ts Vedi File

import { useShowDeleteConfirm } from '@/hooks/common-hooks'; import { useShowDeleteConfirm } from '@/hooks/common-hooks';
import { IKnowledge } from '@/interfaces/database/knowledge';
import { ResponsePostType } from '@/interfaces/database/base';
import { IKnowledge, ITestingResult } from '@/interfaces/database/knowledge';
import i18n from '@/locales/config'; import i18n from '@/locales/config';
import kbService from '@/services/knowledge-service'; import kbService from '@/services/knowledge-service';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import {
useIsMutating,
useMutation,
useMutationState,
useQuery,
useQueryClient,
} from '@tanstack/react-query';
import { message } from 'antd'; import { message } from 'antd';
import { useCallback, useEffect } from 'react'; import { useCallback, useEffect } from 'react';
import { useDispatch, useSearchParams, useSelector } from 'umi'; import { useDispatch, useSearchParams, useSelector } from 'umi';
import { useSetPaginationParams } from './route-hook';


export const useKnowledgeBaseId = (): string => { export const useKnowledgeBaseId = (): string => {
const [searchParams] = useSearchParams(); const [searchParams] = useSearchParams();


//#region Retrieval testing //#region Retrieval testing


export const useTestChunkRetrieval = () => {
const dispatch = useDispatch();
export const useTestChunkRetrieval = (): ResponsePostType<ITestingResult> & {
testChunk: (...params: any[]) => void;
} => {
const knowledgeBaseId = useKnowledgeBaseId(); const knowledgeBaseId = useKnowledgeBaseId();
const { page, size: pageSize } = useSetPaginationParams();


const testChunk = useCallback(
(values: any) => {
dispatch({
type: 'testingModel/testDocumentChunk',
payload: {
...values,
kb_id: knowledgeBaseId,
},
});
},
[dispatch, knowledgeBaseId],
);

return testChunk;
};

export const useTestNextChunkRetrieval = () => {
const { const {
data, data,
isPending: loading, isPending: loading,
mutateAsync, mutateAsync,
} = useMutation({ } = useMutation({
mutationKey: ['testChunk'],
mutationFn: async (canvasIds: string[]) => {
const { data } = await kbService.retrieval_test({ canvasIds });
mutationKey: ['testChunk'], // This method is invalid
mutationFn: async (values: any) => {
const { data } = await kbService.retrieval_test({
...values,
kb_id: knowledgeBaseId,
page,
size: pageSize,
});
if (data.retcode === 0) { if (data.retcode === 0) {
const res = data.data;
return {
chunks: res.chunks,
documents: res.doc_aggs,
total: res.total,
};
} }
return data?.data ?? [];
return (
data?.data ?? {
chunks: [],
documents: [],
total: 0,
}
);
}, },
}); });


return { data, loading, testChunk: mutateAsync };
return {
data: data ?? { chunks: [], documents: [], total: 0 },
loading,
testChunk: mutateAsync,
};
};

export const useChunkIsTesting = () => {
return useIsMutating({ mutationKey: ['testChunk'] }) > 0;
};

export const useSelectTestingResult = (): ITestingResult => {
const data = useMutationState({
filters: { mutationKey: ['testChunk'] },
select: (mutation) => {
return mutation.state.data;
},
});
return (data.at(-1) ?? {
chunks: [],
documents: [],
total: 0,
}) as ITestingResult;
}; };
//#endregion //#endregion

+ 16
- 13
web/src/hooks/logic-hooks.ts Vedi File

return changeLanguage; return changeLanguage;
}; };


export const useGetNextPagination = () => {
export const useGetPaginationWithRouter = () => {
const { t } = useTranslate('common'); const { t } = useTranslate('common');
const { const {
setPaginationParams, setPaginationParams,
}; };
}; };


export const useGetPagination = (
total: number,
page: number,
pageSize: number,
onPageChange: PaginationProps['onChange'],
) => {
export const useGetPagination = () => {
const [pagination, setPagination] = useState({ page: 1, pageSize: 10 });
const { t } = useTranslate('common'); const { t } = useTranslate('common');


const pagination: PaginationProps = useMemo(() => {
const onPageChange: PaginationProps['onChange'] = useCallback(
(pageNumber: number, pageSize: number) => {
setPagination({ page: pageNumber, pageSize });
},
[],
);

const currentPagination: PaginationProps = useMemo(() => {
return { return {
showQuickJumper: true, showQuickJumper: true,
total,
total: 0,
showSizeChanger: true, showSizeChanger: true,
current: page,
pageSize: pageSize,
current: pagination.page,
pageSize: pagination.pageSize,
pageSizeOptions: [1, 2, 10, 20, 50, 100], pageSizeOptions: [1, 2, 10, 20, 50, 100],
onChange: onPageChange, onChange: onPageChange,
showTotal: (total) => `${t('total')} ${total}`, showTotal: (total) => `${t('total')} ${total}`,
}; };
}, [t, onPageChange, page, pageSize, total]);
}, [t, onPageChange, pagination]);


return { return {
pagination,
pagination: currentPagination,
}; };
}; };



+ 6
- 0
web/src/interfaces/database/base.ts Vedi File

data: T; data: T;
loading?: boolean; loading?: boolean;
} }

export interface ResponsePostType<T = any> {
data: T;
loading?: boolean;
[key: string]: unknown;
}

+ 1
- 1
web/src/interfaces/database/knowledge.ts Vedi File



export interface ITestingResult { export interface ITestingResult {
chunks: ITestingChunk[]; chunks: ITestingChunk[];
doc_aggs: Record<string, number>;
documents: ITestingDocument[];
total: number; total: number;
} }

+ 3
- 12
web/src/pages/add-knowledge/components/knowledge-testing/index.tsx Vedi File

import { useTestChunkRetrieval } from '@/hooks/knowledge-hooks'; import { useTestChunkRetrieval } from '@/hooks/knowledge-hooks';
import { Flex, Form } from 'antd'; import { Flex, Form } from 'antd';
import { useEffect } from 'react';
import { useDispatch } from 'umi';
import TestingControl from './testing-control'; import TestingControl from './testing-control';
import TestingResult from './testing-result'; import TestingResult from './testing-result';




const KnowledgeTesting = () => { const KnowledgeTesting = () => {
const [form] = Form.useForm(); const [form] = Form.useForm();
const testChunk = useTestChunkRetrieval();
const { testChunk } = useTestChunkRetrieval();


const dispatch = useDispatch();

const handleTesting = async () => {
const handleTesting = async (documentIds: string[] = []) => {
const values = await form.validateFields(); const values = await form.validateFields();
testChunk({ testChunk({
...values, ...values,
doc_ids: Array.isArray(documentIds) ? documentIds : [],
vector_similarity_weight: 1 - values.vector_similarity_weight, vector_similarity_weight: 1 - values.vector_similarity_weight,
}); });
}; };


useEffect(() => {
return () => {
dispatch({ type: 'testingModel/reset' });
};
}, [dispatch]);

return ( return (
<Flex className={styles.testingWrapper} gap={16}> <Flex className={styles.testingWrapper} gap={16}>
<TestingControl <TestingControl

+ 0
- 72
web/src/pages/add-knowledge/components/knowledge-testing/model.ts Vedi File

import { BaseState } from '@/interfaces/common';
import {
ITestingChunk,
ITestingDocument,
} from '@/interfaces/database/knowledge';
import kbService from '@/services/knowledge-service';
import { DvaModel } from 'umi';

export interface TestingModelState extends Pick<BaseState, 'pagination'> {
chunks: ITestingChunk[];
documents: ITestingDocument[];
total: number;
selectedDocumentIds: string[] | undefined;
}

const initialState = {
chunks: [],
documents: [],
total: 0,
pagination: {
current: 1,
pageSize: 10,
},
selectedDocumentIds: undefined,
};

const model: DvaModel<TestingModelState> = {
namespace: 'testingModel',
state: initialState,
reducers: {
setChunksAndDocuments(state, { payload }) {
return {
...state,
...payload,
};
},
setPagination(state, { payload }) {
return { ...state, pagination: { ...state.pagination, ...payload } };
},
setSelectedDocumentIds(state, { payload }) {
return { ...state, selectedDocumentIds: payload };
},
reset() {
return initialState;
},
},
effects: {
*testDocumentChunk({ payload = {} }, { call, put, select }) {
const { pagination, selectedDocumentIds }: TestingModelState =
yield select((state: any) => state.testingModel);

const { data } = yield call(kbService.retrieval_test, {
...payload,
doc_ids: selectedDocumentIds,
page: pagination.current,
size: pagination.pageSize,
});
const { retcode, data: res } = data;
if (retcode === 0) {
yield put({
type: 'setChunksAndDocuments',
payload: {
chunks: res.chunks,
documents: res.doc_aggs,
total: res.total,
},
});
}
},
},
};
export default model;

+ 2
- 4
web/src/pages/add-knowledge/components/knowledge-testing/testing-control/index.tsx Vedi File

import Rerank from '@/components/rerank'; import Rerank from '@/components/rerank';
import SimilaritySlider from '@/components/similarity-slider'; import SimilaritySlider from '@/components/similarity-slider';
import { useTranslate } from '@/hooks/common-hooks'; import { useTranslate } from '@/hooks/common-hooks';
import { useOneNamespaceEffectsLoading } from '@/hooks/store-hooks';
import { Button, Card, Divider, Flex, Form, Input } from 'antd'; import { Button, Card, Divider, Flex, Form, Input } from 'antd';
import { FormInstance } from 'antd/lib'; import { FormInstance } from 'antd/lib';


import { useChunkIsTesting } from '@/hooks/knowledge-hooks';
import styles from './index.less'; import styles from './index.less';


type FieldType = { type FieldType = {


const TestingControl = ({ form, handleTesting }: IProps) => { const TestingControl = ({ form, handleTesting }: IProps) => {
const question = Form.useWatch('question', { form, preserve: true }); const question = Form.useWatch('question', { form, preserve: true });
const loading = useOneNamespaceEffectsLoading('testingModel', [
'testDocumentChunk',
]);
const loading = useChunkIsTesting();
const { t } = useTranslate('knowledgeDetails'); const { t } = useTranslate('knowledgeDetails');


const buttonDisabled = const buttonDisabled =

+ 22
- 22
web/src/pages/add-knowledge/components/knowledge-testing/testing-result/index.tsx Vedi File

Space, Space,
} from 'antd'; } from 'antd';
import camelCase from 'lodash/camelCase'; import camelCase from 'lodash/camelCase';
import { useDispatch, useSelector } from 'umi';
import { TestingModelState } from '../model';
import SelectFiles from './select-files'; import SelectFiles from './select-files';


import { useSelectTestingResult } from '@/hooks/knowledge-hooks';
import { useGetPaginationWithRouter } from '@/hooks/logic-hooks';
import { useCallback, useState } from 'react';
import styles from './index.less'; import styles from './index.less';


const similarityList: Array<{ field: keyof ITestingChunk; label: string }> = [ const similarityList: Array<{ field: keyof ITestingChunk; label: string }> = [
}; };


interface IProps { interface IProps {
handleTesting: () => Promise<any>;
handleTesting: (documentIds?: string[]) => Promise<any>;
} }


const TestingResult = ({ handleTesting }: IProps) => { const TestingResult = ({ handleTesting }: IProps) => {
const {
documents,
chunks,
total,
pagination,
selectedDocumentIds,
}: TestingModelState = useSelector((state: any) => state.testingModel);
const dispatch = useDispatch();
const [selectedDocumentIds, setSelectedDocumentIds] = useState<string[]>([]);
const { documents, chunks, total } = useSelectTestingResult();
const { t } = useTranslate('knowledgeDetails'); const { t } = useTranslate('knowledgeDetails');
const { pagination, setPagination } = useGetPaginationWithRouter();


const onChange: PaginationProps['onChange'] = (pageNumber, pageSize) => { const onChange: PaginationProps['onChange'] = (pageNumber, pageSize) => {
console.log('Page: ', pageNumber, pageSize);
dispatch({
type: 'testingModel/setPagination',
payload: { current: pageNumber, pageSize },
});
handleTesting();
pagination.onChange?.(pageNumber, pageSize);
handleTesting(selectedDocumentIds);
}; };


const onTesting = useCallback(
(ids: string[]) => {
setPagination({ page: 1 });
handleTesting(ids);
},
[setPagination, handleTesting],
);

return ( return (
<section className={styles.testingResultWrapper}> <section className={styles.testingResultWrapper}>
<Collapse <Collapse
), ),
children: ( children: (
<div> <div>
<SelectFiles handleTesting={handleTesting}></SelectFiles>
<SelectFiles
setSelectedDocumentIds={setSelectedDocumentIds}
handleTesting={onTesting}
></SelectFiles>
</div> </div>
), ),
}, },
))} ))}
</Flex> </Flex>
<Pagination <Pagination
{...pagination}
size={'small'} size={'small'}
showQuickJumper
current={pagination.current}
pageSize={pagination.pageSize}
total={total} total={total}
showSizeChanger
onChange={onChange} onChange={onChange}
/> />
</section> </section>

+ 7
- 13
web/src/pages/add-knowledge/components/knowledge-testing/testing-result/select-files.tsx Vedi File

import NewDocumentLink from '@/components/new-document-link'; import NewDocumentLink from '@/components/new-document-link';
import { useTranslate } from '@/hooks/common-hooks'; import { useTranslate } from '@/hooks/common-hooks';
import { useSelectTestingResult } from '@/hooks/knowledge-hooks';
import { ITestingDocument } from '@/interfaces/database/knowledge'; import { ITestingDocument } from '@/interfaces/database/knowledge';
import { EyeOutlined } from '@ant-design/icons'; import { EyeOutlined } from '@ant-design/icons';
import { Button, Table, TableProps, Tooltip } from 'antd'; import { Button, Table, TableProps, Tooltip } from 'antd';
import { useDispatch, useSelector } from 'umi';


interface IProps { interface IProps {
handleTesting: () => Promise<any>;
handleTesting: (ids: string[]) => void;
setSelectedDocumentIds: (ids: string[]) => void;
} }


const SelectFiles = ({ handleTesting }: IProps) => {
const documents: ITestingDocument[] = useSelector(
(state: any) => state.testingModel.documents,
);
const SelectFiles = ({ setSelectedDocumentIds, handleTesting }: IProps) => {
const { documents } = useSelectTestingResult();
const { t } = useTranslate('fileManager'); const { t } = useTranslate('fileManager');


const dispatch = useDispatch();

const columns: TableProps<ITestingDocument>['columns'] = [ const columns: TableProps<ITestingDocument>['columns'] = [
{ {
title: 'Name', title: 'Name',


const rowSelection = { const rowSelection = {
onChange: (selectedRowKeys: React.Key[]) => { onChange: (selectedRowKeys: React.Key[]) => {
dispatch({
type: 'testingModel/setSelectedDocumentIds',
payload: selectedRowKeys,
});
handleTesting();
handleTesting(selectedRowKeys as string[]);
setSelectedDocumentIds(selectedRowKeys as string[]);
}, },
getCheckboxProps: (record: ITestingDocument) => ({ getCheckboxProps: (record: ITestingDocument) => ({
disabled: record.doc_name === 'Disabled User', // Column configuration not to be checked disabled: record.doc_name === 'Disabled User', // Column configuration not to be checked

+ 0
- 2
web/typings.d.ts Vedi File

import { ChunkModelState } from '@/pages/add-knowledge/components/knowledge-chunk/model'; import { ChunkModelState } from '@/pages/add-knowledge/components/knowledge-chunk/model';
import { KFModelState } from '@/pages/add-knowledge/components/knowledge-file/model'; import { KFModelState } from '@/pages/add-knowledge/components/knowledge-file/model';
import { TestingModelState } from '@/pages/add-knowledge/components/knowledge-testing/model';
import { ChatModelState } from '@/pages/chat/model'; import { ChatModelState } from '@/pages/chat/model';


declare module 'lodash'; declare module 'lodash';
chatModel: ChatModelState; chatModel: ChatModelState;
kFModel: KFModelState; kFModel: KFModelState;
chunkModel: ChunkModelState; chunkModel: ChunkModelState;
testingModel: TestingModelState;
} }


declare global { declare global {

Loading…
Annulla
Salva