Sfoglia il codice sorgente

feat: select the corresponding parsing method according to the file type and after the document is successfully uploaded, use the ChunkMethodModal to select the parsing method. and remove ChunkMethodModal from knowledge-file (#158)

* feat: select the corresponding parsing method according to the file type

* feat: after the document is successfully uploaded, use the ChunkMethodModal  to select the parsing method.

* feat: add pdf types to ParserListMap

* feat: remove ChunkMethodModal from knowledge-file
tags/v0.1.0
balibabu 1 anno fa
parent
commit
44541a8c33
Nessun account collegato all'indirizzo email del committer

+ 82
- 0
web/src/components/chunk-method-modal/hooks.ts Vedi File

@@ -0,0 +1,82 @@
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { useEffect, useMemo, useState } from 'react';

const ParserListMap = new Map([
[
['pdf'],
[
'naive',
'resume',
'manual',
'paper',
'book',
'laws',
'presentation',
'one',
],
],
[
['doc', 'docx'],
['naive', 'resume', 'book', 'laws', 'one'],
],
[
['xlsx', 'xls'],
['naive', 'qa', 'table', 'one'],
],
[['ppt', 'pptx'], ['presentation']],
[
['jpg', 'jpeg', 'png', 'gif', 'bmp', 'tif', 'tiff', 'webp', 'svg', 'ico'],
['picture'],
],
[['txt'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']],
[['csv'], ['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table']],
]);

const getParserList = (
values: string[],
parserList: Array<{
value: string;
label: string;
}>,
) => {
return parserList.filter((x) => values?.some((y) => y === x.value));
};

export const useFetchParserListOnMount = (
parserId: string,
documentExtension: string,
) => {
const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList();

const nextParserList = useMemo(() => {
const key = [...ParserListMap.keys()].find((x) =>
x.some((y) => y === documentExtension),
);
if (key) {
const values = ParserListMap.get(key);
return getParserList(values ?? [], parserList);
}

return getParserList(
['naive', 'resume', 'book', 'laws', 'one', 'qa', 'table'],
parserList,
);
}, [parserList, documentExtension]);

useFetchTenantInfo();

useEffect(() => {
setSelectedTag(parserId);
}, [parserId]);

const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
setSelectedTag(nextSelectedTag);
};

return { parserList: nextParserList, handleChange, selectedTag };
};

+ 10
- 0
web/src/components/chunk-method-modal/index.less Vedi File

@@ -0,0 +1,10 @@
.pageInputNumber {
width: 220px;
}

.questionIcon {
margin-inline-start: 4px;
color: rgba(0, 0, 0, 0.45);
cursor: help;
writing-mode: horizontal-tb;
}

web/src/pages/add-knowledge/components/knowledge-file/chunk-method-modal.tsx → web/src/components/chunk-method-modal/index.tsx Vedi File

@@ -35,7 +35,7 @@ interface IProps extends Omit<IModalManagerChildrenProps, 'showModal'> {
showModal?(): void;
parserId: string;
parserConfig: IKnowledgeFileParserConfig;
documentType: string;
documentExtension: string;
}

const hidePagesChunkMethods = ['qa', 'table', 'picture', 'resume', 'one'];
@@ -45,11 +45,13 @@ const ChunkMethodModal: React.FC<IProps> = ({
onOk,
hideModal,
visible,
documentType,
documentExtension,
parserConfig,
}) => {
const { parserList, handleChange, selectedTag } =
useFetchParserListOnMount(parserId);
const { parserList, handleChange, selectedTag } = useFetchParserListOnMount(
parserId,
documentExtension,
);
const [form] = Form.useForm();

const handleOk = async () => {
@@ -62,11 +64,8 @@ const ChunkMethodModal: React.FC<IProps> = ({
};

const showPages = useMemo(() => {
return (
documentType === 'pdf' &&
hidePagesChunkMethods.every((x) => x !== selectedTag)
);
}, [documentType, selectedTag]);
return hidePagesChunkMethods.every((x) => x !== selectedTag);
}, [selectedTag]);

const showOne = useMemo(() => {
return showPages || selectedTag === 'one';
@@ -114,7 +113,7 @@ const ChunkMethodModal: React.FC<IProps> = ({
</Space>
<Divider></Divider>

{
{documentExtension === 'pdf' && (
<Form name="dynamic_form_nest_item" autoComplete="off" form={form}>
{showPages && (
<>
@@ -271,7 +270,7 @@ const ChunkMethodModal: React.FC<IProps> = ({

{selectedTag === 'naive' && <MaxTokenNumber></MaxTokenNumber>}
</Form>
}
)}
</Modal>
);
};

+ 45
- 0
web/src/hooks/documentHooks.ts Vedi File

@@ -2,6 +2,7 @@ import { IChunk, IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import { api_host } from '@/utils/api';
import { buildChunkHighlights } from '@/utils/documentUtils';
import { UploadFile } from 'antd';
import { useCallback, useMemo, useState } from 'react';
import { IHighlight } from 'react-pdf-highlighter';
import { useDispatch, useSelector } from 'umi';
@@ -174,3 +175,47 @@ export const useRemoveDocument = (documentId: string) => {

return removeDocument;
};

export const useUploadDocument = () => {
const dispatch = useDispatch();
const { knowledgeId } = useGetKnowledgeSearchParams();

const uploadDocument = useCallback(
(file: UploadFile) => {
try {
return dispatch<any>({
type: 'kFModel/upload_document',
payload: {
file,
kb_id: knowledgeId,
},
});
} catch (errorInfo) {
console.log('Failed:', errorInfo);
}
},
[dispatch, knowledgeId],
);

return uploadDocument;
};

export const useRunDocument = () => {
const dispatch = useDispatch();

const runDocumentByIds = useCallback(
(ids: string[]) => {
try {
return dispatch<any>({
type: 'kFModel/document_run',
payload: { doc_ids: ids, run: 1 },
});
} catch (errorInfo) {
console.log('Failed:', errorInfo);
}
},
[dispatch],
);

return runDocumentByIds;
};

+ 47
- 0
web/src/hooks/logicHooks.ts Vedi File

@@ -0,0 +1,47 @@
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
import { useCallback, useState } from 'react';
import { useSetModalState } from './commonHooks';
import { useSetDocumentParser } from './documentHooks';
import { useOneNamespaceEffectsLoading } from './storeHooks';

export const useChangeDocumentParser = (documentId: string) => {
const setDocumentParser = useSetDocumentParser();

const {
visible: changeParserVisible,
hideModal: hideChangeParserModal,
showModal: showChangeParserModal,
} = useSetModalState();
const loading = useOneNamespaceEffectsLoading('kFModel', [
'document_change_parser',
]);

const onChangeParserOk = useCallback(
async (parserId: string, parserConfig: IChangeParserConfigRequestBody) => {
const ret = await setDocumentParser(parserId, documentId, parserConfig);
if (ret === 0) {
hideChangeParserModal();
}
},
[hideChangeParserModal, setDocumentParser, documentId],
);

return {
changeParserLoading: loading,
onChangeParserOk,
changeParserVisible,
hideChangeParserModal,
showChangeParserModal,
};
};

export const useSetSelectedRecord = <T = IKnowledgeFile>() => {
const [currentRecord, setCurrentRecord] = useState<T>({} as T);

const setRecord = (record: T) => {
setCurrentRecord(record);
};

return { currentRecord, setRecord };
};

+ 120
- 121
web/src/pages/add-knowledge/components/knowledge-dataset/knowledge-upload-file/index.tsx Vedi File

@@ -1,18 +1,24 @@
import { ReactComponent as SelectFilesEndIcon } from '@/assets/svg/select-files-end.svg';
import { ReactComponent as SelectFilesStartIcon } from '@/assets/svg/select-files-start.svg';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { KnowledgeRouteKey } from '@/constants/knowledge';
import {
useRunDocument,
useSelectDocumentList,
useUploadDocument,
} from '@/hooks/documentHooks';
import {
useDeleteDocumentById,
useFetchKnowledgeDetail,
useGetDocumentDefaultParser,
useKnowledgeBaseId,
} from '@/hooks/knowledgeHook';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import uploadService from '@/services/uploadService';
import { isFileUploadDone } from '@/utils/documentUtils';
useChangeDocumentParser,
useSetSelectedRecord,
} from '@/hooks/logicHooks';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { getExtension, isFileUploadDone } from '@/utils/documentUtils';
import {
ArrowLeftOutlined,
CloudUploadOutlined,
@@ -24,27 +30,16 @@ import {
Button,
Card,
Flex,
Popover,
Progress,
Radio,
RadioChangeEvent,
Space,
Upload,
UploadFile,
UploadProps,
} from 'antd';
import classNames from 'classnames';
import {
ReactElement,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from 'react';
import { Link, useDispatch, useNavigate } from 'umi';
import { ReactElement, useCallback, useMemo, useRef, useState } from 'react';
import { Link, useNavigate } from 'umi';

import { useSetDocumentParser } from '@/hooks/documentHooks';
import styles from './index.less';

const { Dragger } = Upload;
@@ -57,48 +52,21 @@ const UploaderItem = ({
file,
isUpload,
remove,
handleEdit,
}: {
isUpload: boolean;
originNode: ReactElement;
file: UploadFile;
fileList: object[];
showModal: () => void;
remove: (id: string) => void;
setRecord: (record: IKnowledgeFile) => void;
handleEdit: (id: string) => void;
}) => {
const { parserConfig, defaultParserId } = useGetDocumentDefaultParser();
const { removeDocument } = useDeleteDocumentById();
const [value, setValue] = useState(defaultParserId);
const setDocumentParser = useSetDocumentParser();

const documentId = file?.response?.id;

const parserList = useSelectParserList();

const saveParser = (parserId: string) => {
setDocumentParser(parserId, documentId, parserConfig as any);
};

const onChange = (e: RadioChangeEvent) => {
const val = e.target.value;
setValue(val);
saveParser(val);
};

const content = (
<Radio.Group onChange={onChange} value={value}>
<Space direction="vertical">
{parserList.map(
(
x, // value is lowercase, key is uppercase
) => (
<Radio value={x.value} key={x.value}>
{x.label}
</Radio>
),
)}
</Space>
</Radio.Group>
);

const handleRemove = async () => {
if (file.status === 'error') {
remove(documentId);
@@ -110,9 +78,11 @@ const UploaderItem = ({
}
};

useEffect(() => {
setValue(defaultParserId);
}, [defaultParserId]);
const handleEditClick = () => {
if (file.status === 'done') {
handleEdit(documentId);
}
};

return (
<Card className={styles.uploaderItem}>
@@ -130,9 +100,7 @@ const UploaderItem = ({
onClick={handleRemove}
/>
) : (
<Popover content={content} placement="bottom">
<EditOutlined />
</Popover>
<EditOutlined onClick={handleEditClick} />
)}
</Flex>
<Flex>
@@ -153,10 +121,20 @@ const UploaderItem = ({
const KnowledgeUploadFile = () => {
const knowledgeBaseId = useKnowledgeBaseId();
const [isUpload, setIsUpload] = useState(true);
const dispatch = useDispatch();
const [uploadedFileIds, setUploadedFileIds] = useState<string[]>([]);
const fileListRef = useRef<UploadFile[]>([]);
const navigate = useNavigate();
const { currentRecord, setRecord } = useSetSelectedRecord();
const {
changeParserLoading,
onChangeParserOk,
changeParserVisible,
hideChangeParserModal,
showChangeParserModal,
} = useChangeDocumentParser(currentRecord.id);
const documentList = useSelectDocumentList();
const runDocumentByIds = useRunDocument();
const uploadDocument = useUploadDocument();

const enabled = useMemo(() => {
if (isUpload) {
@@ -175,8 +153,7 @@ const KnowledgeUploadFile = () => {
onError,
// onProgress,
}) {
const ret = await uploadService.uploadFile(file, knowledgeBaseId);
const data = ret?.data;
const data = await uploadDocument(file as UploadFile);
if (data?.retcode === 0) {
setUploadedFileIds((pre) => {
return pre.concat(data.data.id);
@@ -197,6 +174,17 @@ const KnowledgeUploadFile = () => {
});
}, []);

const handleItemEdit = useCallback(
(id: string) => {
const document = documentList.find((x) => x.id === id);
if (document) {
setRecord(document);
}
showChangeParserModal();
},
[documentList, showChangeParserModal, setRecord],
);

const props: UploadProps = {
name: 'file',
multiple: true,
@@ -215,6 +203,9 @@ const KnowledgeUploadFile = () => {
fileList={fileList}
originNode={originNode}
remove={remove}
showModal={showChangeParserModal}
setRecord={setRecord}
handleEdit={handleItemEdit}
></UploaderItem>
);
},
@@ -226,10 +217,7 @@ const KnowledgeUploadFile = () => {

const runSelectedDocument = () => {
const ids = fileListRef.current.map((x) => x.response.id);
dispatch({
type: 'kFModel/document_run',
payload: { doc_ids: ids, run: 1 },
});
runDocumentByIds(ids);
};

const handleNextClick = () => {
@@ -245,67 +233,78 @@ const KnowledgeUploadFile = () => {
useFetchKnowledgeDetail();

return (
<div className={styles.uploadWrapper}>
<section>
<Space className={styles.backToList}>
<ArrowLeftOutlined />
<Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
Back to select files
</Link>
</Space>
<div className={styles.progressWrapper}>
<Flex align="center" justify="center">
<SelectFilesStartIcon></SelectFilesStartIcon>
<Progress
percent={100}
showInfo={false}
className={styles.progress}
strokeColor="
rgba(127, 86, 217, 1)
"
/>
<SelectFilesEndIcon></SelectFilesEndIcon>
</Flex>
<Flex justify="space-around">
<p className={styles.selectFilesText}>
<b>Select files</b>
<>
<div className={styles.uploadWrapper}>
<section>
<Space className={styles.backToList}>
<ArrowLeftOutlined />
<Link to={`/knowledge/dataset?id=${knowledgeBaseId}`}>
Back to select files
</Link>
</Space>
<div className={styles.progressWrapper}>
<Flex align="center" justify="center">
<SelectFilesStartIcon></SelectFilesStartIcon>
<Progress
percent={100}
showInfo={false}
className={styles.progress}
strokeColor="
rgba(127, 86, 217, 1)
"
/>
<SelectFilesEndIcon></SelectFilesEndIcon>
</Flex>
<Flex justify="space-around">
<p className={styles.selectFilesText}>
<b>Select files</b>
</p>
<p className={styles.changeSpecificCategoryText}>
<b>Change specific category</b>
</p>
</Flex>
</div>
</section>
<section className={styles.uploadContent}>
<Dragger
{...props}
className={classNames(styles.uploader, {
[styles.hiddenUploader]: !isUpload,
})}
>
<Button className={styles.uploaderButton}>
<CloudUploadOutlined className={styles.uploaderIcon} />
</Button>
<p className="ant-upload-text">
Click or drag file to this area to upload
</p>
<p className={styles.changeSpecificCategoryText}>
<b>Change specific category</b>
<p className="ant-upload-hint">
Support for a single or bulk upload. Strictly prohibited from
uploading company data or other banned files.
</p>
</Flex>
</div>
</section>
<section className={styles.uploadContent}>
<Dragger
{...props}
className={classNames(styles.uploader, {
[styles.hiddenUploader]: !isUpload,
})}
>
<Button className={styles.uploaderButton}>
<CloudUploadOutlined className={styles.uploaderIcon} />
</Dragger>
</section>
<section className={styles.footer}>
<Button
type="primary"
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>
<p className="ant-upload-text">
Click or drag file to this area to upload
</p>
<p className="ant-upload-hint">
Support for a single or bulk upload. Strictly prohibited from
uploading company data or other banned files.
</p>
</Dragger>
</section>
<section className={styles.footer}>
<Button
type="primary"
// className={styles.nextButton}
onClick={handleNextClick}
disabled={!enabled}
>
Next
</Button>
</section>
</div>
</section>
</div>
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}
loading={changeParserLoading}
/>
</>
);
};


+ 1
- 22
web/src/pages/add-knowledge/components/knowledge-file/hooks.ts Vedi File

@@ -7,10 +7,7 @@ import {
} from '@/hooks/documentHooks';
import { useGetKnowledgeSearchParams } from '@/hooks/routeHook';
import { useOneNamespaceEffectsLoading } from '@/hooks/storeHooks';
import {
useFetchTenantInfo,
useSelectParserList,
} from '@/hooks/userSettingHook';
import { useFetchTenantInfo } from '@/hooks/userSettingHook';
import { Pagination } from '@/interfaces/common';
import { IKnowledgeFile } from '@/interfaces/database/knowledge';
import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
@@ -243,21 +240,3 @@ export const useChangeDocumentParser = (documentId: string) => {
showChangeParserModal,
};
};

export const useFetchParserListOnMount = (parserId: string) => {
const [selectedTag, setSelectedTag] = useState('');
const parserList = useSelectParserList();

useFetchTenantInfo();

useEffect(() => {
setSelectedTag(parserId);
}, [parserId]);

const handleChange = (tag: string, checked: boolean) => {
const nextSelectedTag = checked ? tag : selectedTag;
setSelectedTag(nextSelectedTag);
};

return { parserList, handleChange, selectedTag };
};

+ 3
- 2
web/src/pages/add-knowledge/components/knowledge-file/index.tsx Vedi File

@@ -23,7 +23,6 @@ import {
} from 'antd';
import type { ColumnsType } from 'antd/es/table';
import { useMemo } from 'react';
import ChunkMethodModal from './chunk-method-modal';
import CreateFileModal from './create-file-modal';
import {
useChangeDocumentParser,
@@ -39,6 +38,8 @@ import ParsingActionCell from './parsing-action-cell';
import ParsingStatusCell from './parsing-status-cell';
import RenameModal from './rename-modal';
import ChunkMethodModal from '@/components/chunk-method-modal';
import { getExtension } from '@/utils/documentUtils';
import styles from './index.less';
const KnowledgeFile = () => {
@@ -227,7 +228,7 @@ const KnowledgeFile = () => {
<ChunkMethodModal
parserId={currentRecord.parser_id}
parserConfig={currentRecord.parser_config}
documentType={currentRecord.type}
documentExtension={getExtension(currentRecord.name)}
onOk={onChangeParserOk}
visible={changeParserVisible}
hideModal={hideChangeParserModal}

+ 13
- 0
web/src/pages/add-knowledge/components/knowledge-file/model.ts Vedi File

@@ -209,6 +209,19 @@ const model: DvaModel<KFModelState> = {
console.warn(error);
}
},
*upload_document({ payload = {} }, { call, put }) {
const formData = new FormData();
formData.append('file', payload.file);
formData.append('kb_id', payload.kb_id);
const { data } = yield call(kbService.document_upload, formData);
if (data.retcode === 0) {
yield put({
type: 'getKfList',
payload: { kb_id: payload.kb_id },
});
}
return data;
},
},
};
export default model;

+ 0
- 42
web/src/pages/add-knowledge/components/knowledge-file/upload.tsx Vedi File

@@ -1,42 +0,0 @@
import { useKnowledgeBaseId } from '@/hooks/knowledgeHook';
import uploadService from '@/services/uploadService';
import type { UploadProps } from 'antd';
import React from 'react';
import { Link } from 'umi';
interface PropsType {
kb_id: string;
getKfList: () => void;
}
type UploadRequestOption = Parameters<
NonNullable<UploadProps['customRequest']>
>[0];
const FileUpload: React.FC<PropsType> = ({ kb_id, getKfList }) => {
const knowledgeBaseId = useKnowledgeBaseId();
const createRequest: (props: UploadRequestOption) => void = async function ({
file,
onSuccess,
onError,
}) {
const { retcode, data } = await uploadService.uploadFile(file, kb_id);
if (retcode === 0) {
onSuccess && onSuccess(data, file);
} else {
onError && onError(data);
}
getKfList && getKfList();
};
const uploadProps: UploadProps = {
customRequest: createRequest,
showUploadList: false,
};
return (
// <Upload {...uploadProps}>
<Link to={`/knowledge/dataset/upload?id=${knowledgeBaseId}`}>导入文件</Link>
// </Upload>
);
};
export default FileUpload;

+ 5
- 0
web/src/services/kbService.ts Vedi File

@@ -25,6 +25,7 @@ const {
document_rename,
document_run,
get_document_file,
document_upload,
} = api;
const methods = {
@@ -82,6 +83,10 @@ const methods = {
url: document_thumbnails,
method: 'get',
},
document_upload: {
url: document_upload,
method: 'post',
},
// chunk管理
chunk_list: {
url: chunk_list,

+ 0
- 21
web/src/services/uploadService.ts Vedi File

@@ -1,21 +0,0 @@
import api from '@/utils/api';
import request from '@/utils/request';
const { upload } = api;
const uploadService = {
uploadFile: function (file: any, kb_id: string) {
const formData = new FormData();
formData.append('file', file);
formData.append('kb_id', kb_id);
const options = {
method: 'post',
data: formData,
};
return request(upload, options);
},
};
export default uploadService;

+ 1
- 0
web/src/utils/api.ts Vedi File

@@ -45,6 +45,7 @@ export default {
document_change_parser: `${api_host}/document/change_parser`,
document_thumbnails: `${api_host}/document/thumbnails`,
get_document_file: `${api_host}/document/get`,
document_upload: `${api_host}/document/upload`,
setDialog: `${api_host}/dialog/set`,
getDialog: `${api_host}/dialog/get`,

+ 3
- 0
web/src/utils/documentUtils.ts Vedi File

@@ -35,3 +35,6 @@ export const buildChunkHighlights = (
};

export const isFileUploadDone = (file: UploadFile) => file.status === 'done';

export const getExtension = (name: string) =>
name?.slice(name.lastIndexOf('.') + 1).toLowerCase() ?? '';

Loading…
Annulla
Salva