Feat: Upload file UI/UX enhancements (#5359)

### What problem does this PR solve?

Modifies the UX for uploading process on the website.

- Adds option to parse on creation the files
- Adds progress bar to display progress of chunk
- Adds per file feedback on uploading operation

#### Screenshots:

- Show files uploading:

![image](https://github.com/user-attachments/assets/a5693f42-8232-4d5c-a240-20ed343634a5)

- Errors on specific files

![image](https://github.com/user-attachments/assets/986a7f54-ab32-4634-89ab-a098fe1954aa)


### Type of change

- [X] New Feature (non-breaking change which adds functionality)

api/apps/document_app.py

@@ -71,11 +71,13 @@ def upload():
     if not e:
         raise LookupError("Can't find this knowledgebase!")
 
-    err, _ = FileService.upload_document(kb, file_objs, current_user.id)
+    err, files = FileService.upload_document(kb, file_objs, current_user.id)
+    files = [f[0] for f in files] # remove the blob
+    
     if err:
         return get_json_result(
-            data=False, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
-    return get_json_result(data=True)
+            data=files, message="\n".join(err), code=settings.RetCode.SERVER_ERROR)
+    return get_json_result(data=files)
 
 
 @manager.route('/web_crawl', methods=['POST'])  # noqa: F821

web/src/components/file-upload-modal/index.tsx

@@ -2,8 +2,10 @@ import { useTranslate } from '@/hooks/common-hooks';
 import { IModalProps } from '@/interfaces/common';
 import { InboxOutlined } from '@ant-design/icons';
 import {
+  Checkbox,
   Flex,
   Modal,
+  Progress,
   Segmented,
   Tabs,
   TabsProps,
@@ -21,10 +23,12 @@ const FileUpload = ({
   directory,
   fileList,
   setFileList,
+  uploadProgress,
 }: {
   directory: boolean;
   fileList: UploadFile[];
   setFileList: Dispatch<SetStateAction<UploadFile[]>>;
+  uploadProgress: number;
 }) => {
   const { t } = useTranslate('fileManager');
   const props: UploadProps = {
@@ -35,7 +39,7 @@ const FileUpload = ({
       newFileList.splice(index, 1);
       setFileList(newFileList);
     },
-    beforeUpload: (file) => {
+    beforeUpload: (file: UploadFile) => {
       setFileList((pre) => {
         return [...pre, file];
       });
@@ -44,38 +48,59 @@ const FileUpload = ({
     },
     directory,
     fileList,
+    progress: {
+      strokeWidth: 2,
+    },
   };
 
   return (
-    <Dragger {...props} className={styles.uploader}>
-      <p className="ant-upload-drag-icon">
-        <InboxOutlined />
-      </p>
-      <p className="ant-upload-text">{t('uploadTitle')}</p>
-      <p className="ant-upload-hint">{t('uploadDescription')}</p>
-      {false && <p className={styles.uploadLimit}>{t('uploadLimit')}</p>}
-    </Dragger>
+    <>
+      <Progress percent={uploadProgress} showInfo={false} />
+      <Dragger {...props} className={styles.uploader}>
+        <p className="ant-upload-drag-icon">
+          <InboxOutlined />
+        </p>
+        <p className="ant-upload-text">{t('uploadTitle')}</p>
+        <p className="ant-upload-hint">{t('uploadDescription')}</p>
+        {false && <p className={styles.uploadLimit}>{t('uploadLimit')}</p>}
+      </Dragger>
+    </>
   );
 };
 
+interface IFileUploadModalProps extends IModalProps<boolean> {
+  uploadFileList: UploadFile[];
+  setUploadFileList: Dispatch<SetStateAction<UploadFile[]>>;
+  uploadProgress: number;
+  setUploadProgress: Dispatch<SetStateAction<number>>;
+}
+
 const FileUploadModal = ({
   visible,
   hideModal,
   loading,
   onOk: onFileUploadOk,
-}: IModalProps<UploadFile[]>) => {
+  uploadFileList: fileList,
+  setUploadFileList: setFileList,
+  uploadProgress,
+  setUploadProgress,
+}: IFileUploadModalProps) => {
   const { t } = useTranslate('fileManager');
   const [value, setValue] = useState<string | number>('local');
-  const [fileList, setFileList] = useState<UploadFile[]>([]);
-  const [directoryFileList, setDirectoryFileList] = useState<UploadFile[]>([]);
+  const [parseOnCreation, setParseOnCreation] = useState(false);
 
   const clearFileList = () => {
     setFileList([]);
-    setDirectoryFileList([]);
+    setUploadProgress(0);
   };
 
   const onOk = async () => {
-    const ret = await onFileUploadOk?.([...fileList, ...directoryFileList]);
+    if (uploadProgress === 100) {
+      hideModal?.();
+      return;
+    }
+
+    const ret = await onFileUploadOk?.(parseOnCreation);
     return ret;
   };
 
@@ -92,6 +117,7 @@ const FileUploadModal = ({
           directory={false}
           fileList={fileList}
           setFileList={setFileList}
+          uploadProgress={uploadProgress}
         ></FileUpload>
       ),
     },
@@ -101,8 +127,9 @@ const FileUploadModal = ({
       children: (
         <FileUpload
           directory
-          fileList={directoryFileList}
-          setFileList={setDirectoryFileList}
+          fileList={fileList}
+          setFileList={setFileList}
+          uploadProgress={uploadProgress}
         ></FileUpload>
       ),
     },
@@ -129,7 +156,15 @@ const FileUploadModal = ({
             onChange={setValue}
           />
           {value === 'local' ? (
-            <Tabs defaultActiveKey="1" items={items} />
+            <>
+              <Checkbox
+                checked={parseOnCreation}
+                onChange={(e) => setParseOnCreation(e.target.checked)}
+              >
+                {t('parseOnCreation')}
+              </Checkbox>
+              <Tabs defaultActiveKey="1" items={items} />
+            </>
           ) : (
             t('comingSoon', { keyPrefix: 'common' })
           )}

web/src/hooks/document-hooks.ts

@@ -248,60 +248,27 @@ export const useUploadNextDocument = () => {
   } = useMutation({
     mutationKey: ['uploadDocument'],
     mutationFn: async (fileList: UploadFile[]) => {
-      const partitionedFileList = fileList.reduce<UploadFile[][]>(
-        (acc, cur, index) => {
-          const partIndex = Math.floor(index / 20); // Uploads 20 documents at a time
-          if (!acc[partIndex]) {
-            acc[partIndex] = [];
-          }
-          acc[partIndex].push(cur);
-          return acc;
-        },
-        [],
-      );
-
-      let allRet = [];
-      for (const listPart of partitionedFileList) {
-        const formData = new FormData();
-        formData.append('kb_id', knowledgeId);
-        listPart.forEach((file: any) => {
-          formData.append('file', file);
-        });
+      const formData = new FormData();
+      formData.append('kb_id', knowledgeId);
+      fileList.forEach((file: any) => {
+        formData.append('file', file);
+      });
 
-        try {
-          const ret = await kbService.document_upload(formData);
-          allRet.push(ret);
-        } catch (error) {
-          allRet.push({ data: { code: 500 } });
+      try {
+        const ret = await kbService.document_upload(formData);
+        const code = get(ret, 'data.code');
 
-          const filenames = listPart.map((file: any) => file.name).join(', ');
-          console.warn(error);
-          console.warn('Error uploading files:', filenames);
+        if (code === 0 || code === 500) {
+          queryClient.invalidateQueries({ queryKey: ['fetchDocumentList'] });
         }
+        return ret?.data;
+      } catch (error) {
+        console.warn(error);
+        return {
+          code: 500,
+          message: error + '',
+        };
       }
-
-      const succeed = allRet.every((ret) => get(ret, 'data.code') === 0);
-      const any500 = allRet.some((ret) => get(ret, 'data.code') === 500);
-
-      if (succeed) {
-        message.success(i18n.t('message.uploaded'));
-      }
-
-      if (succeed || any500) {
-        queryClient.invalidateQueries({ queryKey: ['fetchDocumentList'] });
-      }
-
-      const allData = {
-        code: any500
-          ? 500
-          : succeed
-            ? 0
-            : allRet.filter((ret) => get(ret, 'data.code') !== 0)[0]?.data
-                ?.code,
-        data: succeed,
-        message: allRet.map((ret) => get(ret, 'data.message')).join('/n'),
-      };
-      return allData;
     },
   });
 

web/src/locales/en.ts

@@ -126,9 +126,9 @@ export default {
       filesSelected: 'Files selected',
       upload: 'Upload',
       run: 'Parse',
-      runningStatus0: 'UNParsed',
-      runningStatus1: 'Parsing',
-      runningStatus2: 'CANCEL',
+      runningStatus0: 'PENDING',
+      runningStatus1: 'PARSING',
+      runningStatus2: 'CANCELED',
       runningStatus3: 'SUCCESS',
       runningStatus4: 'FAIL',
       pageRanges: 'Page Ranges',
@@ -743,6 +743,7 @@ This auto-tag feature enhances retrieval by adding another layer of domain-speci
       newFolder: 'New Folder',
       file: 'File',
       uploadFile: 'Upload File',
+      parseOnCreation: 'Parse on creation',
       directory: 'Directory',
       uploadTitle: 'Drag and drop your file here to upload',
       uploadDescription:

web/src/locales/es.ts

@@ -480,6 +480,7 @@ export default {
       newFolder: 'Nueva carpeta',
       file: 'Archivo',
       uploadFile: 'Subir archivo',
+      parseOnCreation: 'Ejecutar en la creación',
       directory: 'Directorio',
       uploadTitle: 'Haz clic o arrastra el archivo a esta área para subir',
       uploadDescription:

web/src/locales/id.ts

@@ -648,6 +648,7 @@ export default {
       newFolder: 'Folder Baru',
       file: 'File',
       uploadFile: 'Unggah File',
+      parseOnCreation: 'Memparsing saat dibuat',
       directory: 'Direktori',
       uploadTitle: 'Klik atau seret file ke area ini untuk mengunggah',
       uploadDescription:

web/src/locales/ja.ts

@@ -653,6 +653,7 @@ export default {
       newFolder: '新しいフォルダ',
       file: 'ファイル',
       uploadFile: 'ファイルをアップロード',
+      parseOnCreation: '作成時に解析',
       directory: 'ディレクトリ',
       uploadTitle: 'クリックまたはドラッグしてファイルをアップロード',
       uploadDescription:

web/src/locales/pt-br.ts

@@ -639,6 +639,7 @@ export default {
       newFolder: 'Nova Pasta',
       file: 'Arquivo',
       uploadFile: 'Carregar Arquivo',
+      parseOnCreation: 'Executar na criação',
       directory: 'Diretório',
       uploadTitle:
         'Clique ou arraste o arquivo para esta área para fazer o upload',

web/src/locales/vi.ts

@@ -707,6 +707,7 @@ export default {
       newFolder: 'Thư mục mới',
       file: 'Tệp',
       uploadFile: 'Tải tệp lên',
+      parseOnCreation: 'Phân tích khi tạo',
       directory: 'Thư mục',
       uploadTitle: 'Nhấp hoặc kéo thả tệp vào khu vực này để tải lên',
       uploadDescription:

web/src/locales/zh-traditional.ts

@@ -708,6 +708,7 @@ export default {
       pleaseSelect: '請選擇',
       newFolder: '新建文件夾',
       uploadFile: '上傳文件',
+      parseOnCreation: '創建時解析',
       uploadTitle: '點擊或拖拽文件至此區域即可上傳',
       uploadDescription:
         '支持單次或批量上傳。單個檔案大小不超過10MB，最多上傳128份檔案。',

web/src/locales/zh.ts

@@ -726,6 +726,7 @@ General：实体和关系提取提示来自 GitHub - microsoft/graphrag：基于
       pleaseSelect: '请选择',
       newFolder: '新建文件夹',
       uploadFile: '上传文件',
+      parseOnCreation: '创建时解析',
       uploadTitle: '点击或拖拽文件至此区域即可上传',
       uploadDescription:
         '支持单次或批量上传。 单个文件大小不超过10MB，最多上传128份文件。严禁上传违禁文件。',

web/src/pages/add-knowledge/components/knowledge-file/hooks.ts

@@ -10,7 +10,6 @@ import {
 } from '@/hooks/document-hooks';
 import { useGetKnowledgeSearchParams } from '@/hooks/route-hook';
 import { IChangeParserConfigRequestBody } from '@/interfaces/request/document';
-import { getUnSupportedFilesCount } from '@/utils/document-util';
 import { UploadFile } from 'antd';
 import { useCallback, useState } from 'react';
 import { useNavigate } from 'umi';
@@ -143,29 +142,103 @@ export const useHandleUploadDocument = () => {
     hideModal: hideDocumentUploadModal,
     showModal: showDocumentUploadModal,
   } = useSetModalState();
+  const [fileList, setFileList] = useState<UploadFile[]>([]);
+  const [uploadProgress, setUploadProgress] = useState<number>(0);
   const { uploadDocument, loading } = useUploadNextDocument();
+  const { runDocumentByIds, loading: _ } = useRunNextDocument();
 
   const onDocumentUploadOk = useCallback(
-    async (fileList: UploadFile[]): Promise<number | undefined> => {
-      if (fileList.length > 0) {
-        const ret: any = await uploadDocument(fileList);
-        if (typeof ret?.message !== 'string') {
-          return;
-        }
-        const count = getUnSupportedFilesCount(ret?.message);
-        /// 500 error code indicates that some file types are not supported
-        let code = ret?.code;
-        if (
-          ret?.code === 0 ||
-          (ret?.code === 500 && count !== fileList.length) // Some files were not uploaded successfully, but some were uploaded successfully.
-        ) {
-          code = 0;
-          hideDocumentUploadModal();
-        }
-        return code;
+    async (parseOnCreation: boolean): Promise<number | undefined> => {
+      const processFileGroup = async (filesPart: UploadFile[]) => {
+        // set status to uploading on files
+        setFileList(
+          fileList.map((file) => {
+            if (!filesPart.includes(file)) {
+              return file;
+            }
+
+            let newFile = file;
+            newFile.status = 'uploading';
+            newFile.percent = 1;
+            return newFile;
+          }),
+        );
+
+        const ret = await uploadDocument(filesPart);
+
+        const files = ret?.data || [];
+        const succesfulFilenames = files.map((file: any) => file.name);
+
+        // set status to done or error on files (based on response)
+        setFileList(
+          fileList.map((file) => {
+            if (!filesPart.includes(file)) {
+              return file;
+            }
+
+            let newFile = file;
+            newFile.status = succesfulFilenames.includes(file.name)
+              ? 'done'
+              : 'error';
+            newFile.percent = 100;
+            newFile.response = ret.message;
+            return newFile;
+          }),
+        );
+
+        return {
+          code: ret?.code,
+          fileIds: files.map((file: any) => file.id),
+          totalSuccess: succesfulFilenames.length,
+        };
+      };
+
+      const totalFiles = fileList.length;
+
+      if (totalFiles === 0) {
+        console.log('No files to upload');
+        hideDocumentUploadModal();
+        return 0;
+      }
+
+      let totalSuccess = 0;
+      let codes = [];
+      let toRunFileIds: any[] = [];
+      for (let i = 0; i < totalFiles; i += 10) {
+        setUploadProgress(Math.floor((i / totalFiles) * 100));
+        const files = fileList.slice(i, i + 10);
+        const {
+          code,
+          totalSuccess: count,
+          fileIds,
+        } = await processFileGroup(files);
+        codes.push(code);
+        totalSuccess += count;
+        toRunFileIds = toRunFileIds.concat(fileIds);
+      }
+
+      const allSuccess = codes.every((code) => code === 0);
+      const any500 = codes.some((code) => code === 500);
+
+      let code = 500;
+      if (allSuccess || (any500 && totalSuccess === totalFiles)) {
+        code = 0;
+        hideDocumentUploadModal();
+      }
+
+      if (parseOnCreation) {
+        await runDocumentByIds({
+          documentIds: toRunFileIds,
+          run: 1,
+          shouldDelete: false,
+        });
       }
+
+      setUploadProgress(100);
+
+      return code;
     },
-    [uploadDocument, hideDocumentUploadModal],
+    [uploadDocument, hideDocumentUploadModal, fileList],
   );
 
   return {
@@ -174,6 +247,10 @@ export const useHandleUploadDocument = () => {
     documentUploadVisible,
     hideDocumentUploadModal,
     showDocumentUploadModal,
+    uploadFileList: fileList,
+    setUploadFileList: setFileList,
+    uploadProgress,
+    setUploadProgress,
   };
 };
 

web/src/pages/add-knowledge/components/knowledge-file/index.tsx

@@ -69,6 +69,10 @@ const KnowledgeFile = () => {
     showDocumentUploadModal,
     onDocumentUploadOk,
     documentUploadLoading,
+    uploadFileList,
+    setUploadFileList,
+    uploadProgress,
+    setUploadProgress,
   } = useHandleUploadDocument();
   const {
     webCrawlUploadVisible,
@@ -229,6 +233,10 @@ const KnowledgeFile = () => {
         hideModal={hideDocumentUploadModal}
         loading={documentUploadLoading}
         onOk={onDocumentUploadOk}
+        uploadFileList={uploadFileList}
+        setUploadFileList={setUploadFileList}
+        uploadProgress={uploadProgress}
+        setUploadProgress={setUploadProgress}
       ></FileUploadModal>
       <WebCrawlModal
         visible={webCrawlUploadVisible}