浏览代码

fix: big xls file error (#4859)

### What problem does this PR solve?

if *.xls file is too large, .eg >50M, I get error.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.17.0
SkyfireWXY 8 个月前
父节点
当前提交
8fcca1b958
没有帐户链接到提交者的电子邮件
共有 1 个文件被更改,包括 72 次插入11 次删除
  1. 72
    11
      deepdoc/parser/excel_parser.py

+ 72
- 11
deepdoc/parser/excel_parser.py 查看文件

@@ -1,6 +1,3 @@
#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
@@ -14,19 +11,51 @@
# limitations under the License.
#

from openpyxl import load_workbook
from openpyxl import load_workbook, Workbook
import sys
from io import BytesIO

from rag.nlp import find_codec

import pandas as pd


class RAGFlowExcelParser:
def html(self, fnm, chunk_rows=256):
if isinstance(fnm, str):
wb = load_workbook(fnm)

# if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))++

s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else:
wb = load_workbook(BytesIO(fnm))
pass

try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
# if len(wb.worksheets) > 0:
# del wb.worksheets[0]
# else: pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass

tb_chunks = []
for sheetname in wb.sheetnames:
@@ -45,7 +74,7 @@ class RAGFlowExcelParser:
tb += f"<table><caption>{sheetname}</caption>"
tb += tb_rows_0
for r in list(
rows[1 + chunk_i * chunk_rows : 1 + (chunk_i + 1) * chunk_rows]
rows[1 + chunk_i * chunk_rows: 1 + (chunk_i + 1) * chunk_rows]
):
tb += "<tr>"
for i, c in enumerate(r):
@@ -60,10 +89,41 @@ class RAGFlowExcelParser:
return tb_chunks

def __call__(self, fnm):
if isinstance(fnm, str):
wb = load_workbook(fnm)
# if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))

s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else:
wb = load_workbook(BytesIO(fnm))
pass

try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
if len(wb.worksheets) > 0:
del wb.worksheets[0]
else:
pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass

res = []
for sheetname in wb.sheetnames:
ws = wb[sheetname]
@@ -104,3 +164,4 @@ class RAGFlowExcelParser:
if __name__ == "__main__":
psr = RAGFlowExcelParser()
psr(sys.argv[1])


正在加载...
取消
保存