瀏覽代碼

fix: big xls file error (#4859)

### What problem does this PR solve?

if *.xls file is too large, .eg >50M, I get error.

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.17.0
SkyfireWXY 8 月之前
父節點
當前提交
8fcca1b958
沒有連結到貢獻者的電子郵件帳戶。
共有 1 個檔案被更改,包括 72 行新增11 行删除
  1. 72
    11
      deepdoc/parser/excel_parser.py

+ 72
- 11
deepdoc/parser/excel_parser.py 查看文件

#
# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License. # you may not use this file except in compliance with the License.
# You may obtain a copy of the License at # You may obtain a copy of the License at
# limitations under the License. # limitations under the License.
# #


from openpyxl import load_workbook
from openpyxl import load_workbook, Workbook
import sys import sys
from io import BytesIO from io import BytesIO


from rag.nlp import find_codec from rag.nlp import find_codec


import pandas as pd



class RAGFlowExcelParser: class RAGFlowExcelParser:
def html(self, fnm, chunk_rows=256): def html(self, fnm, chunk_rows=256):
if isinstance(fnm, str):
wb = load_workbook(fnm)

# if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))++

s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else: else:
wb = load_workbook(BytesIO(fnm))
pass

try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
# if len(wb.worksheets) > 0:
# del wb.worksheets[0]
# else: pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass


tb_chunks = [] tb_chunks = []
for sheetname in wb.sheetnames: for sheetname in wb.sheetnames:
tb += f"<table><caption>{sheetname}</caption>" tb += f"<table><caption>{sheetname}</caption>"
tb += tb_rows_0 tb += tb_rows_0
for r in list( for r in list(
rows[1 + chunk_i * chunk_rows : 1 + (chunk_i + 1) * chunk_rows]
rows[1 + chunk_i * chunk_rows: 1 + (chunk_i + 1) * chunk_rows]
): ):
tb += "<tr>" tb += "<tr>"
for i, c in enumerate(r): for i, c in enumerate(r):
return tb_chunks return tb_chunks


def __call__(self, fnm): def __call__(self, fnm):
if isinstance(fnm, str):
wb = load_workbook(fnm)
# if isinstance(fnm, str):
# wb = load_workbook(fnm)
# else:
# wb = load_workbook(BytesIO(fnm))

s_fnm = fnm
if not isinstance(fnm, str):
s_fnm = BytesIO(fnm)
else: else:
wb = load_workbook(BytesIO(fnm))
pass

try:
wb = load_workbook(s_fnm)
except Exception as e:
print(f'****wxy: file parser error: {e}, s_fnm={s_fnm}, trying convert files')
df = pd.read_excel(s_fnm)
wb = Workbook()
if len(wb.worksheets) > 0:
del wb.worksheets[0]
else:
pass
ws = wb.active
ws.title = "Data"
for col_num, column_name in enumerate(df.columns, 1):
ws.cell(row=1, column=col_num, value=column_name)
else:
pass
for row_num, row in enumerate(df.values, 2):
for col_num, value in enumerate(row, 1):
ws.cell(row=row_num, column=col_num, value=value)
else:
pass
else:
pass

res = [] res = []
for sheetname in wb.sheetnames: for sheetname in wb.sheetnames:
ws = wb[sheetname] ws = wb[sheetname]
if __name__ == "__main__": if __name__ == "__main__":
psr = RAGFlowExcelParser() psr = RAGFlowExcelParser()
psr(sys.argv[1]) psr(sys.argv[1])


Loading…
取消
儲存