Browse Source

refine text decode (#657)

### What problem does this PR solve?
#651 

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.5.0
KevinHuSh 1 year ago
parent
commit
7013d7f620
No account linked to committer's email address
7 changed files with 7 additions and 7 deletions
  1. 1
    1
      deepdoc/parser/excel_parser.py
  2. 1
    1
      rag/app/book.py
  3. 1
    1
      rag/app/laws.py
  4. 1
    1
      rag/app/naive.py
  5. 1
    1
      rag/app/one.py
  6. 1
    1
      rag/app/qa.py
  7. 1
    1
      rag/app/table.py

+ 1
- 1
deepdoc/parser/excel_parser.py View File



if fnm.split(".")[-1].lower() in ["csv", "txt"]: if fnm.split(".")[-1].lower() in ["csv", "txt"]:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
return len(txt.split("\n")) return len(txt.split("\n"))





+ 1
- 1
rag/app/book.py View File

txt = "" txt = ""
if binary: if binary:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
else: else:
with open(filename, "r") as f: with open(filename, "r") as f:
while True: while True:

+ 1
- 1
rag/app/laws.py View File

txt = "" txt = ""
if binary: if binary:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
else: else:
with open(filename, "r") as f: with open(filename, "r") as f:
while True: while True:

+ 1
- 1
rag/app/naive.py View File

txt = "" txt = ""
if binary: if binary:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
else: else:
with open(filename, "r") as f: with open(filename, "r") as f:
while True: while True:

+ 1
- 1
rag/app/one.py View File

txt = "" txt = ""
if binary: if binary:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
else: else:
with open(filename, "r") as f: with open(filename, "r") as f:
while True: while True:

+ 1
- 1
rag/app/qa.py View File

txt = "" txt = ""
if binary: if binary:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
else: else:
with open(filename, "r") as f: with open(filename, "r") as f:
while True: while True:

+ 1
- 1
rag/app/table.py View File

txt = "" txt = ""
if binary: if binary:
encoding = find_codec(binary) encoding = find_codec(binary)
txt = binary.decode(encoding)
txt = binary.decode(encoding, errors="ignore")
else: else:
with open(filename, "r") as f: with open(filename, "r") as f:
while True: while True:

Loading…
Cancel
Save