Browse Source

Fix: fix special delimiter parsing issue (#5448)

### What problem does this PR solve?

Fix special delimiter parsing issue #5382 

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.17.0
Yongteng Lei 8 months ago
parent
commit
83d0949498
No account linked to committer's email address
1 changed files with 3 additions and 1 deletions
  1. 3
    1
      deepdoc/parser/txt_parser.py

+ 3
- 1
deepdoc/parser/txt_parser.py View File

@@ -51,11 +51,13 @@ class RAGFlowTxtParser:
s = t
if s < len(delimiter):
dels.extend(list(delimiter[s:]))
dels = [re.escape(d) for d in delimiter if d]
dels = [re.escape(d) for d in dels if d]
dels = [d for d in dels if d]
dels = "|".join(dels)
secs = re.split(r"(%s)" % dels, txt)
for sec in secs:
if re.match(f"^{dels}$", sec):
continue
add_chunk(sec)

return [[c, ""] for c in cks]

Loading…
Cancel
Save