### What problem does this PR solve? #6085 ### Type of change - [x] Bug Fix (non-breaking change which fixes an issue)tags/v0.18.0
| @@ -118,6 +118,8 @@ class RagTokenizer: | |||
| def dfs_(self, chars, s, preTks, tkslist): | |||
| res = s | |||
| if len(tkslist) >= 2048: | |||
| return res | |||
| # if s > MAX_L or s>= len(chars): | |||
| if s >= len(chars): | |||
| tkslist.append(preTks) | |||