| @@ -416,9 +416,14 @@ class IndexingRunner: | |||
| if separator: | |||
| separator = separator.replace('\\n', '\n') | |||
| if 'chunk_overlap' in segmentation and segmentation['chunk_overlap']: | |||
| chunk_overlap = segmentation['chunk_overlap'] | |||
| else: | |||
| chunk_overlap = 0 | |||
| character_splitter = FixedRecursiveCharacterTextSplitter.from_encoder( | |||
| chunk_size=segmentation["max_tokens"], | |||
| chunk_overlap=segmentation.get('chunk_overlap', 0), | |||
| chunk_overlap=chunk_overlap, | |||
| fixed_separator=separator, | |||
| separators=["\n\n", "。", ".", " ", ""], | |||
| embedding_model_instance=embedding_model_instance | |||