Co-authored-by: lizb <lizb@sugon.com>tags/1.5.0
| @@ -512,6 +512,20 @@ const StepTwo = ({ | |||
| setOverlap(overlap!) | |||
| setRules(rules.pre_processing_rules) | |||
| setDefaultConfig(rules) | |||
| if (documentDetail.dataset_process_rule.mode === 'hierarchical') { | |||
| setParentChildConfig({ | |||
| chunkForContext: rules.parent_mode || 'paragraph', | |||
| parent: { | |||
| delimiter: escape(rules.segmentation.separator), | |||
| maxLength: rules.segmentation.max_tokens, | |||
| }, | |||
| child: { | |||
| delimiter: escape(rules.subchunk_segmentation.separator), | |||
| maxLength: rules.subchunk_segmentation.max_tokens, | |||
| }, | |||
| }) | |||
| } | |||
| } | |||
| } | |||