|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class MindMapExtractor: |
|
|
class MindMapExtractor: |
|
|
|
|
|
|
|
|
_llm: CompletionLLM |
|
|
_llm: CompletionLLM |
|
|
_input_text_key: str |
|
|
_input_text_key: str |
|
|
_mind_map_prompt: str |
|
|
_mind_map_prompt: str |
|
|
|
|
|
|
|
|
max_workers = int(os.environ.get('MINDMAP_EXTRACTOR_MAX_WORKERS', 12)) |
|
|
max_workers = int(os.environ.get('MINDMAP_EXTRACTOR_MAX_WORKERS', 12)) |
|
|
exe = ThreadPoolExecutor(max_workers=max_workers) |
|
|
exe = ThreadPoolExecutor(max_workers=max_workers) |
|
|
threads = [] |
|
|
threads = [] |
|
|
token_count = max(self._llm.max_length * 0.8, self._llm.max_length-512) |
|
|
|
|
|
|
|
|
token_count = max(self._llm.max_length * 0.8, self._llm.max_length - 512) |
|
|
texts = [] |
|
|
texts = [] |
|
|
res = [] |
|
|
res = [] |
|
|
cnt = 0 |
|
|
cnt = 0 |
|
|
|
|
|
|
|
|
elif isinstance(value, list): |
|
|
elif isinstance(value, list): |
|
|
new_value = {} |
|
|
new_value = {} |
|
|
for i in range(len(value)): |
|
|
for i in range(len(value)): |
|
|
if isinstance(value[i], list): |
|
|
|
|
|
|
|
|
if isinstance(value[i], list) and i > 0: |
|
|
new_value[value[i - 1]] = value[i][0] |
|
|
new_value[value[i - 1]] = value[i][0] |
|
|
data[key] = new_value |
|
|
data[key] = new_value |
|
|
else: |
|
|
else: |
|
|
continue |
|
|
continue |
|
|
return data |
|
|
return data |
|
|
|
|
|
|
|
|
def _todict(self, layer:collections.OrderedDict): |
|
|
|
|
|
|
|
|
def _todict(self, layer: collections.OrderedDict): |
|
|
to_ret = layer |
|
|
to_ret = layer |
|
|
if isinstance(layer, collections.OrderedDict): |
|
|
if isinstance(layer, collections.OrderedDict): |
|
|
to_ret = dict(layer) |
|
|
to_ret = dict(layer) |