浏览代码

Fix nursery.start_soon. Close #5575 (#5591)

### What problem does this PR solve?

Fix nursery.start_soon. Close #5575

### Type of change

- [x] Bug Fix (non-breaking change which fixes an issue)
tags/v0.17.1
Zhichang Yu 8 个月前
父节点
当前提交
4d6484b03e
没有帐户链接到提交者的电子邮件
共有 4 个文件被更改,包括 9 次插入9 次删除
  1. 1
    1
      graphrag/entity_resolution.py
  2. 3
    3
      graphrag/general/extractor.py
  3. 2
    2
      graphrag/general/mind_map_extractor.py
  4. 3
    3
      rag/svr/task_executor.py

+ 1
- 1
graphrag/entity_resolution.py 查看文件

@@ -99,7 +99,7 @@ class EntityResolution(Extractor):
for candidate_resolution_i in candidate_resolution.items():
if not candidate_resolution_i[1]:
continue
nursery.start_soon(self._resolve_candidate(candidate_resolution_i, resolution_result))
nursery.start_soon(lambda: self._resolve_candidate(candidate_resolution_i, resolution_result))

connect_graph = nx.Graph()
removed_entities = []

+ 3
- 3
graphrag/general/extractor.py 查看文件

@@ -101,7 +101,7 @@ class Extractor:
async with trio.open_nursery() as nursery:
for i, (cid, ck) in enumerate(chunks):
ck = truncate(ck, int(self._llm.max_length*0.8))
nursery.start_soon(self._process_single_content, (cid, ck), i, len(chunks), out_results)
nursery.start_soon(lambda: self._process_single_content((cid, ck), i, len(chunks), out_results))

maybe_nodes = defaultdict(list)
maybe_edges = defaultdict(list)
@@ -120,7 +120,7 @@ class Extractor:
all_entities_data = []
async with trio.open_nursery() as nursery:
for en_nm, ents in maybe_nodes.items():
nursery.start_soon(self._merge_nodes, en_nm, ents, all_entities_data)
nursery.start_soon(lambda: self._merge_nodes(en_nm, ents, all_entities_data))
now = trio.current_time()
if callback:
callback(msg = f"Entities merging done, {now-start_ts:.2f}s.")
@@ -130,7 +130,7 @@ class Extractor:
all_relationships_data = []
async with trio.open_nursery() as nursery:
for (src, tgt), rels in maybe_edges.items():
nursery.start_soon(self._merge_edges, src, tgt, rels, all_relationships_data)
nursery.start_soon(lambda: self._merge_edges(src, tgt, rels, all_relationships_data))
now = trio.current_time()
if callback:
callback(msg = f"Relationships merging done, {now-start_ts:.2f}s.")

+ 2
- 2
graphrag/general/mind_map_extractor.py 查看文件

@@ -93,13 +93,13 @@ class MindMapExtractor(Extractor):
for i in range(len(sections)):
section_cnt = num_tokens_from_string(sections[i])
if cnt + section_cnt >= token_count and texts:
nursery.start_soon(self._process_document, "".join(texts), prompt_variables, res)
nursery.start_soon(lambda: self._process_document("".join(texts), prompt_variables, res))
texts = []
cnt = 0
texts.append(sections[i])
cnt += section_cnt
if texts:
nursery.start_soon(self._process_document, "".join(texts), prompt_variables, res)
nursery.start_soon(lambda: self._process_document("".join(texts), prompt_variables, res))
if not res:
return MindMapResult(output={"id": "root", "children": []})
merge_json = reduce(self._merge, res)

+ 3
- 3
rag/svr/task_executor.py 查看文件

@@ -297,7 +297,7 @@ async def build_chunks(task, progress_callback):
return
async with trio.open_nursery() as nursery:
for d in docs:
nursery.start_soon(doc_keyword_extraction, chat_mdl, d, task["parser_config"]["auto_keywords"])
nursery.start_soon(lambda: doc_keyword_extraction(chat_mdl, d, task["parser_config"]["auto_keywords"]))
progress_callback(msg="Keywords generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))

if task["parser_config"].get("auto_questions", 0):
@@ -316,7 +316,7 @@ async def build_chunks(task, progress_callback):
d["question_tks"] = rag_tokenizer.tokenize("\n".join(d["question_kwd"]))
async with trio.open_nursery() as nursery:
for d in docs:
nursery.start_soon(doc_question_proposal, chat_mdl, d, task["parser_config"]["auto_questions"])
nursery.start_soon(lambda: doc_question_proposal(chat_mdl, d, task["parser_config"]["auto_questions"]))
progress_callback(msg="Question generation {} chunks completed in {:.2f}s".format(len(docs), timer() - st))

if task["kb_parser_config"].get("tag_kb_ids", []):
@@ -356,7 +356,7 @@ async def build_chunks(task, progress_callback):
d[TAG_FLD] = json.loads(cached)
async with trio.open_nursery() as nursery:
for d in docs_to_tag:
nursery.start_soon(doc_content_tagging, chat_mdl, d, topn_tags)
nursery.start_soon(lambda: doc_content_tagging(chat_mdl, d, topn_tags))
progress_callback(msg="Tagging {} chunks completed in {:.2f}s".format(len(docs), timer() - st))

return docs

正在加载...
取消
保存