Browse Source

add callback to entity extraction (#1843)

### What problem does this PR solve?

### Type of change

- [x] Refactoring
- [ ]
tags/v0.10.0
Kevin Hu 1 year ago
parent
commit
a199572bf8
No account linked to committer's email address
1 changed files with 4 additions and 2 deletions
  1. 4
    2
      graphrag/index.py

+ 4
- 2
graphrag/index.py View File

@@ -86,13 +86,15 @@ def build_knowlege_graph_chunks(tenant_id: str, chunks: List[str], callback, ent
for i in range(len(chunks)):
tkn_cnt = num_tokens_from_string(chunks[i])
if cnt+tkn_cnt >= left_token_count and texts:
threads.append(exe.submit(ext, texts, {"entity_types": entity_types}, callback))
for b in range(0, len(texts), 16):
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))
texts = []
cnt = 0
texts.append(chunks[i])
cnt += tkn_cnt
if texts:
threads.append(exe.submit(ext, texts))
for b in range(0, len(texts), 16):
threads.append(exe.submit(ext, ["\n".join(texts[b:b+16])], {"entity_types": entity_types}, callback))

callback(0.5, "Extracting entities.")
graphs = []

Loading…
Cancel
Save