Browse Source

smooth term weight (#3510)

### What problem does this PR solve?

#3499

### Type of change

- [x] Performance Improvement
tags/v0.14.0
Kevin Hu 11 months ago
parent
commit
289034f36e
1 changed files with 2 additions and 2 deletions
  1. 2
    2
      rag/nlp/term_weight.py

+ 2
- 2
rag/nlp/term_weight.py View File

@@ -228,7 +228,7 @@ class Dealer:
idf2 = np.array([idf(df(t), 1000000000) for t in tks])
wts = (0.3 * idf1 + 0.7 * idf2) * \
np.array([ner(t) * postag(t) for t in tks])
wts = [math.exp(s) for s in wts]
wts = [math.pow(s, 2) for s in wts]
tw = list(zip(tks, wts))
else:
for tk in tks:
@@ -237,7 +237,7 @@ class Dealer:
idf2 = np.array([idf(df(t), 1000000000) for t in tt])
wts = (0.3 * idf1 + 0.7 * idf2) * \
np.array([ner(t) * postag(t) for t in tt])
wts = [math.exp(s) for s in wts]
wts = [math.pow(s, 2) for s in wts]
tw.extend(zip(tt, wts))

S = np.sum([s for _, s in tw])

Loading…
Cancel
Save