|
|
|
|
|
|
|
|
ress = [] |
|
|
ress = [] |
|
|
total_tokens = 0 |
|
|
total_tokens = 0 |
|
|
for i in range(0, len(texts), batch_size): |
|
|
for i in range(0, len(texts), batch_size): |
|
|
res = self.client.embeddings.create(input=texts[i : i + batch_size], model=self.model_name) |
|
|
|
|
|
|
|
|
res = None |
|
|
try: |
|
|
try: |
|
|
|
|
|
res = self.client.embeddings.create(input=texts[i : i + batch_size], model=self.model_name) |
|
|
ress.extend([d.embedding for d in res.data]) |
|
|
ress.extend([d.embedding for d in res.data]) |
|
|
total_tokens += self.total_token_count(res) |
|
|
total_tokens += self.total_token_count(res) |
|
|
except Exception as _e: |
|
|
except Exception as _e: |
|
|
|
|
|
|
|
|
return np.array(ress), total_tokens |
|
|
return np.array(ress), total_tokens |
|
|
|
|
|
|
|
|
def encode_queries(self, text): |
|
|
def encode_queries(self, text): |
|
|
res = self.client.embeddings.create(input=[text], model=self.model_name) |
|
|
|
|
|
|
|
|
res = None |
|
|
try: |
|
|
try: |
|
|
|
|
|
res = self.client.embeddings.create(input=[text], model=self.model_name) |
|
|
return np.array(res.data[0].embedding), self.total_token_count(res) |
|
|
return np.array(res.data[0].embedding), self.total_token_count(res) |
|
|
except Exception as _e: |
|
|
except Exception as _e: |
|
|
log_exception(_e, res) |
|
|
log_exception(_e, res) |