Explorar el Código

Refa: ollama keep alive issue. (#8216)

### What problem does this PR solve?

#8122

### Type of change

- [x] Refactoring
tags/v0.19.1
Kevin Hu hace 4 meses
padre
commit
d5236b71f4
No account linked to committer's email address
Se han modificado 2 ficheros con 6 adiciones y 4 borrados
  1. 2
    2
      rag/llm/chat_model.py
  2. 4
    2
      rag/llm/cv_model.py

+ 2
- 2
rag/llm/chat_model.py Ver fichero

ctx_size = self._calculate_dynamic_ctx(history) ctx_size = self._calculate_dynamic_ctx(history)


gen_conf["num_ctx"] = ctx_size gen_conf["num_ctx"] = ctx_size
response = self.client.chat(model=self.model_name, messages=history, options=gen_conf)
response = self.client.chat(model=self.model_name, messages=history, options=gen_conf, keep_alive=-1)
ans = response["message"]["content"].strip() ans = response["message"]["content"].strip()
token_count = response.get("eval_count", 0) + response.get("prompt_eval_count", 0) token_count = response.get("eval_count", 0) + response.get("prompt_eval_count", 0)
return ans, token_count return ans, token_count


ans = "" ans = ""
try: try:
response = self.client.chat(model=self.model_name, messages=history, stream=True, options=options)
response = self.client.chat(model=self.model_name, messages=history, stream=True, options=options, keep_alive=-1)
for resp in response: for resp in response:
if resp["done"]: if resp["done"]:
token_count = resp.get("prompt_eval_count", 0) + resp.get("eval_count", 0) token_count = resp.get("prompt_eval_count", 0) + resp.get("eval_count", 0)

+ 4
- 2
rag/llm/cv_model.py Ver fichero

response = self.client.chat( response = self.client.chat(
model=self.model_name, model=self.model_name,
messages=history, messages=history,
options=options
options=options,
keep_alive=-1
) )


ans = response["message"]["content"].strip() ans = response["message"]["content"].strip()
model=self.model_name, model=self.model_name,
messages=history, messages=history,
stream=True, stream=True,
options=options
options=options,
keep_alive=-1
) )
for resp in response: for resp in response:
if resp["done"]: if resp["done"]:

Cargando…
Cancelar
Guardar