Browse Source

Integrates LLM Azure OpenAI (#1318)

### What problem does this PR solve?

feat: Integrates LLM Azure OpenAI #716 

### Type of change

- [x] New Feature (non-breaking change which adds functionality)

### Other
It's just the back-end code, the front-end needs to provide the Azure
OpenAI model addition form.
   
#### Required parameters

- base_url
- api_key

---------

Co-authored-by: yonghui li <yonghui.li@bondex.com.cn>
tags/v0.8.0
LiYongHui 1 year ago
parent
commit
a6765e9ca4
No account linked to committer's email address
6 changed files with 123 additions and 0 deletions
  1. 82
    0
      api/db/init_data.py
  2. 6
    0
      api/settings.py
  3. 3
    0
      rag/llm/__init__.py
  4. 6
    0
      rag/llm/chat_model.py
  5. 20
    0
      rag/llm/cv_model.py
  6. 6
    0
      rag/llm/embedding_model.py

+ 82
- 0
api/db/init_data.py View File

"logo": "", "logo": "",
"tags": "LLM,TEXT EMBEDDING", "tags": "LLM,TEXT EMBEDDING",
"status": "1", "status": "1",
},{
"name": "Azure-OpenAI",
"logo": "",
"tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION",
"status": "1",
} }
# { # {
# "name": "文心一言", # "name": "文心一言",
"max_tokens": 8192, "max_tokens": 8192,
"model_type": LLMType.EMBEDDING "model_type": LLMType.EMBEDDING
}, },
# ------------------------ Azure OpenAI -----------------------
# Please ensure the llm_name is the same as the name in Azure
# OpenAI deployment name (e.g., azure-gpt-4o). And the llm_name
# must different from the OpenAI llm_name
#
# Each model must be deployed in the Azure OpenAI service, otherwise,
# you will receive an error message 'The API deployment for
# this resource does not exist'
{
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-4o",
"tags": "LLM,CHAT,128K",
"max_tokens": 128000,
"model_type": LLMType.CHAT.value + "," + LLMType.IMAGE2TEXT.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-35-turbo",
"tags": "LLM,CHAT,4K",
"max_tokens": 4096,
"model_type": LLMType.CHAT.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-35-turbo-16k",
"tags": "LLM,CHAT,16k",
"max_tokens": 16385,
"model_type": LLMType.CHAT.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-text-embedding-ada-002",
"tags": "TEXT EMBEDDING,8K",
"max_tokens": 8191,
"model_type": LLMType.EMBEDDING.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-text-embedding-3-small",
"tags": "TEXT EMBEDDING,8K",
"max_tokens": 8191,
"model_type": LLMType.EMBEDDING.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-text-embedding-3-large",
"tags": "TEXT EMBEDDING,8K",
"max_tokens": 8191,
"model_type": LLMType.EMBEDDING.value
},{
"fid": factory_infos[15]["name"],
"llm_name": "azure-whisper-1",
"tags": "SPEECH2TEXT",
"max_tokens": 25 * 1024 * 1024,
"model_type": LLMType.SPEECH2TEXT.value
},
{
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-4",
"tags": "LLM,CHAT,8K",
"max_tokens": 8191,
"model_type": LLMType.CHAT.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-4-turbo",
"tags": "LLM,CHAT,8K",
"max_tokens": 8191,
"model_type": LLMType.CHAT.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-4-32k",
"tags": "LLM,CHAT,32K",
"max_tokens": 32768,
"model_type": LLMType.CHAT.value
}, {
"fid": factory_infos[15]["name"],
"llm_name": "azure-gpt-4-vision-preview",
"tags": "LLM,CHAT,IMAGE2TEXT",
"max_tokens": 765,
"model_type": LLMType.IMAGE2TEXT.value
},
] ]
for info in factory_infos: for info in factory_infos:
try: try:

+ 6
- 0
api/settings.py View File

"image2text_model": "gpt-4-vision-preview", "image2text_model": "gpt-4-vision-preview",
"asr_model": "whisper-1", "asr_model": "whisper-1",
}, },
"Azure-OpenAI": {
"chat_model": "azure-gpt-35-turbo",
"embedding_model": "azure-text-embedding-ada-002",
"image2text_model": "azure-gpt-4-vision-preview",
"asr_model": "azure-whisper-1",
},
"ZHIPU-AI": { "ZHIPU-AI": {
"chat_model": "glm-3-turbo", "chat_model": "glm-3-turbo",
"embedding_model": "embedding-2", "embedding_model": "embedding-2",

+ 3
- 0
rag/llm/__init__.py View File

EmbeddingModel = { EmbeddingModel = {
"Ollama": OllamaEmbed, "Ollama": OllamaEmbed,
"OpenAI": OpenAIEmbed, "OpenAI": OpenAIEmbed,
"Azure-OpenAI": AzureEmbed,
"Xinference": XinferenceEmbed, "Xinference": XinferenceEmbed,
"Tongyi-Qianwen": QWenEmbed, "Tongyi-Qianwen": QWenEmbed,
"ZHIPU-AI": ZhipuEmbed, "ZHIPU-AI": ZhipuEmbed,


CvModel = { CvModel = {
"OpenAI": GptV4, "OpenAI": GptV4,
"Azure-OpenAI": AzureGptV4,
"Ollama": OllamaCV, "Ollama": OllamaCV,
"Xinference": XinferenceCV, "Xinference": XinferenceCV,
"Tongyi-Qianwen": QWenCV, "Tongyi-Qianwen": QWenCV,


ChatModel = { ChatModel = {
"OpenAI": GptTurbo, "OpenAI": GptTurbo,
"Azure-OpenAI": AzureChat,
"ZHIPU-AI": ZhipuChat, "ZHIPU-AI": ZhipuChat,
"Tongyi-Qianwen": QWenChat, "Tongyi-Qianwen": QWenChat,
"Ollama": OllamaChat, "Ollama": OllamaChat,

+ 6
- 0
rag/llm/chat_model.py View File

# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
from openai.lib.azure import AzureOpenAI
from zhipuai import ZhipuAI from zhipuai import ZhipuAI
from dashscope import Generation from dashscope import Generation
from abc import ABC from abc import ABC
if not base_url: base_url="https://api.deepseek.com/v1" if not base_url: base_url="https://api.deepseek.com/v1"
super().__init__(key, model_name, base_url) super().__init__(key, model_name, base_url)


class AzureChat(Base):
def __init__(self, key, model_name, **kwargs):
self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01")
self.model_name = model_name



class BaiChuanChat(Base): class BaiChuanChat(Base):
def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"): def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"):

+ 20
- 0
rag/llm/cv_model.py View File

# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
# #
from openai.lib.azure import AzureOpenAI
from zhipuai import ZhipuAI from zhipuai import ZhipuAI
import io import io
from abc import ABC from abc import ABC
) )
return res.choices[0].message.content.strip(), res.usage.total_tokens return res.choices[0].message.content.strip(), res.usage.total_tokens


class AzureGptV4(Base):
def __init__(self, key, model_name, lang="Chinese", **kwargs):
self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01")
self.model_name = model_name
self.lang = lang

def describe(self, image, max_tokens=300):
b64 = self.image2base64(image)
prompt = self.prompt(b64)
for i in range(len(prompt)):
for c in prompt[i]["content"]:
if "text" in c: c["type"] = "text"

res = self.client.chat.completions.create(
model=self.model_name,
messages=prompt,
max_tokens=max_tokens,
)
return res.choices[0].message.content.strip(), res.usage.total_tokens


class QWenCV(Base): class QWenCV(Base):
def __init__(self, key, model_name="qwen-vl-chat-v1", lang="Chinese", **kwargs): def __init__(self, key, model_name="qwen-vl-chat-v1", lang="Chinese", **kwargs):

+ 6
- 0
rag/llm/embedding_model.py View File

import threading import threading
import requests import requests
from huggingface_hub import snapshot_download from huggingface_hub import snapshot_download
from openai.lib.azure import AzureOpenAI
from zhipuai import ZhipuAI from zhipuai import ZhipuAI
import os import os
from abc import ABC from abc import ABC
return np.array(res.data[0].embedding), res.usage.total_tokens return np.array(res.data[0].embedding), res.usage.total_tokens




class AzureEmbed(Base):
def __init__(self, key, model_name, **kwargs):
self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01")
self.model_name = model_name

class BaiChuanEmbed(OpenAIEmbed): class BaiChuanEmbed(OpenAIEmbed):
def __init__(self, key, def __init__(self, key,
model_name='Baichuan-Text-Embedding', model_name='Baichuan-Text-Embedding',

Loading…
Cancel
Save