### What problem does this PR solve? feat: Integrates LLM Azure OpenAI #716 ### Type of change - [x] New Feature (non-breaking change which adds functionality) ### Other It's just the back-end code, the front-end needs to provide the Azure OpenAI model addition form. #### Required parameters - base_url - api_key --------- Co-authored-by: yonghui li <yonghui.li@bondex.com.cn>tags/v0.8.0
| "logo": "", | "logo": "", | ||||
| "tags": "LLM,TEXT EMBEDDING", | "tags": "LLM,TEXT EMBEDDING", | ||||
| "status": "1", | "status": "1", | ||||
| },{ | |||||
| "name": "Azure-OpenAI", | |||||
| "logo": "", | |||||
| "tags": "LLM,TEXT EMBEDDING,SPEECH2TEXT,MODERATION", | |||||
| "status": "1", | |||||
| } | } | ||||
| # { | # { | ||||
| # "name": "文心一言", | # "name": "文心一言", | ||||
| "max_tokens": 8192, | "max_tokens": 8192, | ||||
| "model_type": LLMType.EMBEDDING | "model_type": LLMType.EMBEDDING | ||||
| }, | }, | ||||
| # ------------------------ Azure OpenAI ----------------------- | |||||
| # Please ensure the llm_name is the same as the name in Azure | |||||
| # OpenAI deployment name (e.g., azure-gpt-4o). And the llm_name | |||||
| # must different from the OpenAI llm_name | |||||
| # | |||||
| # Each model must be deployed in the Azure OpenAI service, otherwise, | |||||
| # you will receive an error message 'The API deployment for | |||||
| # this resource does not exist' | |||||
| { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-4o", | |||||
| "tags": "LLM,CHAT,128K", | |||||
| "max_tokens": 128000, | |||||
| "model_type": LLMType.CHAT.value + "," + LLMType.IMAGE2TEXT.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-35-turbo", | |||||
| "tags": "LLM,CHAT,4K", | |||||
| "max_tokens": 4096, | |||||
| "model_type": LLMType.CHAT.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-35-turbo-16k", | |||||
| "tags": "LLM,CHAT,16k", | |||||
| "max_tokens": 16385, | |||||
| "model_type": LLMType.CHAT.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-text-embedding-ada-002", | |||||
| "tags": "TEXT EMBEDDING,8K", | |||||
| "max_tokens": 8191, | |||||
| "model_type": LLMType.EMBEDDING.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-text-embedding-3-small", | |||||
| "tags": "TEXT EMBEDDING,8K", | |||||
| "max_tokens": 8191, | |||||
| "model_type": LLMType.EMBEDDING.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-text-embedding-3-large", | |||||
| "tags": "TEXT EMBEDDING,8K", | |||||
| "max_tokens": 8191, | |||||
| "model_type": LLMType.EMBEDDING.value | |||||
| },{ | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-whisper-1", | |||||
| "tags": "SPEECH2TEXT", | |||||
| "max_tokens": 25 * 1024 * 1024, | |||||
| "model_type": LLMType.SPEECH2TEXT.value | |||||
| }, | |||||
| { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-4", | |||||
| "tags": "LLM,CHAT,8K", | |||||
| "max_tokens": 8191, | |||||
| "model_type": LLMType.CHAT.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-4-turbo", | |||||
| "tags": "LLM,CHAT,8K", | |||||
| "max_tokens": 8191, | |||||
| "model_type": LLMType.CHAT.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-4-32k", | |||||
| "tags": "LLM,CHAT,32K", | |||||
| "max_tokens": 32768, | |||||
| "model_type": LLMType.CHAT.value | |||||
| }, { | |||||
| "fid": factory_infos[15]["name"], | |||||
| "llm_name": "azure-gpt-4-vision-preview", | |||||
| "tags": "LLM,CHAT,IMAGE2TEXT", | |||||
| "max_tokens": 765, | |||||
| "model_type": LLMType.IMAGE2TEXT.value | |||||
| }, | |||||
| ] | ] | ||||
| for info in factory_infos: | for info in factory_infos: | ||||
| try: | try: |
| "image2text_model": "gpt-4-vision-preview", | "image2text_model": "gpt-4-vision-preview", | ||||
| "asr_model": "whisper-1", | "asr_model": "whisper-1", | ||||
| }, | }, | ||||
| "Azure-OpenAI": { | |||||
| "chat_model": "azure-gpt-35-turbo", | |||||
| "embedding_model": "azure-text-embedding-ada-002", | |||||
| "image2text_model": "azure-gpt-4-vision-preview", | |||||
| "asr_model": "azure-whisper-1", | |||||
| }, | |||||
| "ZHIPU-AI": { | "ZHIPU-AI": { | ||||
| "chat_model": "glm-3-turbo", | "chat_model": "glm-3-turbo", | ||||
| "embedding_model": "embedding-2", | "embedding_model": "embedding-2", |
| EmbeddingModel = { | EmbeddingModel = { | ||||
| "Ollama": OllamaEmbed, | "Ollama": OllamaEmbed, | ||||
| "OpenAI": OpenAIEmbed, | "OpenAI": OpenAIEmbed, | ||||
| "Azure-OpenAI": AzureEmbed, | |||||
| "Xinference": XinferenceEmbed, | "Xinference": XinferenceEmbed, | ||||
| "Tongyi-Qianwen": QWenEmbed, | "Tongyi-Qianwen": QWenEmbed, | ||||
| "ZHIPU-AI": ZhipuEmbed, | "ZHIPU-AI": ZhipuEmbed, | ||||
| CvModel = { | CvModel = { | ||||
| "OpenAI": GptV4, | "OpenAI": GptV4, | ||||
| "Azure-OpenAI": AzureGptV4, | |||||
| "Ollama": OllamaCV, | "Ollama": OllamaCV, | ||||
| "Xinference": XinferenceCV, | "Xinference": XinferenceCV, | ||||
| "Tongyi-Qianwen": QWenCV, | "Tongyi-Qianwen": QWenCV, | ||||
| ChatModel = { | ChatModel = { | ||||
| "OpenAI": GptTurbo, | "OpenAI": GptTurbo, | ||||
| "Azure-OpenAI": AzureChat, | |||||
| "ZHIPU-AI": ZhipuChat, | "ZHIPU-AI": ZhipuChat, | ||||
| "Tongyi-Qianwen": QWenChat, | "Tongyi-Qianwen": QWenChat, | ||||
| "Ollama": OllamaChat, | "Ollama": OllamaChat, |
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| from openai.lib.azure import AzureOpenAI | |||||
| from zhipuai import ZhipuAI | from zhipuai import ZhipuAI | ||||
| from dashscope import Generation | from dashscope import Generation | ||||
| from abc import ABC | from abc import ABC | ||||
| if not base_url: base_url="https://api.deepseek.com/v1" | if not base_url: base_url="https://api.deepseek.com/v1" | ||||
| super().__init__(key, model_name, base_url) | super().__init__(key, model_name, base_url) | ||||
| class AzureChat(Base): | |||||
| def __init__(self, key, model_name, **kwargs): | |||||
| self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01") | |||||
| self.model_name = model_name | |||||
| class BaiChuanChat(Base): | class BaiChuanChat(Base): | ||||
| def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"): | def __init__(self, key, model_name="Baichuan3-Turbo", base_url="https://api.baichuan-ai.com/v1"): |
| # See the License for the specific language governing permissions and | # See the License for the specific language governing permissions and | ||||
| # limitations under the License. | # limitations under the License. | ||||
| # | # | ||||
| from openai.lib.azure import AzureOpenAI | |||||
| from zhipuai import ZhipuAI | from zhipuai import ZhipuAI | ||||
| import io | import io | ||||
| from abc import ABC | from abc import ABC | ||||
| ) | ) | ||||
| return res.choices[0].message.content.strip(), res.usage.total_tokens | return res.choices[0].message.content.strip(), res.usage.total_tokens | ||||
| class AzureGptV4(Base): | |||||
| def __init__(self, key, model_name, lang="Chinese", **kwargs): | |||||
| self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01") | |||||
| self.model_name = model_name | |||||
| self.lang = lang | |||||
| def describe(self, image, max_tokens=300): | |||||
| b64 = self.image2base64(image) | |||||
| prompt = self.prompt(b64) | |||||
| for i in range(len(prompt)): | |||||
| for c in prompt[i]["content"]: | |||||
| if "text" in c: c["type"] = "text" | |||||
| res = self.client.chat.completions.create( | |||||
| model=self.model_name, | |||||
| messages=prompt, | |||||
| max_tokens=max_tokens, | |||||
| ) | |||||
| return res.choices[0].message.content.strip(), res.usage.total_tokens | |||||
| class QWenCV(Base): | class QWenCV(Base): | ||||
| def __init__(self, key, model_name="qwen-vl-chat-v1", lang="Chinese", **kwargs): | def __init__(self, key, model_name="qwen-vl-chat-v1", lang="Chinese", **kwargs): |
| import threading | import threading | ||||
| import requests | import requests | ||||
| from huggingface_hub import snapshot_download | from huggingface_hub import snapshot_download | ||||
| from openai.lib.azure import AzureOpenAI | |||||
| from zhipuai import ZhipuAI | from zhipuai import ZhipuAI | ||||
| import os | import os | ||||
| from abc import ABC | from abc import ABC | ||||
| return np.array(res.data[0].embedding), res.usage.total_tokens | return np.array(res.data[0].embedding), res.usage.total_tokens | ||||
| class AzureEmbed(Base): | |||||
| def __init__(self, key, model_name, **kwargs): | |||||
| self.client = AzureOpenAI(api_key=key, azure_endpoint=kwargs["base_url"], api_version="2024-02-01") | |||||
| self.model_name = model_name | |||||
| class BaiChuanEmbed(OpenAIEmbed): | class BaiChuanEmbed(OpenAIEmbed): | ||||
| def __init__(self, key, | def __init__(self, key, | ||||
| model_name='Baichuan-Text-Embedding', | model_name='Baichuan-Text-Embedding', |