provide a bit more info in logs when parsing api schema error (#3026)

api/core/tools/utils/parser.py

@@ -1,10 +1,12 @@
 
 import re
 import uuid
+from json import dumps as json_dumps
 from json import loads as json_loads
+from json.decoder import JSONDecodeError
 
 from requests import get
-from yaml import FullLoader, load
+from yaml import YAMLError, safe_load
 
 from core.tools.entities.common_entities import I18nObject
 from core.tools.entities.tool_bundle import ApiBasedToolBundle
@@ -184,27 +186,11 @@ class ApiBasedToolSchemaParser:
         warning = warning if warning is not None else {}
         extra_info = extra_info if extra_info is not None else {}
 
-        openapi: dict = load(yaml, Loader=FullLoader)
+        openapi: dict = safe_load(yaml)
         if openapi is None:
             raise ToolApiSchemaError('Invalid openapi yaml.')
         return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
     
-    @staticmethod
-    def parse_openapi_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
-        """
-            parse openapi yaml to tool bundle
-
-            :param yaml: the yaml string
-            :return: the tool bundle
-        """
-        warning = warning if warning is not None else {}
-        extra_info = extra_info if extra_info is not None else {}
-
-        openapi: dict = json_loads(json)
-        if openapi is None:
-            raise ToolApiSchemaError('Invalid openapi json.')
-        return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
-    
     @staticmethod
     def parse_swagger_to_openapi(swagger: dict, extra_info: dict = None, warning: dict = None) -> dict:
         """
@@ -271,38 +257,6 @@ class ApiBasedToolSchemaParser:
 
         return openapi
 
-    @staticmethod
-    def parse_swagger_yaml_to_tool_bundle(yaml: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
-        """
-            parse swagger yaml to tool bundle
-
-            :param yaml: the yaml string
-            :return: the tool bundle
-        """
-        warning = warning if warning is not None else {}
-        extra_info = extra_info if extra_info is not None else {}
-
-        swagger: dict = load(yaml, Loader=FullLoader)
-
-        openapi = ApiBasedToolSchemaParser.parse_swagger_to_openapi(swagger, extra_info=extra_info, warning=warning)
-        return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
-
-    @staticmethod
-    def parse_swagger_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
-        """
-            parse swagger yaml to tool bundle
-
-            :param yaml: the yaml string
-            :return: the tool bundle
-        """
-        warning = warning if warning is not None else {}
-        extra_info = extra_info if extra_info is not None else {}
-
-        swagger: dict = json_loads(json)
-
-        openapi = ApiBasedToolSchemaParser.parse_swagger_to_openapi(swagger, extra_info=extra_info, warning=warning)
-        return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)
-
     @staticmethod
     def parse_openai_plugin_json_to_tool_bundle(json: str, extra_info: dict = None, warning: dict = None) -> list[ApiBasedToolBundle]:
         """
@@ -346,40 +300,50 @@ class ApiBasedToolSchemaParser:
         warning = warning if warning is not None else {}
         extra_info = extra_info if extra_info is not None else {}
 
-        json_possible = False
         content = content.strip()
+        loaded_content = None
+        json_error = None
+        yaml_error = None
+        
+        try:
+            loaded_content = json_loads(content)
+        except JSONDecodeError as e:
+            json_error = e
 
-        if content.startswith('{') and content.endswith('}'):
-            json_possible = True
-
-        if json_possible:
-            try:
-                return ApiBasedToolSchemaParser.parse_openapi_json_to_tool_bundle(content, extra_info=extra_info, warning=warning), \
-                    ApiProviderSchemaType.OPENAPI.value
-            except:
-                pass
-
-            try:
-                return ApiBasedToolSchemaParser.parse_swagger_json_to_tool_bundle(content, extra_info=extra_info, warning=warning), \
-                    ApiProviderSchemaType.SWAGGER.value
-            except:
-                pass
-            try:
-                return ApiBasedToolSchemaParser.parse_openai_plugin_json_to_tool_bundle(content, extra_info=extra_info, warning=warning), \
-                    ApiProviderSchemaType.OPENAI_PLUGIN.value
-            except:
-                pass
-        else:
-            try:
-                return ApiBasedToolSchemaParser.parse_openapi_yaml_to_tool_bundle(content, extra_info=extra_info, warning=warning), \
-                    ApiProviderSchemaType.OPENAPI.value
-            except:
-                pass
-
+        if loaded_content is None:
             try:
-                return ApiBasedToolSchemaParser.parse_swagger_yaml_to_tool_bundle(content, extra_info=extra_info, warning=warning), \
-                    ApiProviderSchemaType.SWAGGER.value
-            except:
-                pass
+                loaded_content = safe_load(content)
+            except YAMLError as e:
+                yaml_error = e
+        if loaded_content is None:
+            raise ToolApiSchemaError(f'Invalid api schema, schema is neither json nor yaml. json error: {str(json_error)}, yaml error: {str(yaml_error)}')
+
+        swagger_error = None
+        openapi_error = None
+        openapi_plugin_error = None
+        schema_type = None
+        
+        try:
+            openapi = ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(loaded_content, extra_info=extra_info, warning=warning)
+            schema_type = ApiProviderSchemaType.OPENAPI.value
+            return openapi, schema_type
+        except ToolApiSchemaError as e:
+            openapi_error = e
+        
+        # openai parse error, fallback to swagger
+        try:
+            converted_swagger = ApiBasedToolSchemaParser.parse_swagger_to_openapi(loaded_content, extra_info=extra_info, warning=warning)
+            schema_type = ApiProviderSchemaType.SWAGGER.value
+            return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(converted_swagger, extra_info=extra_info, warning=warning), schema_type
+        except ToolApiSchemaError as e:
+            swagger_error = e
+        
+        # swagger parse error, fallback to openai plugin
+        try:
+            openapi_plugin = ApiBasedToolSchemaParser.parse_openai_plugin_json_to_tool_bundle(json_dumps(loaded_content), extra_info=extra_info, warning=warning)
+            return openapi_plugin, ApiProviderSchemaType.OPENAI_PLUGIN.value
+        except ToolNotSupportedError as e:
+            # maybe it's not plugin at all
+            openapi_plugin_error = e
 
-        raise ToolApiSchemaError('Invalid api schema.')
+        raise ToolApiSchemaError(f'Invalid api schema, openapi error: {str(openapi_error)}, swagger error: {str(swagger_error)}, openapi plugin error: {str(openapi_plugin_error)}')

api/services/tools_manage_service.py

@@ -1,4 +1,5 @@
 import json
+import logging
 
 from flask import current_app
 from httpx import get
@@ -24,6 +25,8 @@ from extensions.ext_database import db
 from models.tools import ApiToolProvider, BuiltinToolProvider
 from services.model_provider_service import ModelProviderService
 
+logger = logging.getLogger(__name__)
+
 
 class ToolManageService:
     @staticmethod
@@ -309,6 +312,7 @@ class ToolManageService:
             # try to parse schema, avoid SSRF attack
             ToolManageService.parser_api_schema(schema)
         except Exception as e:
+            logger.error(f"parse api schema error: {str(e)}")
             raise ValueError('invalid schema, please check the url you provided')
         
         return {
@@ -655,4 +659,4 @@ class ToolManageService:
         except Exception as e:
             return { 'error': str(e) }
         
-        return { 'result': result or 'empty response' }
+        return { 'result': result or 'empty response' }