feat(backend): support import DSL from URL (#6287)

api/controllers/console/app/app.py

@@ -15,6 +15,7 @@ from fields.app_fields import (
     app_pagination_fields,
 )
 from libs.login import login_required
+from services.app_dsl_service import AppDslService
 from services.app_service import AppService
 
 ALLOW_CREATE_APP_MODES = ['chat', 'agent-chat', 'advanced-chat', 'workflow', 'completion']
@@ -97,8 +98,42 @@ class AppImportApi(Resource):
         parser.add_argument('icon_background', type=str, location='json')
         args = parser.parse_args()
 
-        app_service = AppService()
-        app = app_service.import_app(current_user.current_tenant_id, args['data'], args, current_user)
+        app = AppDslService.import_and_create_new_app(
+            tenant_id=current_user.current_tenant_id,
+            data=args['data'],
+            args=args,
+            account=current_user
+        )
+
+        return app, 201
+
+
+class AppImportFromUrlApi(Resource):
+    @setup_required
+    @login_required
+    @account_initialization_required
+    @marshal_with(app_detail_fields_with_site)
+    @cloud_edition_billing_resource_check('apps')
+    def post(self):
+        """Import app from url"""
+        # The role of the current user in the ta table must be admin, owner, or editor
+        if not current_user.is_editor:
+            raise Forbidden()
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('url', type=str, required=True, nullable=False, location='json')
+        parser.add_argument('name', type=str, location='json')
+        parser.add_argument('description', type=str, location='json')
+        parser.add_argument('icon', type=str, location='json')
+        parser.add_argument('icon_background', type=str, location='json')
+        args = parser.parse_args()
+
+        app = AppDslService.import_and_create_new_app_from_url(
+            tenant_id=current_user.current_tenant_id,
+            url=args['url'],
+            args=args,
+            account=current_user
+        )
 
         return app, 201
 
@@ -177,9 +212,13 @@ class AppCopyApi(Resource):
         parser.add_argument('icon_background', type=str, location='json')
         args = parser.parse_args()
 
-        app_service = AppService()
-        data = app_service.export_app(app_model)
-        app = app_service.import_app(current_user.current_tenant_id, data, args, current_user)
+        data = AppDslService.export_dsl(app_model=app_model)
+        app = AppDslService.import_and_create_new_app(
+            tenant_id=current_user.current_tenant_id,
+            data=data,
+            args=args,
+            account=current_user
+        )
 
         return app, 201
 
@@ -195,10 +234,8 @@ class AppExportApi(Resource):
         if not current_user.is_editor:
             raise Forbidden()
 
-        app_service = AppService()
-
         return {
-            "data": app_service.export_app(app_model)
+            "data": AppDslService.export_dsl(app_model=app_model)
         }
 
 
@@ -322,6 +359,7 @@ class AppTraceApi(Resource):
 
 api.add_resource(AppListApi, '/apps')
 api.add_resource(AppImportApi, '/apps/import')
+api.add_resource(AppImportFromUrlApi, '/apps/import/url')
 api.add_resource(AppApi, '/apps/<uuid:app_id>')
 api.add_resource(AppCopyApi, '/apps/<uuid:app_id>/copy')
 api.add_resource(AppExportApi, '/apps/<uuid:app_id>/export')

api/controllers/console/app/workflow.py

@@ -20,6 +20,7 @@ from libs import helper
 from libs.helper import TimestampField, uuid_value
 from libs.login import current_user, login_required
 from models.model import App, AppMode
+from services.app_dsl_service import AppDslService
 from services.app_generate_service import AppGenerateService
 from services.errors.app import WorkflowHashNotEqualError
 from services.workflow_service import WorkflowService
@@ -128,8 +129,7 @@ class DraftWorkflowImportApi(Resource):
         parser.add_argument('data', type=str, required=True, nullable=False, location='json')
         args = parser.parse_args()
 
-        workflow_service = WorkflowService()
-        workflow = workflow_service.import_draft_workflow(
+        workflow = AppDslService.import_and_overwrite_workflow(
             app_model=app_model,
             data=args['data'],
             account=current_user

api/services/app_dsl_service.py

@@ -0,0 +1,407 @@
+import logging
+
+import httpx
+import yaml  # type: ignore
+
+from events.app_event import app_model_config_was_updated, app_was_created
+from extensions.ext_database import db
+from models.account import Account
+from models.model import App, AppMode, AppModelConfig
+from models.workflow import Workflow
+from services.workflow_service import WorkflowService
+
+logger = logging.getLogger(__name__)
+
+current_dsl_version = "0.1.0"
+dsl_to_dify_version_mapping: dict[str, str] = {
+    "0.1.0": "0.6.0",  # dsl version -> from dify version
+}
+
+
+class AppDslService:
+    @classmethod
+    def import_and_create_new_app_from_url(cls, tenant_id: str, url: str, args: dict, account: Account) -> App:
+        """
+        Import app dsl from url and create new app
+        :param tenant_id: tenant id
+        :param url: import url
+        :param args: request args
+        :param account: Account instance
+        """
+        try:
+            max_size = 10 * 1024 * 1024  # 10MB
+            timeout = httpx.Timeout(10.0)
+            with httpx.stream("GET", url.strip(), follow_redirects=True, timeout=timeout) as response:
+                response.raise_for_status()
+                total_size = 0
+                content = b""
+                for chunk in response.iter_bytes():
+                    total_size += len(chunk)
+                    if total_size > max_size:
+                        raise ValueError("File size exceeds the limit of 10MB")
+                    content += chunk
+        except httpx.HTTPStatusError as http_err:
+            raise ValueError(f"HTTP error occurred: {http_err}")
+        except httpx.RequestError as req_err:
+            raise ValueError(f"Request error occurred: {req_err}")
+        except Exception as e:
+            raise ValueError(f"Failed to fetch DSL from URL: {e}")
+
+        if not content:
+            raise ValueError("Empty content from url")
+
+        try:
+            data = content.decode("utf-8")
+        except UnicodeDecodeError as e:
+            raise ValueError(f"Error decoding content: {e}")
+
+        return cls.import_and_create_new_app(tenant_id, data, args, account)
+
+    @classmethod
+    def import_and_create_new_app(cls, tenant_id: str, data: str, args: dict, account: Account) -> App:
+        """
+        Import app dsl and create new app
+        :param tenant_id: tenant id
+        :param data: import data
+        :param args: request args
+        :param account: Account instance
+        """
+        try:
+            import_data = yaml.safe_load(data)
+        except yaml.YAMLError:
+            raise ValueError("Invalid YAML format in data argument.")
+
+        # check or repair dsl version
+        import_data = cls._check_or_fix_dsl(import_data)
+
+        app_data = import_data.get('app')
+        if not app_data:
+            raise ValueError("Missing app in data argument")
+
+        # get app basic info
+        name = args.get("name") if args.get("name") else app_data.get('name')
+        description = args.get("description") if args.get("description") else app_data.get('description', '')
+        icon = args.get("icon") if args.get("icon") else app_data.get('icon')
+        icon_background = args.get("icon_background") if args.get("icon_background") \
+            else app_data.get('icon_background')
+
+        # import dsl and create app
+        app_mode = AppMode.value_of(app_data.get('mode'))
+        if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
+            app = cls._import_and_create_new_workflow_based_app(
+                tenant_id=tenant_id,
+                app_mode=app_mode,
+                workflow_data=import_data.get('workflow'),
+                account=account,
+                name=name,
+                description=description,
+                icon=icon,
+                icon_background=icon_background
+            )
+        elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]:
+            app = cls._import_and_create_new_model_config_based_app(
+                tenant_id=tenant_id,
+                app_mode=app_mode,
+                model_config_data=import_data.get('model_config'),
+                account=account,
+                name=name,
+                description=description,
+                icon=icon,
+                icon_background=icon_background
+            )
+        else:
+            raise ValueError("Invalid app mode")
+
+        return app
+
+    @classmethod
+    def import_and_overwrite_workflow(cls, app_model: App, data: str, account: Account) -> Workflow:
+        """
+        Import app dsl and overwrite workflow
+        :param app_model: App instance
+        :param data: import data
+        :param account: Account instance
+        """
+        try:
+            import_data = yaml.safe_load(data)
+        except yaml.YAMLError:
+            raise ValueError("Invalid YAML format in data argument.")
+
+        # check or repair dsl version
+        import_data = cls._check_or_fix_dsl(import_data)
+
+        app_data = import_data.get('app')
+        if not app_data:
+            raise ValueError("Missing app in data argument")
+
+        # import dsl and overwrite app
+        app_mode = AppMode.value_of(app_data.get('mode'))
+        if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
+            raise ValueError("Only support import workflow in advanced-chat or workflow app.")
+
+        if app_data.get('mode') != app_model.mode:
+            raise ValueError(
+                f"App mode {app_data.get('mode')} is not matched with current app mode {app_mode.value}")
+
+        return cls._import_and_overwrite_workflow_based_app(
+            app_model=app_model,
+            workflow_data=import_data.get('workflow'),
+            account=account,
+        )
+
+    @classmethod
+    def export_dsl(cls, app_model: App) -> str:
+        """
+        Export app
+        :param app_model: App instance
+        :return:
+        """
+        app_mode = AppMode.value_of(app_model.mode)
+
+        export_data = {
+            "version": current_dsl_version,
+            "kind": "app",
+            "app": {
+                "name": app_model.name,
+                "mode": app_model.mode,
+                "icon": app_model.icon,
+                "icon_background": app_model.icon_background,
+                "description": app_model.description
+            }
+        }
+
+        if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
+            cls._append_workflow_export_data(export_data, app_model)
+        else:
+            cls._append_model_config_export_data(export_data, app_model)
+
+        return yaml.dump(export_data)
+
+    @classmethod
+    def _check_or_fix_dsl(cls, import_data: dict) -> dict:
+        """
+        Check or fix dsl
+
+        :param import_data: import data
+        """
+        if not import_data.get('version'):
+            import_data['version'] = "0.1.0"
+
+        if not import_data.get('kind') or import_data.get('kind') != "app":
+            import_data['kind'] = "app"
+
+        if import_data.get('version') != current_dsl_version:
+            # Currently only one DSL version, so no difference checks or compatibility fixes will be performed.
+            logger.warning(f"DSL version {import_data.get('version')} is not compatible "
+                           f"with current version {current_dsl_version}, related to "
+                           f"Dify version {dsl_to_dify_version_mapping.get(current_dsl_version)}.")
+
+        return import_data
+
+    @classmethod
+    def _import_and_create_new_workflow_based_app(cls,
+                                                  tenant_id: str,
+                                                  app_mode: AppMode,
+                                                  workflow_data: dict,
+                                                  account: Account,
+                                                  name: str,
+                                                  description: str,
+                                                  icon: str,
+                                                  icon_background: str) -> App:
+        """
+        Import app dsl and create new workflow based app
+
+        :param tenant_id: tenant id
+        :param app_mode: app mode
+        :param workflow_data: workflow data
+        :param account: Account instance
+        :param name: app name
+        :param description: app description
+        :param icon: app icon
+        :param icon_background: app icon background
+        """
+        if not workflow_data:
+            raise ValueError("Missing workflow in data argument "
+                             "when app mode is advanced-chat or workflow")
+
+        app = cls._create_app(
+            tenant_id=tenant_id,
+            app_mode=app_mode,
+            account=account,
+            name=name,
+            description=description,
+            icon=icon,
+            icon_background=icon_background
+        )
+
+        # init draft workflow
+        workflow_service = WorkflowService()
+        draft_workflow = workflow_service.sync_draft_workflow(
+            app_model=app,
+            graph=workflow_data.get('graph', {}),
+            features=workflow_data.get('../core/app/features', {}),
+            unique_hash=None,
+            account=account
+        )
+        workflow_service.publish_workflow(
+            app_model=app,
+            account=account,
+            draft_workflow=draft_workflow
+        )
+
+        return app
+
+    @classmethod
+    def _import_and_overwrite_workflow_based_app(cls,
+                                                 app_model: App,
+                                                 workflow_data: dict,
+                                                 account: Account) -> Workflow:
+        """
+        Import app dsl and overwrite workflow based app
+
+        :param app_model: App instance
+        :param workflow_data: workflow data
+        :param account: Account instance
+        """
+        if not workflow_data:
+            raise ValueError("Missing workflow in data argument "
+                             "when app mode is advanced-chat or workflow")
+
+        # fetch draft workflow by app_model
+        workflow_service = WorkflowService()
+        current_draft_workflow = workflow_service.get_draft_workflow(app_model=app_model)
+        if current_draft_workflow:
+            unique_hash = current_draft_workflow.unique_hash
+        else:
+            unique_hash = None
+
+        # sync draft workflow
+        draft_workflow = workflow_service.sync_draft_workflow(
+            app_model=app_model,
+            graph=workflow_data.get('graph', {}),
+            features=workflow_data.get('features', {}),
+            unique_hash=unique_hash,
+            account=account
+        )
+
+        return draft_workflow
+
+    @classmethod
+    def _import_and_create_new_model_config_based_app(cls,
+                                                      tenant_id: str,
+                                                      app_mode: AppMode,
+                                                      model_config_data: dict,
+                                                      account: Account,
+                                                      name: str,
+                                                      description: str,
+                                                      icon: str,
+                                                      icon_background: str) -> App:
+        """
+        Import app dsl and create new model config based app
+
+        :param tenant_id: tenant id
+        :param app_mode: app mode
+        :param model_config_data: model config data
+        :param account: Account instance
+        :param name: app name
+        :param description: app description
+        :param icon: app icon
+        :param icon_background: app icon background
+        """
+        if not model_config_data:
+            raise ValueError("Missing model_config in data argument "
+                             "when app mode is chat, agent-chat or completion")
+
+        app = cls._create_app(
+            tenant_id=tenant_id,
+            app_mode=app_mode,
+            account=account,
+            name=name,
+            description=description,
+            icon=icon,
+            icon_background=icon_background
+        )
+
+        app_model_config = AppModelConfig()
+        app_model_config = app_model_config.from_model_config_dict(model_config_data)
+        app_model_config.app_id = app.id
+
+        db.session.add(app_model_config)
+        db.session.commit()
+
+        app.app_model_config_id = app_model_config.id
+
+        app_model_config_was_updated.send(
+            app,
+            app_model_config=app_model_config
+        )
+
+        return app
+
+    @classmethod
+    def _create_app(cls,
+                    tenant_id: str,
+                    app_mode: AppMode,
+                    account: Account,
+                    name: str,
+                    description: str,
+                    icon: str,
+                    icon_background: str) -> App:
+        """
+        Create new app
+
+        :param tenant_id: tenant id
+        :param app_mode: app mode
+        :param account: Account instance
+        :param name: app name
+        :param description: app description
+        :param icon: app icon
+        :param icon_background: app icon background
+        """
+        app = App(
+            tenant_id=tenant_id,
+            mode=app_mode.value,
+            name=name,
+            description=description,
+            icon=icon,
+            icon_background=icon_background,
+            enable_site=True,
+            enable_api=True
+        )
+
+        db.session.add(app)
+        db.session.commit()
+
+        app_was_created.send(app, account=account)
+
+        return app
+
+    @classmethod
+    def _append_workflow_export_data(cls, export_data: dict, app_model: App) -> None:
+        """
+        Append workflow export data
+        :param export_data: export data
+        :param app_model: App instance
+        """
+        workflow_service = WorkflowService()
+        workflow = workflow_service.get_draft_workflow(app_model)
+        if not workflow:
+            raise ValueError("Missing draft workflow configuration, please check.")
+
+        export_data['workflow'] = {
+            "graph": workflow.graph_dict,
+            "features": workflow.features_dict
+        }
+
+    @classmethod
+    def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None:
+        """
+        Append model config export data
+        :param export_data: export data
+        :param app_model: App instance
+        """
+        app_model_config = app_model.app_model_config
+        if not app_model_config:
+            raise ValueError("Missing app configuration, please check.")
+
+        export_data['model_config'] = app_model_config.to_dict()

api/services/app_service.py

@@ -3,7 +3,6 @@ import logging
 from datetime import datetime, timezone
 from typing import cast
 
-import yaml
 from flask_login import current_user
 from flask_sqlalchemy.pagination import Pagination
 
@@ -17,13 +16,12 @@ from core.model_runtime.entities.model_entities import ModelPropertyKey, ModelTy
 from core.model_runtime.model_providers.__base.large_language_model import LargeLanguageModel
 from core.tools.tool_manager import ToolManager
 from core.tools.utils.configuration import ToolParameterConfigurationManager
-from events.app_event import app_model_config_was_updated, app_was_created
+from events.app_event import app_was_created
 from extensions.ext_database import db
 from models.account import Account
 from models.model import App, AppMode, AppModelConfig
 from models.tools import ApiToolProvider
 from services.tag_service import TagService
-from services.workflow_service import WorkflowService
 from tasks.remove_app_and_related_data_task import remove_app_and_related_data_task
 
 
@@ -144,120 +142,6 @@ class AppService:
 
         return app
 
-    def import_app(self, tenant_id: str, data: str, args: dict, account: Account) -> App:
-        """
-        Import app
-        :param tenant_id: tenant id
-        :param data: import data
-        :param args: request args
-        :param account: Account instance
-        """
-        try:
-            import_data = yaml.safe_load(data)
-        except yaml.YAMLError as e:
-            raise ValueError("Invalid YAML format in data argument.")
-
-        app_data = import_data.get('app')
-        model_config_data = import_data.get('model_config')
-        workflow = import_data.get('workflow')
-
-        if not app_data:
-            raise ValueError("Missing app in data argument")
-
-        app_mode = AppMode.value_of(app_data.get('mode'))
-        if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
-            if not workflow:
-                raise ValueError("Missing workflow in data argument "
-                                 "when app mode is advanced-chat or workflow")
-        elif app_mode in [AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION]:
-            if not model_config_data:
-                raise ValueError("Missing model_config in data argument "
-                                 "when app mode is chat, agent-chat or completion")
-        else:
-            raise ValueError("Invalid app mode")
-
-        app = App(
-            tenant_id=tenant_id,
-            mode=app_data.get('mode'),
-            name=args.get("name") if args.get("name") else app_data.get('name'),
-            description=args.get("description") if args.get("description") else app_data.get('description', ''),
-            icon=args.get("icon") if args.get("icon") else app_data.get('icon'),
-            icon_background=args.get("icon_background") if args.get("icon_background") \
-                else app_data.get('icon_background'),
-            enable_site=True,
-            enable_api=True
-        )
-
-        db.session.add(app)
-        db.session.commit()
-
-        app_was_created.send(app, account=account)
-
-        if workflow:
-            # init draft workflow
-            workflow_service = WorkflowService()
-            draft_workflow = workflow_service.sync_draft_workflow(
-                app_model=app,
-                graph=workflow.get('graph'),
-                features=workflow.get('features'),
-                unique_hash=None,
-                account=account
-            )
-            workflow_service.publish_workflow(
-                app_model=app,
-                account=account,
-                draft_workflow=draft_workflow
-            )
-
-        if model_config_data:
-            app_model_config = AppModelConfig()
-            app_model_config = app_model_config.from_model_config_dict(model_config_data)
-            app_model_config.app_id = app.id
-
-            db.session.add(app_model_config)
-            db.session.commit()
-
-            app.app_model_config_id = app_model_config.id
-
-            app_model_config_was_updated.send(
-                app,
-                app_model_config=app_model_config
-            )
-
-        return app
-
-    def export_app(self, app: App) -> str:
-        """
-        Export app
-        :param app: App instance
-        :return:
-        """
-        app_mode = AppMode.value_of(app.mode)
-
-        export_data = {
-            "app": {
-                "name": app.name,
-                "mode": app.mode,
-                "icon": app.icon,
-                "icon_background": app.icon_background,
-                "description": app.description
-            }
-        }
-
-        if app_mode in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
-            workflow_service = WorkflowService()
-            workflow = workflow_service.get_draft_workflow(app)
-            export_data['workflow'] = {
-                "graph": workflow.graph_dict,
-                "features": workflow.features_dict
-            }
-        else:
-            app_model_config = app.app_model_config
-
-            export_data['model_config'] = app_model_config.to_dict()
-
-        return yaml.dump(export_data)
-
     def get_app(self, app: App) -> App:
         """
         Get App

api/services/recommended_app_service.py

@@ -4,12 +4,13 @@ from os import path
 from typing import Optional
 
 import requests
+from flask import current_app
 
 from configs import dify_config
 from constants.languages import languages
 from extensions.ext_database import db
 from models.model import App, RecommendedApp
-from services.app_service import AppService
+from services.app_dsl_service import AppDslService
 
 logger = logging.getLogger(__name__)
 
@@ -186,16 +187,13 @@ class RecommendedAppService:
         if not app_model or not app_model.is_public:
             return None
 
-        app_service = AppService()
-        export_str = app_service.export_app(app_model)
-
         return {
             'id': app_model.id,
             'name': app_model.name,
             'icon': app_model.icon,
             'icon_background': app_model.icon_background,
             'mode': app_model.mode,
-            'export_data': export_str
+            'export_data': AppDslService.export_dsl(app_model=app_model)
         }
 
     @classmethod

api/services/workflow_service.py

@@ -3,8 +3,6 @@ import time
 from datetime import datetime, timezone
 from typing import Optional
 
-import yaml
-
 from core.app.apps.advanced_chat.app_config_manager import AdvancedChatAppConfigManager
 from core.app.apps.workflow.app_config_manager import WorkflowAppConfigManager
 from core.model_runtime.utils.encoders import jsonable_encoder
@@ -114,56 +112,6 @@ class WorkflowService:
         # return draft workflow
         return workflow
 
-    def import_draft_workflow(self, app_model: App,
-                              data: str,
-                              account: Account) -> Workflow:
-        """
-        Import draft workflow
-        :param app_model: App instance
-        :param data: import data
-        :param account: Account instance
-        :return:
-        """
-        try:
-            import_data = yaml.safe_load(data)
-        except yaml.YAMLError as e:
-            raise ValueError("Invalid YAML format in data argument.")
-
-        app_data = import_data.get('app')
-        workflow = import_data.get('workflow')
-
-        if not app_data:
-            raise ValueError("Missing app in data argument")
-
-        app_mode = AppMode.value_of(app_data.get('mode'))
-        if app_mode not in [AppMode.ADVANCED_CHAT, AppMode.WORKFLOW]:
-            raise ValueError("Only support import workflow in advanced-chat or workflow app.")
-
-        if app_data.get('mode') != app_model.mode:
-            raise ValueError(f"App mode {app_data.get('mode')} is not matched with current app mode {app_model.mode}")
-
-        if not workflow:
-            raise ValueError("Missing workflow in data argument "
-                             "when app mode is advanced-chat or workflow")
-
-        # fetch draft workflow by app_model
-        current_draft_workflow = self.get_draft_workflow(app_model=app_model)
-        if current_draft_workflow:
-            unique_hash = current_draft_workflow.unique_hash
-        else:
-            unique_hash = None
-
-        # sync draft workflow
-        draft_workflow = self.sync_draft_workflow(
-            app_model=app_model,
-            graph=workflow.get('graph'),
-            features=workflow.get('features'),
-            unique_hash=unique_hash,
-            account=account
-        )
-
-        return draft_workflow
-
     def publish_workflow(self, app_model: App,
                          account: Account,
                          draft_workflow: Optional[Workflow] = None) -> Workflow: