Nevar pievienot vairāk kā 25 tēmas Tēmai ir jāsākas ar burtu vai ciparu, tā var saturēt domu zīmes ('-') un var būt līdz 35 simboliem gara.

app_dsl_service.py 33KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832
  1. import base64
  2. import hashlib
  3. import logging
  4. import uuid
  5. from collections.abc import Mapping
  6. from enum import StrEnum
  7. from typing import Optional
  8. from urllib.parse import urlparse
  9. from uuid import uuid4
  10. import yaml # type: ignore
  11. from Crypto.Cipher import AES
  12. from Crypto.Util.Padding import pad, unpad
  13. from packaging import version
  14. from packaging.version import parse as parse_version
  15. from pydantic import BaseModel, Field
  16. from sqlalchemy import select
  17. from sqlalchemy.orm import Session
  18. from configs import dify_config
  19. from core.helper import ssrf_proxy
  20. from core.model_runtime.utils.encoders import jsonable_encoder
  21. from core.plugin.entities.plugin import PluginDependency
  22. from core.workflow.nodes.enums import NodeType
  23. from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData
  24. from core.workflow.nodes.llm.entities import LLMNodeData
  25. from core.workflow.nodes.parameter_extractor.entities import ParameterExtractorNodeData
  26. from core.workflow.nodes.question_classifier.entities import QuestionClassifierNodeData
  27. from core.workflow.nodes.tool.entities import ToolNodeData
  28. from events.app_event import app_model_config_was_updated, app_was_created
  29. from extensions.ext_redis import redis_client
  30. from factories import variable_factory
  31. from models import Account, App, AppMode
  32. from models.model import AppModelConfig
  33. from models.workflow import Workflow
  34. from services.plugin.dependencies_analysis import DependenciesAnalysisService
  35. from services.workflow_draft_variable_service import WorkflowDraftVariableService
  36. from services.workflow_service import WorkflowService
  37. logger = logging.getLogger(__name__)
  38. IMPORT_INFO_REDIS_KEY_PREFIX = "app_import_info:"
  39. CHECK_DEPENDENCIES_REDIS_KEY_PREFIX = "app_check_dependencies:"
  40. IMPORT_INFO_REDIS_EXPIRY = 10 * 60 # 10 minutes
  41. DSL_MAX_SIZE = 10 * 1024 * 1024 # 10MB
  42. CURRENT_DSL_VERSION = "0.4.0"
  43. class ImportMode(StrEnum):
  44. YAML_CONTENT = "yaml-content"
  45. YAML_URL = "yaml-url"
  46. class ImportStatus(StrEnum):
  47. COMPLETED = "completed"
  48. COMPLETED_WITH_WARNINGS = "completed-with-warnings"
  49. PENDING = "pending"
  50. FAILED = "failed"
  51. class Import(BaseModel):
  52. id: str
  53. status: ImportStatus
  54. app_id: Optional[str] = None
  55. app_mode: Optional[str] = None
  56. current_dsl_version: str = CURRENT_DSL_VERSION
  57. imported_dsl_version: str = ""
  58. error: str = ""
  59. class CheckDependenciesResult(BaseModel):
  60. leaked_dependencies: list[PluginDependency] = Field(default_factory=list)
  61. def _check_version_compatibility(imported_version: str) -> ImportStatus:
  62. """Determine import status based on version comparison"""
  63. try:
  64. current_ver = version.parse(CURRENT_DSL_VERSION)
  65. imported_ver = version.parse(imported_version)
  66. except version.InvalidVersion:
  67. return ImportStatus.FAILED
  68. # If imported version is newer than current, always return PENDING
  69. if imported_ver > current_ver:
  70. return ImportStatus.PENDING
  71. # If imported version is older than current's major, return PENDING
  72. if imported_ver.major < current_ver.major:
  73. return ImportStatus.PENDING
  74. # If imported version is older than current's minor, return COMPLETED_WITH_WARNINGS
  75. if imported_ver.minor < current_ver.minor:
  76. return ImportStatus.COMPLETED_WITH_WARNINGS
  77. # If imported version equals or is older than current's micro, return COMPLETED
  78. return ImportStatus.COMPLETED
  79. class PendingData(BaseModel):
  80. import_mode: str
  81. yaml_content: str
  82. name: str | None = None
  83. description: str | None = None
  84. icon_type: str | None = None
  85. icon: str | None = None
  86. icon_background: str | None = None
  87. app_id: str | None = None
  88. class CheckDependenciesPendingData(BaseModel):
  89. dependencies: list[PluginDependency]
  90. app_id: str | None = None
  91. class AppDslService:
  92. def __init__(self, session: Session):
  93. self._session = session
  94. def import_app(
  95. self,
  96. *,
  97. account: Account,
  98. import_mode: str,
  99. yaml_content: Optional[str] = None,
  100. yaml_url: Optional[str] = None,
  101. name: Optional[str] = None,
  102. description: Optional[str] = None,
  103. icon_type: Optional[str] = None,
  104. icon: Optional[str] = None,
  105. icon_background: Optional[str] = None,
  106. app_id: Optional[str] = None,
  107. ) -> Import:
  108. """Import an app from YAML content or URL."""
  109. import_id = str(uuid.uuid4())
  110. # Validate import mode
  111. try:
  112. mode = ImportMode(import_mode)
  113. except ValueError:
  114. raise ValueError(f"Invalid import_mode: {import_mode}")
  115. # Get YAML content
  116. content: str = ""
  117. if mode == ImportMode.YAML_URL:
  118. if not yaml_url:
  119. return Import(
  120. id=import_id,
  121. status=ImportStatus.FAILED,
  122. error="yaml_url is required when import_mode is yaml-url",
  123. )
  124. try:
  125. parsed_url = urlparse(yaml_url)
  126. if (
  127. parsed_url.scheme == "https"
  128. and parsed_url.netloc == "github.com"
  129. and parsed_url.path.endswith((".yml", ".yaml"))
  130. ):
  131. yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com")
  132. yaml_url = yaml_url.replace("/blob/", "/")
  133. response = ssrf_proxy.get(yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
  134. response.raise_for_status()
  135. content = response.content.decode()
  136. if len(content) > DSL_MAX_SIZE:
  137. return Import(
  138. id=import_id,
  139. status=ImportStatus.FAILED,
  140. error="File size exceeds the limit of 10MB",
  141. )
  142. if not content:
  143. return Import(
  144. id=import_id,
  145. status=ImportStatus.FAILED,
  146. error="Empty content from url",
  147. )
  148. except Exception as e:
  149. return Import(
  150. id=import_id,
  151. status=ImportStatus.FAILED,
  152. error=f"Error fetching YAML from URL: {str(e)}",
  153. )
  154. elif mode == ImportMode.YAML_CONTENT:
  155. if not yaml_content:
  156. return Import(
  157. id=import_id,
  158. status=ImportStatus.FAILED,
  159. error="yaml_content is required when import_mode is yaml-content",
  160. )
  161. content = yaml_content
  162. # Process YAML content
  163. try:
  164. # Parse YAML to validate format
  165. data = yaml.safe_load(content)
  166. if not isinstance(data, dict):
  167. return Import(
  168. id=import_id,
  169. status=ImportStatus.FAILED,
  170. error="Invalid YAML format: content must be a mapping",
  171. )
  172. # Validate and fix DSL version
  173. if not data.get("version"):
  174. data["version"] = "0.1.0"
  175. if not data.get("kind") or data.get("kind") != "app":
  176. data["kind"] = "app"
  177. imported_version = data.get("version", "0.1.0")
  178. # check if imported_version is a float-like string
  179. if not isinstance(imported_version, str):
  180. raise ValueError(f"Invalid version type, expected str, got {type(imported_version)}")
  181. status = _check_version_compatibility(imported_version)
  182. # Extract app data
  183. app_data = data.get("app")
  184. if not app_data:
  185. return Import(
  186. id=import_id,
  187. status=ImportStatus.FAILED,
  188. error="Missing app data in YAML content",
  189. )
  190. # If app_id is provided, check if it exists
  191. app = None
  192. if app_id:
  193. stmt = select(App).where(App.id == app_id, App.tenant_id == account.current_tenant_id)
  194. app = self._session.scalar(stmt)
  195. if not app:
  196. return Import(
  197. id=import_id,
  198. status=ImportStatus.FAILED,
  199. error="App not found",
  200. )
  201. if app.mode not in [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]:
  202. return Import(
  203. id=import_id,
  204. status=ImportStatus.FAILED,
  205. error="Only workflow or advanced chat apps can be overwritten",
  206. )
  207. # If major version mismatch, store import info in Redis
  208. if status == ImportStatus.PENDING:
  209. pending_data = PendingData(
  210. import_mode=import_mode,
  211. yaml_content=content,
  212. name=name,
  213. description=description,
  214. icon_type=icon_type,
  215. icon=icon,
  216. icon_background=icon_background,
  217. app_id=app_id,
  218. )
  219. redis_client.setex(
  220. f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}",
  221. IMPORT_INFO_REDIS_EXPIRY,
  222. pending_data.model_dump_json(),
  223. )
  224. return Import(
  225. id=import_id,
  226. status=status,
  227. app_id=app_id,
  228. imported_dsl_version=imported_version,
  229. )
  230. # Extract dependencies
  231. dependencies = data.get("dependencies", [])
  232. check_dependencies_pending_data = None
  233. if dependencies:
  234. check_dependencies_pending_data = [PluginDependency.model_validate(d) for d in dependencies]
  235. elif parse_version(imported_version) <= parse_version("0.1.5"):
  236. if "workflow" in data:
  237. graph = data.get("workflow", {}).get("graph", {})
  238. dependencies_list = self._extract_dependencies_from_workflow_graph(graph)
  239. else:
  240. dependencies_list = self._extract_dependencies_from_model_config(data.get("model_config", {}))
  241. check_dependencies_pending_data = DependenciesAnalysisService.generate_latest_dependencies(
  242. dependencies_list
  243. )
  244. # Create or update app
  245. app = self._create_or_update_app(
  246. app=app,
  247. data=data,
  248. account=account,
  249. name=name,
  250. description=description,
  251. icon_type=icon_type,
  252. icon=icon,
  253. icon_background=icon_background,
  254. dependencies=check_dependencies_pending_data,
  255. )
  256. draft_var_srv = WorkflowDraftVariableService(session=self._session)
  257. draft_var_srv.delete_workflow_variables(app_id=app.id)
  258. return Import(
  259. id=import_id,
  260. status=status,
  261. app_id=app.id,
  262. app_mode=app.mode,
  263. imported_dsl_version=imported_version,
  264. )
  265. except yaml.YAMLError as e:
  266. return Import(
  267. id=import_id,
  268. status=ImportStatus.FAILED,
  269. error=f"Invalid YAML format: {str(e)}",
  270. )
  271. except Exception as e:
  272. logger.exception("Failed to import app")
  273. return Import(
  274. id=import_id,
  275. status=ImportStatus.FAILED,
  276. error=str(e),
  277. )
  278. def confirm_import(self, *, import_id: str, account: Account) -> Import:
  279. """
  280. Confirm an import that requires confirmation
  281. """
  282. redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}"
  283. pending_data = redis_client.get(redis_key)
  284. if not pending_data:
  285. return Import(
  286. id=import_id,
  287. status=ImportStatus.FAILED,
  288. error="Import information expired or does not exist",
  289. )
  290. try:
  291. if not isinstance(pending_data, str | bytes):
  292. return Import(
  293. id=import_id,
  294. status=ImportStatus.FAILED,
  295. error="Invalid import information",
  296. )
  297. pending_data = PendingData.model_validate_json(pending_data)
  298. data = yaml.safe_load(pending_data.yaml_content)
  299. app = None
  300. if pending_data.app_id:
  301. stmt = select(App).where(App.id == pending_data.app_id, App.tenant_id == account.current_tenant_id)
  302. app = self._session.scalar(stmt)
  303. # Create or update app
  304. app = self._create_or_update_app(
  305. app=app,
  306. data=data,
  307. account=account,
  308. name=pending_data.name,
  309. description=pending_data.description,
  310. icon_type=pending_data.icon_type,
  311. icon=pending_data.icon,
  312. icon_background=pending_data.icon_background,
  313. )
  314. # Delete import info from Redis
  315. redis_client.delete(redis_key)
  316. return Import(
  317. id=import_id,
  318. status=ImportStatus.COMPLETED,
  319. app_id=app.id,
  320. app_mode=app.mode,
  321. current_dsl_version=CURRENT_DSL_VERSION,
  322. imported_dsl_version=data.get("version", "0.1.0"),
  323. )
  324. except Exception as e:
  325. logger.exception("Error confirming import")
  326. return Import(
  327. id=import_id,
  328. status=ImportStatus.FAILED,
  329. error=str(e),
  330. )
  331. def check_dependencies(
  332. self,
  333. *,
  334. app_model: App,
  335. ) -> CheckDependenciesResult:
  336. """Check dependencies"""
  337. # Get dependencies from Redis
  338. redis_key = f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app_model.id}"
  339. dependencies = redis_client.get(redis_key)
  340. if not dependencies:
  341. return CheckDependenciesResult()
  342. # Extract dependencies
  343. dependencies = CheckDependenciesPendingData.model_validate_json(dependencies)
  344. # Get leaked dependencies
  345. leaked_dependencies = DependenciesAnalysisService.get_leaked_dependencies(
  346. tenant_id=app_model.tenant_id, dependencies=dependencies.dependencies
  347. )
  348. return CheckDependenciesResult(
  349. leaked_dependencies=leaked_dependencies,
  350. )
  351. def _create_or_update_app(
  352. self,
  353. *,
  354. app: Optional[App],
  355. data: dict,
  356. account: Account,
  357. name: Optional[str] = None,
  358. description: Optional[str] = None,
  359. icon_type: Optional[str] = None,
  360. icon: Optional[str] = None,
  361. icon_background: Optional[str] = None,
  362. dependencies: Optional[list[PluginDependency]] = None,
  363. ) -> App:
  364. """Create a new app or update an existing one."""
  365. app_data = data.get("app", {})
  366. app_mode = app_data.get("mode")
  367. if not app_mode:
  368. raise ValueError("loss app mode")
  369. app_mode = AppMode(app_mode)
  370. # Set icon type
  371. icon_type_value = icon_type or app_data.get("icon_type")
  372. if icon_type_value in ["emoji", "link", "image"]:
  373. icon_type = icon_type_value
  374. else:
  375. icon_type = "emoji"
  376. icon = icon or str(app_data.get("icon", ""))
  377. if app:
  378. # Update existing app
  379. app.name = name or app_data.get("name", app.name)
  380. app.description = description or app_data.get("description", app.description)
  381. app.icon_type = icon_type
  382. app.icon = icon
  383. app.icon_background = icon_background or app_data.get("icon_background", app.icon_background)
  384. app.updated_by = account.id
  385. else:
  386. if account.current_tenant_id is None:
  387. raise ValueError("Current tenant is not set")
  388. # Create new app
  389. app = App()
  390. app.id = str(uuid4())
  391. app.tenant_id = account.current_tenant_id
  392. app.mode = app_mode.value
  393. app.name = name or app_data.get("name", "")
  394. app.description = description or app_data.get("description", "")
  395. app.icon_type = icon_type
  396. app.icon = icon
  397. app.icon_background = icon_background or app_data.get("icon_background", "#FFFFFF")
  398. app.enable_site = True
  399. app.enable_api = True
  400. app.use_icon_as_answer_icon = app_data.get("use_icon_as_answer_icon", False)
  401. app.created_by = account.id
  402. app.updated_by = account.id
  403. self._session.add(app)
  404. self._session.commit()
  405. app_was_created.send(app, account=account)
  406. # save dependencies
  407. if dependencies:
  408. redis_client.setex(
  409. f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app.id}",
  410. IMPORT_INFO_REDIS_EXPIRY,
  411. CheckDependenciesPendingData(app_id=app.id, dependencies=dependencies).model_dump_json(),
  412. )
  413. # Initialize app based on mode
  414. if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
  415. workflow_data = data.get("workflow")
  416. if not workflow_data or not isinstance(workflow_data, dict):
  417. raise ValueError("Missing workflow data for workflow/advanced chat app")
  418. environment_variables_list = workflow_data.get("environment_variables", [])
  419. environment_variables = [
  420. variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
  421. ]
  422. conversation_variables_list = workflow_data.get("conversation_variables", [])
  423. conversation_variables = [
  424. variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
  425. ]
  426. workflow_service = WorkflowService()
  427. current_draft_workflow = workflow_service.get_draft_workflow(app_model=app)
  428. if current_draft_workflow:
  429. unique_hash = current_draft_workflow.unique_hash
  430. else:
  431. unique_hash = None
  432. graph = workflow_data.get("graph", {})
  433. for node in graph.get("nodes", []):
  434. if node.get("data", {}).get("type", "") == NodeType.KNOWLEDGE_RETRIEVAL.value:
  435. dataset_ids = node["data"].get("dataset_ids", [])
  436. node["data"]["dataset_ids"] = [
  437. decrypted_id
  438. for dataset_id in dataset_ids
  439. if (decrypted_id := self.decrypt_dataset_id(encrypted_data=dataset_id, tenant_id=app.tenant_id))
  440. ]
  441. workflow_service.sync_draft_workflow(
  442. app_model=app,
  443. graph=workflow_data.get("graph", {}),
  444. features=workflow_data.get("features", {}),
  445. unique_hash=unique_hash,
  446. account=account,
  447. environment_variables=environment_variables,
  448. conversation_variables=conversation_variables,
  449. )
  450. elif app_mode in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION}:
  451. # Initialize model config
  452. model_config = data.get("model_config")
  453. if not model_config or not isinstance(model_config, dict):
  454. raise ValueError("Missing model_config for chat/agent-chat/completion app")
  455. # Initialize or update model config
  456. if not app.app_model_config:
  457. app_model_config = AppModelConfig().from_model_config_dict(model_config)
  458. app_model_config.id = str(uuid4())
  459. app_model_config.app_id = app.id
  460. app_model_config.created_by = account.id
  461. app_model_config.updated_by = account.id
  462. app.app_model_config_id = app_model_config.id
  463. self._session.add(app_model_config)
  464. app_model_config_was_updated.send(app, app_model_config=app_model_config)
  465. else:
  466. raise ValueError("Invalid app mode")
  467. return app
  468. @classmethod
  469. def export_dsl(cls, app_model: App, include_secret: bool = False, workflow_id: Optional[str] = None) -> str:
  470. """
  471. Export app
  472. :param app_model: App instance
  473. :param include_secret: Whether include secret variable
  474. :return:
  475. """
  476. app_mode = AppMode.value_of(app_model.mode)
  477. export_data = {
  478. "version": CURRENT_DSL_VERSION,
  479. "kind": "app",
  480. "app": {
  481. "name": app_model.name,
  482. "mode": app_model.mode,
  483. "icon": "🤖" if app_model.icon_type == "image" else app_model.icon,
  484. "icon_background": "#FFEAD5" if app_model.icon_type == "image" else app_model.icon_background,
  485. "description": app_model.description,
  486. "use_icon_as_answer_icon": app_model.use_icon_as_answer_icon,
  487. },
  488. }
  489. if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
  490. cls._append_workflow_export_data(
  491. export_data=export_data, app_model=app_model, include_secret=include_secret, workflow_id=workflow_id
  492. )
  493. else:
  494. cls._append_model_config_export_data(export_data, app_model)
  495. return yaml.dump(export_data, allow_unicode=True) # type: ignore
  496. @classmethod
  497. def _append_workflow_export_data(
  498. cls, *, export_data: dict, app_model: App, include_secret: bool, workflow_id: Optional[str] = None
  499. ):
  500. """
  501. Append workflow export data
  502. :param export_data: export data
  503. :param app_model: App instance
  504. """
  505. workflow_service = WorkflowService()
  506. workflow = workflow_service.get_draft_workflow(app_model, workflow_id)
  507. if not workflow:
  508. raise ValueError("Missing draft workflow configuration, please check.")
  509. workflow_dict = workflow.to_dict(include_secret=include_secret)
  510. # TODO: refactor: we need a better way to filter workspace related data from nodes
  511. for node in workflow_dict.get("graph", {}).get("nodes", []):
  512. node_data = node.get("data", {})
  513. if not node_data:
  514. continue
  515. data_type = node_data.get("type", "")
  516. if data_type == NodeType.KNOWLEDGE_RETRIEVAL.value:
  517. dataset_ids = node_data.get("dataset_ids", [])
  518. node_data["dataset_ids"] = [
  519. cls.encrypt_dataset_id(dataset_id=dataset_id, tenant_id=app_model.tenant_id)
  520. for dataset_id in dataset_ids
  521. ]
  522. # filter credential id from tool node
  523. if not include_secret and data_type == NodeType.TOOL.value:
  524. node_data.pop("credential_id", None)
  525. # filter credential id from agent node
  526. if not include_secret and data_type == NodeType.AGENT.value:
  527. for tool in node_data.get("agent_parameters", {}).get("tools", {}).get("value", []):
  528. tool.pop("credential_id", None)
  529. export_data["workflow"] = workflow_dict
  530. dependencies = cls._extract_dependencies_from_workflow(workflow)
  531. export_data["dependencies"] = [
  532. jsonable_encoder(d.model_dump())
  533. for d in DependenciesAnalysisService.generate_dependencies(
  534. tenant_id=app_model.tenant_id, dependencies=dependencies
  535. )
  536. ]
  537. @classmethod
  538. def _append_model_config_export_data(cls, export_data: dict, app_model: App):
  539. """
  540. Append model config export data
  541. :param export_data: export data
  542. :param app_model: App instance
  543. """
  544. app_model_config = app_model.app_model_config
  545. if not app_model_config:
  546. raise ValueError("Missing app configuration, please check.")
  547. model_config = app_model_config.to_dict()
  548. # TODO: refactor: we need a better way to filter workspace related data from model config
  549. # filter credential id from model config
  550. for tool in model_config.get("agent_mode", {}).get("tools", []):
  551. tool.pop("credential_id", None)
  552. export_data["model_config"] = model_config
  553. dependencies = cls._extract_dependencies_from_model_config(app_model_config.to_dict())
  554. export_data["dependencies"] = [
  555. jsonable_encoder(d.model_dump())
  556. for d in DependenciesAnalysisService.generate_dependencies(
  557. tenant_id=app_model.tenant_id, dependencies=dependencies
  558. )
  559. ]
  560. @classmethod
  561. def _extract_dependencies_from_workflow(cls, workflow: Workflow) -> list[str]:
  562. """
  563. Extract dependencies from workflow
  564. :param workflow: Workflow instance
  565. :return: dependencies list format like ["langgenius/google"]
  566. """
  567. graph = workflow.graph_dict
  568. dependencies = cls._extract_dependencies_from_workflow_graph(graph)
  569. return dependencies
  570. @classmethod
  571. def _extract_dependencies_from_workflow_graph(cls, graph: Mapping) -> list[str]:
  572. """
  573. Extract dependencies from workflow graph
  574. :param graph: Workflow graph
  575. :return: dependencies list format like ["langgenius/google"]
  576. """
  577. dependencies = []
  578. for node in graph.get("nodes", []):
  579. try:
  580. typ = node.get("data", {}).get("type")
  581. match typ:
  582. case NodeType.TOOL.value:
  583. tool_entity = ToolNodeData(**node["data"])
  584. dependencies.append(
  585. DependenciesAnalysisService.analyze_tool_dependency(tool_entity.provider_id),
  586. )
  587. case NodeType.LLM.value:
  588. llm_entity = LLMNodeData(**node["data"])
  589. dependencies.append(
  590. DependenciesAnalysisService.analyze_model_provider_dependency(llm_entity.model.provider),
  591. )
  592. case NodeType.QUESTION_CLASSIFIER.value:
  593. question_classifier_entity = QuestionClassifierNodeData(**node["data"])
  594. dependencies.append(
  595. DependenciesAnalysisService.analyze_model_provider_dependency(
  596. question_classifier_entity.model.provider
  597. ),
  598. )
  599. case NodeType.PARAMETER_EXTRACTOR.value:
  600. parameter_extractor_entity = ParameterExtractorNodeData(**node["data"])
  601. dependencies.append(
  602. DependenciesAnalysisService.analyze_model_provider_dependency(
  603. parameter_extractor_entity.model.provider
  604. ),
  605. )
  606. case NodeType.KNOWLEDGE_RETRIEVAL.value:
  607. knowledge_retrieval_entity = KnowledgeRetrievalNodeData(**node["data"])
  608. if knowledge_retrieval_entity.retrieval_mode == "multiple":
  609. if knowledge_retrieval_entity.multiple_retrieval_config:
  610. if (
  611. knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
  612. == "reranking_model"
  613. ):
  614. if knowledge_retrieval_entity.multiple_retrieval_config.reranking_model:
  615. dependencies.append(
  616. DependenciesAnalysisService.analyze_model_provider_dependency(
  617. knowledge_retrieval_entity.multiple_retrieval_config.reranking_model.provider
  618. ),
  619. )
  620. elif (
  621. knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
  622. == "weighted_score"
  623. ):
  624. if knowledge_retrieval_entity.multiple_retrieval_config.weights:
  625. vector_setting = (
  626. knowledge_retrieval_entity.multiple_retrieval_config.weights.vector_setting
  627. )
  628. dependencies.append(
  629. DependenciesAnalysisService.analyze_model_provider_dependency(
  630. vector_setting.embedding_provider_name
  631. ),
  632. )
  633. elif knowledge_retrieval_entity.retrieval_mode == "single":
  634. model_config = knowledge_retrieval_entity.single_retrieval_config
  635. if model_config:
  636. dependencies.append(
  637. DependenciesAnalysisService.analyze_model_provider_dependency(
  638. model_config.model.provider
  639. ),
  640. )
  641. case _:
  642. # TODO: Handle default case or unknown node types
  643. pass
  644. except Exception as e:
  645. logger.exception("Error extracting node dependency", exc_info=e)
  646. return dependencies
  647. @classmethod
  648. def _extract_dependencies_from_model_config(cls, model_config: Mapping) -> list[str]:
  649. """
  650. Extract dependencies from model config
  651. :param model_config: model config dict
  652. :return: dependencies list format like ["langgenius/google"]
  653. """
  654. dependencies = []
  655. try:
  656. # completion model
  657. model_dict = model_config.get("model", {})
  658. if model_dict:
  659. dependencies.append(
  660. DependenciesAnalysisService.analyze_model_provider_dependency(model_dict.get("provider", ""))
  661. )
  662. # reranking model
  663. dataset_configs = model_config.get("dataset_configs", {})
  664. if dataset_configs:
  665. for dataset_config in dataset_configs.get("datasets", {}).get("datasets", []):
  666. if dataset_config.get("reranking_model"):
  667. dependencies.append(
  668. DependenciesAnalysisService.analyze_model_provider_dependency(
  669. dataset_config.get("reranking_model", {})
  670. .get("reranking_provider_name", {})
  671. .get("provider")
  672. )
  673. )
  674. # tools
  675. agent_configs = model_config.get("agent_mode", {})
  676. if agent_configs:
  677. for agent_config in agent_configs.get("tools", []):
  678. dependencies.append(
  679. DependenciesAnalysisService.analyze_tool_dependency(agent_config.get("provider_id"))
  680. )
  681. except Exception as e:
  682. logger.exception("Error extracting model config dependency", exc_info=e)
  683. return dependencies
  684. @classmethod
  685. def get_leaked_dependencies(cls, tenant_id: str, dsl_dependencies: list[dict]) -> list[PluginDependency]:
  686. """
  687. Returns the leaked dependencies in current workspace
  688. """
  689. dependencies = [PluginDependency(**dep) for dep in dsl_dependencies]
  690. if not dependencies:
  691. return []
  692. return DependenciesAnalysisService.get_leaked_dependencies(tenant_id=tenant_id, dependencies=dependencies)
  693. @staticmethod
  694. def _generate_aes_key(tenant_id: str) -> bytes:
  695. """Generate AES key based on tenant_id"""
  696. return hashlib.sha256(tenant_id.encode()).digest()
  697. @classmethod
  698. def encrypt_dataset_id(cls, dataset_id: str, tenant_id: str) -> str:
  699. """Encrypt dataset_id using AES-CBC mode or return plain text based on configuration"""
  700. if not dify_config.DSL_EXPORT_ENCRYPT_DATASET_ID:
  701. return dataset_id
  702. key = cls._generate_aes_key(tenant_id)
  703. iv = key[:16]
  704. cipher = AES.new(key, AES.MODE_CBC, iv)
  705. ct_bytes = cipher.encrypt(pad(dataset_id.encode(), AES.block_size))
  706. return base64.b64encode(ct_bytes).decode()
  707. @classmethod
  708. def decrypt_dataset_id(cls, encrypted_data: str, tenant_id: str) -> str | None:
  709. """AES decryption with fallback to plain text UUID"""
  710. # First, check if it's already a plain UUID (not encrypted)
  711. if cls._is_valid_uuid(encrypted_data):
  712. return encrypted_data
  713. # If it's not a UUID, try to decrypt it
  714. try:
  715. key = cls._generate_aes_key(tenant_id)
  716. iv = key[:16]
  717. cipher = AES.new(key, AES.MODE_CBC, iv)
  718. pt = unpad(cipher.decrypt(base64.b64decode(encrypted_data)), AES.block_size)
  719. decrypted_text = pt.decode()
  720. # Validate that the decrypted result is a valid UUID
  721. if cls._is_valid_uuid(decrypted_text):
  722. return decrypted_text
  723. else:
  724. # If decrypted result is not a valid UUID, it's probably not our encrypted data
  725. return None
  726. except Exception:
  727. # If decryption fails completely, return None
  728. return None
  729. @staticmethod
  730. def _is_valid_uuid(value: str) -> bool:
  731. """Check if string is a valid UUID format"""
  732. try:
  733. uuid.UUID(value)
  734. return True
  735. except (ValueError, TypeError):
  736. return False