Вы не можете выбрать более 25 тем Темы должны начинаться с буквы или цифры, могут содержать дефисы(-) и должны содержать не более 35 символов.

app_dsl_service.py 32KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803
  1. import base64
  2. import hashlib
  3. import logging
  4. import uuid
  5. from collections.abc import Mapping
  6. from enum import StrEnum
  7. from typing import Optional
  8. from urllib.parse import urlparse
  9. from uuid import uuid4
  10. import yaml # type: ignore
  11. from Crypto.Cipher import AES
  12. from Crypto.Util.Padding import pad, unpad
  13. from packaging import version
  14. from pydantic import BaseModel, Field
  15. from sqlalchemy import select
  16. from sqlalchemy.orm import Session
  17. from core.helper import ssrf_proxy
  18. from core.model_runtime.utils.encoders import jsonable_encoder
  19. from core.plugin.entities.plugin import PluginDependency
  20. from core.workflow.nodes.enums import NodeType
  21. from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData
  22. from core.workflow.nodes.llm.entities import LLMNodeData
  23. from core.workflow.nodes.parameter_extractor.entities import ParameterExtractorNodeData
  24. from core.workflow.nodes.question_classifier.entities import QuestionClassifierNodeData
  25. from core.workflow.nodes.tool.entities import ToolNodeData
  26. from events.app_event import app_model_config_was_updated, app_was_created
  27. from extensions.ext_redis import redis_client
  28. from factories import variable_factory
  29. from models import Account, App, AppMode
  30. from models.model import AppModelConfig
  31. from models.workflow import Workflow
  32. from services.plugin.dependencies_analysis import DependenciesAnalysisService
  33. from services.workflow_draft_variable_service import WorkflowDraftVariableService
  34. from services.workflow_service import WorkflowService
  35. logger = logging.getLogger(__name__)
  36. IMPORT_INFO_REDIS_KEY_PREFIX = "app_import_info:"
  37. CHECK_DEPENDENCIES_REDIS_KEY_PREFIX = "app_check_dependencies:"
  38. IMPORT_INFO_REDIS_EXPIRY = 10 * 60 # 10 minutes
  39. DSL_MAX_SIZE = 10 * 1024 * 1024 # 10MB
  40. CURRENT_DSL_VERSION = "0.3.1"
  41. class ImportMode(StrEnum):
  42. YAML_CONTENT = "yaml-content"
  43. YAML_URL = "yaml-url"
  44. class ImportStatus(StrEnum):
  45. COMPLETED = "completed"
  46. COMPLETED_WITH_WARNINGS = "completed-with-warnings"
  47. PENDING = "pending"
  48. FAILED = "failed"
  49. class Import(BaseModel):
  50. id: str
  51. status: ImportStatus
  52. app_id: Optional[str] = None
  53. app_mode: Optional[str] = None
  54. current_dsl_version: str = CURRENT_DSL_VERSION
  55. imported_dsl_version: str = ""
  56. error: str = ""
  57. class CheckDependenciesResult(BaseModel):
  58. leaked_dependencies: list[PluginDependency] = Field(default_factory=list)
  59. def _check_version_compatibility(imported_version: str) -> ImportStatus:
  60. """Determine import status based on version comparison"""
  61. try:
  62. current_ver = version.parse(CURRENT_DSL_VERSION)
  63. imported_ver = version.parse(imported_version)
  64. except version.InvalidVersion:
  65. return ImportStatus.FAILED
  66. # If imported version is newer than current, always return PENDING
  67. if imported_ver > current_ver:
  68. return ImportStatus.PENDING
  69. # If imported version is older than current's major, return PENDING
  70. if imported_ver.major < current_ver.major:
  71. return ImportStatus.PENDING
  72. # If imported version is older than current's minor, return COMPLETED_WITH_WARNINGS
  73. if imported_ver.minor < current_ver.minor:
  74. return ImportStatus.COMPLETED_WITH_WARNINGS
  75. # If imported version equals or is older than current's micro, return COMPLETED
  76. return ImportStatus.COMPLETED
  77. class PendingData(BaseModel):
  78. import_mode: str
  79. yaml_content: str
  80. name: str | None
  81. description: str | None
  82. icon_type: str | None
  83. icon: str | None
  84. icon_background: str | None
  85. app_id: str | None
  86. class CheckDependenciesPendingData(BaseModel):
  87. dependencies: list[PluginDependency]
  88. app_id: str | None
  89. class AppDslService:
  90. def __init__(self, session: Session):
  91. self._session = session
  92. def import_app(
  93. self,
  94. *,
  95. account: Account,
  96. import_mode: str,
  97. yaml_content: Optional[str] = None,
  98. yaml_url: Optional[str] = None,
  99. name: Optional[str] = None,
  100. description: Optional[str] = None,
  101. icon_type: Optional[str] = None,
  102. icon: Optional[str] = None,
  103. icon_background: Optional[str] = None,
  104. app_id: Optional[str] = None,
  105. ) -> Import:
  106. """Import an app from YAML content or URL."""
  107. import_id = str(uuid.uuid4())
  108. # Validate import mode
  109. try:
  110. mode = ImportMode(import_mode)
  111. except ValueError:
  112. raise ValueError(f"Invalid import_mode: {import_mode}")
  113. # Get YAML content
  114. content: str = ""
  115. if mode == ImportMode.YAML_URL:
  116. if not yaml_url:
  117. return Import(
  118. id=import_id,
  119. status=ImportStatus.FAILED,
  120. error="yaml_url is required when import_mode is yaml-url",
  121. )
  122. try:
  123. parsed_url = urlparse(yaml_url)
  124. if (
  125. parsed_url.scheme == "https"
  126. and parsed_url.netloc == "github.com"
  127. and parsed_url.path.endswith((".yml", ".yaml"))
  128. ):
  129. yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com")
  130. yaml_url = yaml_url.replace("/blob/", "/")
  131. response = ssrf_proxy.get(yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
  132. response.raise_for_status()
  133. content = response.content.decode()
  134. if len(content) > DSL_MAX_SIZE:
  135. return Import(
  136. id=import_id,
  137. status=ImportStatus.FAILED,
  138. error="File size exceeds the limit of 10MB",
  139. )
  140. if not content:
  141. return Import(
  142. id=import_id,
  143. status=ImportStatus.FAILED,
  144. error="Empty content from url",
  145. )
  146. except Exception as e:
  147. return Import(
  148. id=import_id,
  149. status=ImportStatus.FAILED,
  150. error=f"Error fetching YAML from URL: {str(e)}",
  151. )
  152. elif mode == ImportMode.YAML_CONTENT:
  153. if not yaml_content:
  154. return Import(
  155. id=import_id,
  156. status=ImportStatus.FAILED,
  157. error="yaml_content is required when import_mode is yaml-content",
  158. )
  159. content = yaml_content
  160. # Process YAML content
  161. try:
  162. # Parse YAML to validate format
  163. data = yaml.safe_load(content)
  164. if not isinstance(data, dict):
  165. return Import(
  166. id=import_id,
  167. status=ImportStatus.FAILED,
  168. error="Invalid YAML format: content must be a mapping",
  169. )
  170. # Validate and fix DSL version
  171. if not data.get("version"):
  172. data["version"] = "0.1.0"
  173. if not data.get("kind") or data.get("kind") != "app":
  174. data["kind"] = "app"
  175. imported_version = data.get("version", "0.1.0")
  176. # check if imported_version is a float-like string
  177. if not isinstance(imported_version, str):
  178. raise ValueError(f"Invalid version type, expected str, got {type(imported_version)}")
  179. status = _check_version_compatibility(imported_version)
  180. # Extract app data
  181. app_data = data.get("app")
  182. if not app_data:
  183. return Import(
  184. id=import_id,
  185. status=ImportStatus.FAILED,
  186. error="Missing app data in YAML content",
  187. )
  188. # If app_id is provided, check if it exists
  189. app = None
  190. if app_id:
  191. stmt = select(App).where(App.id == app_id, App.tenant_id == account.current_tenant_id)
  192. app = self._session.scalar(stmt)
  193. if not app:
  194. return Import(
  195. id=import_id,
  196. status=ImportStatus.FAILED,
  197. error="App not found",
  198. )
  199. if app.mode not in [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]:
  200. return Import(
  201. id=import_id,
  202. status=ImportStatus.FAILED,
  203. error="Only workflow or advanced chat apps can be overwritten",
  204. )
  205. # If major version mismatch, store import info in Redis
  206. if status == ImportStatus.PENDING:
  207. pending_data = PendingData(
  208. import_mode=import_mode,
  209. yaml_content=content,
  210. name=name,
  211. description=description,
  212. icon_type=icon_type,
  213. icon=icon,
  214. icon_background=icon_background,
  215. app_id=app_id,
  216. )
  217. redis_client.setex(
  218. f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}",
  219. IMPORT_INFO_REDIS_EXPIRY,
  220. pending_data.model_dump_json(),
  221. )
  222. return Import(
  223. id=import_id,
  224. status=status,
  225. app_id=app_id,
  226. imported_dsl_version=imported_version,
  227. )
  228. # Extract dependencies
  229. dependencies = data.get("dependencies", [])
  230. check_dependencies_pending_data = None
  231. if dependencies:
  232. check_dependencies_pending_data = [PluginDependency.model_validate(d) for d in dependencies]
  233. elif imported_version <= "0.1.5":
  234. if "workflow" in data:
  235. graph = data.get("workflow", {}).get("graph", {})
  236. dependencies_list = self._extract_dependencies_from_workflow_graph(graph)
  237. else:
  238. dependencies_list = self._extract_dependencies_from_model_config(data.get("model_config", {}))
  239. check_dependencies_pending_data = DependenciesAnalysisService.generate_latest_dependencies(
  240. dependencies_list
  241. )
  242. # Create or update app
  243. app = self._create_or_update_app(
  244. app=app,
  245. data=data,
  246. account=account,
  247. name=name,
  248. description=description,
  249. icon_type=icon_type,
  250. icon=icon,
  251. icon_background=icon_background,
  252. dependencies=check_dependencies_pending_data,
  253. )
  254. draft_var_srv = WorkflowDraftVariableService(session=self._session)
  255. draft_var_srv.delete_workflow_variables(app_id=app.id)
  256. return Import(
  257. id=import_id,
  258. status=status,
  259. app_id=app.id,
  260. app_mode=app.mode,
  261. imported_dsl_version=imported_version,
  262. )
  263. except yaml.YAMLError as e:
  264. return Import(
  265. id=import_id,
  266. status=ImportStatus.FAILED,
  267. error=f"Invalid YAML format: {str(e)}",
  268. )
  269. except Exception as e:
  270. logger.exception("Failed to import app")
  271. return Import(
  272. id=import_id,
  273. status=ImportStatus.FAILED,
  274. error=str(e),
  275. )
  276. def confirm_import(self, *, import_id: str, account: Account) -> Import:
  277. """
  278. Confirm an import that requires confirmation
  279. """
  280. redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}"
  281. pending_data = redis_client.get(redis_key)
  282. if not pending_data:
  283. return Import(
  284. id=import_id,
  285. status=ImportStatus.FAILED,
  286. error="Import information expired or does not exist",
  287. )
  288. try:
  289. if not isinstance(pending_data, str | bytes):
  290. return Import(
  291. id=import_id,
  292. status=ImportStatus.FAILED,
  293. error="Invalid import information",
  294. )
  295. pending_data = PendingData.model_validate_json(pending_data)
  296. data = yaml.safe_load(pending_data.yaml_content)
  297. app = None
  298. if pending_data.app_id:
  299. stmt = select(App).where(App.id == pending_data.app_id, App.tenant_id == account.current_tenant_id)
  300. app = self._session.scalar(stmt)
  301. # Create or update app
  302. app = self._create_or_update_app(
  303. app=app,
  304. data=data,
  305. account=account,
  306. name=pending_data.name,
  307. description=pending_data.description,
  308. icon_type=pending_data.icon_type,
  309. icon=pending_data.icon,
  310. icon_background=pending_data.icon_background,
  311. )
  312. # Delete import info from Redis
  313. redis_client.delete(redis_key)
  314. return Import(
  315. id=import_id,
  316. status=ImportStatus.COMPLETED,
  317. app_id=app.id,
  318. app_mode=app.mode,
  319. current_dsl_version=CURRENT_DSL_VERSION,
  320. imported_dsl_version=data.get("version", "0.1.0"),
  321. )
  322. except Exception as e:
  323. logger.exception("Error confirming import")
  324. return Import(
  325. id=import_id,
  326. status=ImportStatus.FAILED,
  327. error=str(e),
  328. )
  329. def check_dependencies(
  330. self,
  331. *,
  332. app_model: App,
  333. ) -> CheckDependenciesResult:
  334. """Check dependencies"""
  335. # Get dependencies from Redis
  336. redis_key = f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app_model.id}"
  337. dependencies = redis_client.get(redis_key)
  338. if not dependencies:
  339. return CheckDependenciesResult()
  340. # Extract dependencies
  341. dependencies = CheckDependenciesPendingData.model_validate_json(dependencies)
  342. # Get leaked dependencies
  343. leaked_dependencies = DependenciesAnalysisService.get_leaked_dependencies(
  344. tenant_id=app_model.tenant_id, dependencies=dependencies.dependencies
  345. )
  346. return CheckDependenciesResult(
  347. leaked_dependencies=leaked_dependencies,
  348. )
  349. def _create_or_update_app(
  350. self,
  351. *,
  352. app: Optional[App],
  353. data: dict,
  354. account: Account,
  355. name: Optional[str] = None,
  356. description: Optional[str] = None,
  357. icon_type: Optional[str] = None,
  358. icon: Optional[str] = None,
  359. icon_background: Optional[str] = None,
  360. dependencies: Optional[list[PluginDependency]] = None,
  361. ) -> App:
  362. """Create a new app or update an existing one."""
  363. app_data = data.get("app", {})
  364. app_mode = app_data.get("mode")
  365. if not app_mode:
  366. raise ValueError("loss app mode")
  367. app_mode = AppMode(app_mode)
  368. # Set icon type
  369. icon_type_value = icon_type or app_data.get("icon_type")
  370. if icon_type_value in ["emoji", "link", "image"]:
  371. icon_type = icon_type_value
  372. else:
  373. icon_type = "emoji"
  374. icon = icon or str(app_data.get("icon", ""))
  375. if app:
  376. # Update existing app
  377. app.name = name or app_data.get("name", app.name)
  378. app.description = description or app_data.get("description", app.description)
  379. app.icon_type = icon_type
  380. app.icon = icon
  381. app.icon_background = icon_background or app_data.get("icon_background", app.icon_background)
  382. app.updated_by = account.id
  383. else:
  384. if account.current_tenant_id is None:
  385. raise ValueError("Current tenant is not set")
  386. # Create new app
  387. app = App()
  388. app.id = str(uuid4())
  389. app.tenant_id = account.current_tenant_id
  390. app.mode = app_mode.value
  391. app.name = name or app_data.get("name", "")
  392. app.description = description or app_data.get("description", "")
  393. app.icon_type = icon_type
  394. app.icon = icon
  395. app.icon_background = icon_background or app_data.get("icon_background", "#FFFFFF")
  396. app.enable_site = True
  397. app.enable_api = True
  398. app.use_icon_as_answer_icon = app_data.get("use_icon_as_answer_icon", False)
  399. app.created_by = account.id
  400. app.updated_by = account.id
  401. self._session.add(app)
  402. self._session.commit()
  403. app_was_created.send(app, account=account)
  404. # save dependencies
  405. if dependencies:
  406. redis_client.setex(
  407. f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app.id}",
  408. IMPORT_INFO_REDIS_EXPIRY,
  409. CheckDependenciesPendingData(app_id=app.id, dependencies=dependencies).model_dump_json(),
  410. )
  411. # Initialize app based on mode
  412. if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
  413. workflow_data = data.get("workflow")
  414. if not workflow_data or not isinstance(workflow_data, dict):
  415. raise ValueError("Missing workflow data for workflow/advanced chat app")
  416. environment_variables_list = workflow_data.get("environment_variables", [])
  417. environment_variables = [
  418. variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
  419. ]
  420. conversation_variables_list = workflow_data.get("conversation_variables", [])
  421. conversation_variables = [
  422. variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
  423. ]
  424. workflow_service = WorkflowService()
  425. current_draft_workflow = workflow_service.get_draft_workflow(app_model=app)
  426. if current_draft_workflow:
  427. unique_hash = current_draft_workflow.unique_hash
  428. else:
  429. unique_hash = None
  430. graph = workflow_data.get("graph", {})
  431. for node in graph.get("nodes", []):
  432. if node.get("data", {}).get("type", "") == NodeType.KNOWLEDGE_RETRIEVAL.value:
  433. dataset_ids = node["data"].get("dataset_ids", [])
  434. node["data"]["dataset_ids"] = [
  435. decrypted_id
  436. for dataset_id in dataset_ids
  437. if (decrypted_id := self.decrypt_dataset_id(encrypted_data=dataset_id, tenant_id=app.tenant_id))
  438. ]
  439. workflow_service.sync_draft_workflow(
  440. app_model=app,
  441. graph=workflow_data.get("graph", {}),
  442. features=workflow_data.get("features", {}),
  443. unique_hash=unique_hash,
  444. account=account,
  445. environment_variables=environment_variables,
  446. conversation_variables=conversation_variables,
  447. )
  448. elif app_mode in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION}:
  449. # Initialize model config
  450. model_config = data.get("model_config")
  451. if not model_config or not isinstance(model_config, dict):
  452. raise ValueError("Missing model_config for chat/agent-chat/completion app")
  453. # Initialize or update model config
  454. if not app.app_model_config:
  455. app_model_config = AppModelConfig().from_model_config_dict(model_config)
  456. app_model_config.id = str(uuid4())
  457. app_model_config.app_id = app.id
  458. app_model_config.created_by = account.id
  459. app_model_config.updated_by = account.id
  460. app.app_model_config_id = app_model_config.id
  461. self._session.add(app_model_config)
  462. app_model_config_was_updated.send(app, app_model_config=app_model_config)
  463. else:
  464. raise ValueError("Invalid app mode")
  465. return app
  466. @classmethod
  467. def export_dsl(cls, app_model: App, include_secret: bool = False) -> str:
  468. """
  469. Export app
  470. :param app_model: App instance
  471. :param include_secret: Whether include secret variable
  472. :return:
  473. """
  474. app_mode = AppMode.value_of(app_model.mode)
  475. export_data = {
  476. "version": CURRENT_DSL_VERSION,
  477. "kind": "app",
  478. "app": {
  479. "name": app_model.name,
  480. "mode": app_model.mode,
  481. "icon": "🤖" if app_model.icon_type == "image" else app_model.icon,
  482. "icon_background": "#FFEAD5" if app_model.icon_type == "image" else app_model.icon_background,
  483. "description": app_model.description,
  484. "use_icon_as_answer_icon": app_model.use_icon_as_answer_icon,
  485. },
  486. }
  487. if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
  488. cls._append_workflow_export_data(
  489. export_data=export_data, app_model=app_model, include_secret=include_secret
  490. )
  491. else:
  492. cls._append_model_config_export_data(export_data, app_model)
  493. return yaml.dump(export_data, allow_unicode=True) # type: ignore
  494. @classmethod
  495. def _append_workflow_export_data(cls, *, export_data: dict, app_model: App, include_secret: bool) -> None:
  496. """
  497. Append workflow export data
  498. :param export_data: export data
  499. :param app_model: App instance
  500. """
  501. workflow_service = WorkflowService()
  502. workflow = workflow_service.get_draft_workflow(app_model)
  503. if not workflow:
  504. raise ValueError("Missing draft workflow configuration, please check.")
  505. workflow_dict = workflow.to_dict(include_secret=include_secret)
  506. # TODO: refactor: we need a better way to filter workspace related data from nodes
  507. for node in workflow_dict.get("graph", {}).get("nodes", []):
  508. node_data = node.get("data", {})
  509. if not node_data:
  510. continue
  511. data_type = node_data.get("type", "")
  512. if data_type == NodeType.KNOWLEDGE_RETRIEVAL.value:
  513. dataset_ids = node_data.get("dataset_ids", [])
  514. node_data["dataset_ids"] = [
  515. cls.encrypt_dataset_id(dataset_id=dataset_id, tenant_id=app_model.tenant_id)
  516. for dataset_id in dataset_ids
  517. ]
  518. # filter credential id from tool node
  519. if not include_secret and data_type == NodeType.TOOL.value:
  520. node_data.pop("credential_id", None)
  521. # filter credential id from agent node
  522. if not include_secret and data_type == NodeType.AGENT.value:
  523. for tool in node_data.get("agent_parameters", {}).get("tools", {}).get("value", []):
  524. tool.pop("credential_id", None)
  525. export_data["workflow"] = workflow_dict
  526. dependencies = cls._extract_dependencies_from_workflow(workflow)
  527. export_data["dependencies"] = [
  528. jsonable_encoder(d.model_dump())
  529. for d in DependenciesAnalysisService.generate_dependencies(
  530. tenant_id=app_model.tenant_id, dependencies=dependencies
  531. )
  532. ]
  533. @classmethod
  534. def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None:
  535. """
  536. Append model config export data
  537. :param export_data: export data
  538. :param app_model: App instance
  539. """
  540. app_model_config = app_model.app_model_config
  541. if not app_model_config:
  542. raise ValueError("Missing app configuration, please check.")
  543. model_config = app_model_config.to_dict()
  544. # TODO: refactor: we need a better way to filter workspace related data from model config
  545. # filter credential id from model config
  546. for tool in model_config.get("agent_mode", {}).get("tools", []):
  547. tool.pop("credential_id", None)
  548. export_data["model_config"] = model_config
  549. dependencies = cls._extract_dependencies_from_model_config(app_model_config.to_dict())
  550. export_data["dependencies"] = [
  551. jsonable_encoder(d.model_dump())
  552. for d in DependenciesAnalysisService.generate_dependencies(
  553. tenant_id=app_model.tenant_id, dependencies=dependencies
  554. )
  555. ]
  556. @classmethod
  557. def _extract_dependencies_from_workflow(cls, workflow: Workflow) -> list[str]:
  558. """
  559. Extract dependencies from workflow
  560. :param workflow: Workflow instance
  561. :return: dependencies list format like ["langgenius/google"]
  562. """
  563. graph = workflow.graph_dict
  564. dependencies = cls._extract_dependencies_from_workflow_graph(graph)
  565. return dependencies
  566. @classmethod
  567. def _extract_dependencies_from_workflow_graph(cls, graph: Mapping) -> list[str]:
  568. """
  569. Extract dependencies from workflow graph
  570. :param graph: Workflow graph
  571. :return: dependencies list format like ["langgenius/google"]
  572. """
  573. dependencies = []
  574. for node in graph.get("nodes", []):
  575. try:
  576. typ = node.get("data", {}).get("type")
  577. match typ:
  578. case NodeType.TOOL.value:
  579. tool_entity = ToolNodeData(**node["data"])
  580. dependencies.append(
  581. DependenciesAnalysisService.analyze_tool_dependency(tool_entity.provider_id),
  582. )
  583. case NodeType.LLM.value:
  584. llm_entity = LLMNodeData(**node["data"])
  585. dependencies.append(
  586. DependenciesAnalysisService.analyze_model_provider_dependency(llm_entity.model.provider),
  587. )
  588. case NodeType.QUESTION_CLASSIFIER.value:
  589. question_classifier_entity = QuestionClassifierNodeData(**node["data"])
  590. dependencies.append(
  591. DependenciesAnalysisService.analyze_model_provider_dependency(
  592. question_classifier_entity.model.provider
  593. ),
  594. )
  595. case NodeType.PARAMETER_EXTRACTOR.value:
  596. parameter_extractor_entity = ParameterExtractorNodeData(**node["data"])
  597. dependencies.append(
  598. DependenciesAnalysisService.analyze_model_provider_dependency(
  599. parameter_extractor_entity.model.provider
  600. ),
  601. )
  602. case NodeType.KNOWLEDGE_RETRIEVAL.value:
  603. knowledge_retrieval_entity = KnowledgeRetrievalNodeData(**node["data"])
  604. if knowledge_retrieval_entity.retrieval_mode == "multiple":
  605. if knowledge_retrieval_entity.multiple_retrieval_config:
  606. if (
  607. knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
  608. == "reranking_model"
  609. ):
  610. if knowledge_retrieval_entity.multiple_retrieval_config.reranking_model:
  611. dependencies.append(
  612. DependenciesAnalysisService.analyze_model_provider_dependency(
  613. knowledge_retrieval_entity.multiple_retrieval_config.reranking_model.provider
  614. ),
  615. )
  616. elif (
  617. knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
  618. == "weighted_score"
  619. ):
  620. if knowledge_retrieval_entity.multiple_retrieval_config.weights:
  621. vector_setting = (
  622. knowledge_retrieval_entity.multiple_retrieval_config.weights.vector_setting
  623. )
  624. dependencies.append(
  625. DependenciesAnalysisService.analyze_model_provider_dependency(
  626. vector_setting.embedding_provider_name
  627. ),
  628. )
  629. elif knowledge_retrieval_entity.retrieval_mode == "single":
  630. model_config = knowledge_retrieval_entity.single_retrieval_config
  631. if model_config:
  632. dependencies.append(
  633. DependenciesAnalysisService.analyze_model_provider_dependency(
  634. model_config.model.provider
  635. ),
  636. )
  637. case _:
  638. # TODO: Handle default case or unknown node types
  639. pass
  640. except Exception as e:
  641. logger.exception("Error extracting node dependency", exc_info=e)
  642. return dependencies
  643. @classmethod
  644. def _extract_dependencies_from_model_config(cls, model_config: Mapping) -> list[str]:
  645. """
  646. Extract dependencies from model config
  647. :param model_config: model config dict
  648. :return: dependencies list format like ["langgenius/google"]
  649. """
  650. dependencies = []
  651. try:
  652. # completion model
  653. model_dict = model_config.get("model", {})
  654. if model_dict:
  655. dependencies.append(
  656. DependenciesAnalysisService.analyze_model_provider_dependency(model_dict.get("provider", ""))
  657. )
  658. # reranking model
  659. dataset_configs = model_config.get("dataset_configs", {})
  660. if dataset_configs:
  661. for dataset_config in dataset_configs.get("datasets", {}).get("datasets", []):
  662. if dataset_config.get("reranking_model"):
  663. dependencies.append(
  664. DependenciesAnalysisService.analyze_model_provider_dependency(
  665. dataset_config.get("reranking_model", {})
  666. .get("reranking_provider_name", {})
  667. .get("provider")
  668. )
  669. )
  670. # tools
  671. agent_configs = model_config.get("agent_mode", {})
  672. if agent_configs:
  673. for agent_config in agent_configs.get("tools", []):
  674. dependencies.append(
  675. DependenciesAnalysisService.analyze_tool_dependency(agent_config.get("provider_id"))
  676. )
  677. except Exception as e:
  678. logger.exception("Error extracting model config dependency", exc_info=e)
  679. return dependencies
  680. @classmethod
  681. def get_leaked_dependencies(cls, tenant_id: str, dsl_dependencies: list[dict]) -> list[PluginDependency]:
  682. """
  683. Returns the leaked dependencies in current workspace
  684. """
  685. dependencies = [PluginDependency(**dep) for dep in dsl_dependencies]
  686. if not dependencies:
  687. return []
  688. return DependenciesAnalysisService.get_leaked_dependencies(tenant_id=tenant_id, dependencies=dependencies)
  689. @staticmethod
  690. def _generate_aes_key(tenant_id: str) -> bytes:
  691. """Generate AES key based on tenant_id"""
  692. return hashlib.sha256(tenant_id.encode()).digest()
  693. @classmethod
  694. def encrypt_dataset_id(cls, dataset_id: str, tenant_id: str) -> str:
  695. """Encrypt dataset_id using AES-CBC mode"""
  696. key = cls._generate_aes_key(tenant_id)
  697. iv = key[:16]
  698. cipher = AES.new(key, AES.MODE_CBC, iv)
  699. ct_bytes = cipher.encrypt(pad(dataset_id.encode(), AES.block_size))
  700. return base64.b64encode(ct_bytes).decode()
  701. @classmethod
  702. def decrypt_dataset_id(cls, encrypted_data: str, tenant_id: str) -> str | None:
  703. """AES decryption"""
  704. try:
  705. key = cls._generate_aes_key(tenant_id)
  706. iv = key[:16]
  707. cipher = AES.new(key, AES.MODE_CBC, iv)
  708. pt = unpad(cipher.decrypt(base64.b64decode(encrypted_data)), AES.block_size)
  709. return pt.decode()
  710. except Exception:
  711. return None