You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

app_dsl_service.py 31KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773
  1. import base64
  2. import hashlib
  3. import logging
  4. import uuid
  5. from collections.abc import Mapping
  6. from enum import StrEnum
  7. from typing import Optional
  8. from urllib.parse import urlparse
  9. from uuid import uuid4
  10. import yaml # type: ignore
  11. from Crypto.Cipher import AES
  12. from Crypto.Util.Padding import pad, unpad
  13. from packaging import version
  14. from pydantic import BaseModel, Field
  15. from sqlalchemy import select
  16. from sqlalchemy.orm import Session
  17. from core.helper import ssrf_proxy
  18. from core.model_runtime.utils.encoders import jsonable_encoder
  19. from core.plugin.entities.plugin import PluginDependency
  20. from core.workflow.nodes.enums import NodeType
  21. from core.workflow.nodes.knowledge_retrieval.entities import KnowledgeRetrievalNodeData
  22. from core.workflow.nodes.llm.entities import LLMNodeData
  23. from core.workflow.nodes.parameter_extractor.entities import ParameterExtractorNodeData
  24. from core.workflow.nodes.question_classifier.entities import QuestionClassifierNodeData
  25. from core.workflow.nodes.tool.entities import ToolNodeData
  26. from events.app_event import app_model_config_was_updated, app_was_created
  27. from extensions.ext_redis import redis_client
  28. from factories import variable_factory
  29. from models import Account, App, AppMode
  30. from models.model import AppModelConfig
  31. from models.workflow import Workflow
  32. from services.plugin.dependencies_analysis import DependenciesAnalysisService
  33. from services.workflow_service import WorkflowService
  34. logger = logging.getLogger(__name__)
  35. IMPORT_INFO_REDIS_KEY_PREFIX = "app_import_info:"
  36. CHECK_DEPENDENCIES_REDIS_KEY_PREFIX = "app_check_dependencies:"
  37. IMPORT_INFO_REDIS_EXPIRY = 10 * 60 # 10 minutes
  38. DSL_MAX_SIZE = 10 * 1024 * 1024 # 10MB
  39. CURRENT_DSL_VERSION = "0.2.0"
  40. class ImportMode(StrEnum):
  41. YAML_CONTENT = "yaml-content"
  42. YAML_URL = "yaml-url"
  43. class ImportStatus(StrEnum):
  44. COMPLETED = "completed"
  45. COMPLETED_WITH_WARNINGS = "completed-with-warnings"
  46. PENDING = "pending"
  47. FAILED = "failed"
  48. class Import(BaseModel):
  49. id: str
  50. status: ImportStatus
  51. app_id: Optional[str] = None
  52. app_mode: Optional[str] = None
  53. current_dsl_version: str = CURRENT_DSL_VERSION
  54. imported_dsl_version: str = ""
  55. error: str = ""
  56. class CheckDependenciesResult(BaseModel):
  57. leaked_dependencies: list[PluginDependency] = Field(default_factory=list)
  58. def _check_version_compatibility(imported_version: str) -> ImportStatus:
  59. """Determine import status based on version comparison"""
  60. try:
  61. current_ver = version.parse(CURRENT_DSL_VERSION)
  62. imported_ver = version.parse(imported_version)
  63. except version.InvalidVersion:
  64. return ImportStatus.FAILED
  65. # Compare major version and minor version
  66. if current_ver.major != imported_ver.major or current_ver.minor != imported_ver.minor:
  67. return ImportStatus.PENDING
  68. if current_ver.micro != imported_ver.micro:
  69. return ImportStatus.COMPLETED_WITH_WARNINGS
  70. return ImportStatus.COMPLETED
  71. class PendingData(BaseModel):
  72. import_mode: str
  73. yaml_content: str
  74. name: str | None
  75. description: str | None
  76. icon_type: str | None
  77. icon: str | None
  78. icon_background: str | None
  79. app_id: str | None
  80. class CheckDependenciesPendingData(BaseModel):
  81. dependencies: list[PluginDependency]
  82. app_id: str | None
  83. class AppDslService:
  84. def __init__(self, session: Session):
  85. self._session = session
  86. def import_app(
  87. self,
  88. *,
  89. account: Account,
  90. import_mode: str,
  91. yaml_content: Optional[str] = None,
  92. yaml_url: Optional[str] = None,
  93. name: Optional[str] = None,
  94. description: Optional[str] = None,
  95. icon_type: Optional[str] = None,
  96. icon: Optional[str] = None,
  97. icon_background: Optional[str] = None,
  98. app_id: Optional[str] = None,
  99. ) -> Import:
  100. """Import an app from YAML content or URL."""
  101. import_id = str(uuid.uuid4())
  102. # Validate import mode
  103. try:
  104. mode = ImportMode(import_mode)
  105. except ValueError:
  106. raise ValueError(f"Invalid import_mode: {import_mode}")
  107. # Get YAML content
  108. content: str = ""
  109. if mode == ImportMode.YAML_URL:
  110. if not yaml_url:
  111. return Import(
  112. id=import_id,
  113. status=ImportStatus.FAILED,
  114. error="yaml_url is required when import_mode is yaml-url",
  115. )
  116. try:
  117. parsed_url = urlparse(yaml_url)
  118. if (
  119. parsed_url.scheme == "https"
  120. and parsed_url.netloc == "github.com"
  121. and parsed_url.path.endswith((".yml", ".yaml"))
  122. ):
  123. yaml_url = yaml_url.replace("https://github.com", "https://raw.githubusercontent.com")
  124. yaml_url = yaml_url.replace("/blob/", "/")
  125. response = ssrf_proxy.get(yaml_url.strip(), follow_redirects=True, timeout=(10, 10))
  126. response.raise_for_status()
  127. content = response.content.decode()
  128. if len(content) > DSL_MAX_SIZE:
  129. return Import(
  130. id=import_id,
  131. status=ImportStatus.FAILED,
  132. error="File size exceeds the limit of 10MB",
  133. )
  134. if not content:
  135. return Import(
  136. id=import_id,
  137. status=ImportStatus.FAILED,
  138. error="Empty content from url",
  139. )
  140. except Exception as e:
  141. return Import(
  142. id=import_id,
  143. status=ImportStatus.FAILED,
  144. error=f"Error fetching YAML from URL: {str(e)}",
  145. )
  146. elif mode == ImportMode.YAML_CONTENT:
  147. if not yaml_content:
  148. return Import(
  149. id=import_id,
  150. status=ImportStatus.FAILED,
  151. error="yaml_content is required when import_mode is yaml-content",
  152. )
  153. content = yaml_content
  154. # Process YAML content
  155. try:
  156. # Parse YAML to validate format
  157. data = yaml.safe_load(content)
  158. if not isinstance(data, dict):
  159. return Import(
  160. id=import_id,
  161. status=ImportStatus.FAILED,
  162. error="Invalid YAML format: content must be a mapping",
  163. )
  164. # Validate and fix DSL version
  165. if not data.get("version"):
  166. data["version"] = "0.1.0"
  167. if not data.get("kind") or data.get("kind") != "app":
  168. data["kind"] = "app"
  169. imported_version = data.get("version", "0.1.0")
  170. # check if imported_version is a float-like string
  171. if not isinstance(imported_version, str):
  172. raise ValueError(f"Invalid version type, expected str, got {type(imported_version)}")
  173. status = _check_version_compatibility(imported_version)
  174. # Extract app data
  175. app_data = data.get("app")
  176. if not app_data:
  177. return Import(
  178. id=import_id,
  179. status=ImportStatus.FAILED,
  180. error="Missing app data in YAML content",
  181. )
  182. # If app_id is provided, check if it exists
  183. app = None
  184. if app_id:
  185. stmt = select(App).where(App.id == app_id, App.tenant_id == account.current_tenant_id)
  186. app = self._session.scalar(stmt)
  187. if not app:
  188. return Import(
  189. id=import_id,
  190. status=ImportStatus.FAILED,
  191. error="App not found",
  192. )
  193. if app.mode not in [AppMode.WORKFLOW, AppMode.ADVANCED_CHAT]:
  194. return Import(
  195. id=import_id,
  196. status=ImportStatus.FAILED,
  197. error="Only workflow or advanced chat apps can be overwritten",
  198. )
  199. # If major version mismatch, store import info in Redis
  200. if status == ImportStatus.PENDING:
  201. pending_data = PendingData(
  202. import_mode=import_mode,
  203. yaml_content=content,
  204. name=name,
  205. description=description,
  206. icon_type=icon_type,
  207. icon=icon,
  208. icon_background=icon_background,
  209. app_id=app_id,
  210. )
  211. redis_client.setex(
  212. f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}",
  213. IMPORT_INFO_REDIS_EXPIRY,
  214. pending_data.model_dump_json(),
  215. )
  216. return Import(
  217. id=import_id,
  218. status=status,
  219. app_id=app_id,
  220. imported_dsl_version=imported_version,
  221. )
  222. # Extract dependencies
  223. dependencies = data.get("dependencies", [])
  224. check_dependencies_pending_data = None
  225. if dependencies:
  226. check_dependencies_pending_data = [PluginDependency.model_validate(d) for d in dependencies]
  227. elif imported_version <= "0.1.5":
  228. if "workflow" in data:
  229. graph = data.get("workflow", {}).get("graph", {})
  230. dependencies_list = self._extract_dependencies_from_workflow_graph(graph)
  231. else:
  232. dependencies_list = self._extract_dependencies_from_model_config(data.get("model_config", {}))
  233. check_dependencies_pending_data = DependenciesAnalysisService.generate_latest_dependencies(
  234. dependencies_list
  235. )
  236. # Create or update app
  237. app = self._create_or_update_app(
  238. app=app,
  239. data=data,
  240. account=account,
  241. name=name,
  242. description=description,
  243. icon_type=icon_type,
  244. icon=icon,
  245. icon_background=icon_background,
  246. dependencies=check_dependencies_pending_data,
  247. )
  248. return Import(
  249. id=import_id,
  250. status=status,
  251. app_id=app.id,
  252. app_mode=app.mode,
  253. imported_dsl_version=imported_version,
  254. )
  255. except yaml.YAMLError as e:
  256. return Import(
  257. id=import_id,
  258. status=ImportStatus.FAILED,
  259. error=f"Invalid YAML format: {str(e)}",
  260. )
  261. except Exception as e:
  262. logger.exception("Failed to import app")
  263. return Import(
  264. id=import_id,
  265. status=ImportStatus.FAILED,
  266. error=str(e),
  267. )
  268. def confirm_import(self, *, import_id: str, account: Account) -> Import:
  269. """
  270. Confirm an import that requires confirmation
  271. """
  272. redis_key = f"{IMPORT_INFO_REDIS_KEY_PREFIX}{import_id}"
  273. pending_data = redis_client.get(redis_key)
  274. if not pending_data:
  275. return Import(
  276. id=import_id,
  277. status=ImportStatus.FAILED,
  278. error="Import information expired or does not exist",
  279. )
  280. try:
  281. if not isinstance(pending_data, str | bytes):
  282. return Import(
  283. id=import_id,
  284. status=ImportStatus.FAILED,
  285. error="Invalid import information",
  286. )
  287. pending_data = PendingData.model_validate_json(pending_data)
  288. data = yaml.safe_load(pending_data.yaml_content)
  289. app = None
  290. if pending_data.app_id:
  291. stmt = select(App).where(App.id == pending_data.app_id, App.tenant_id == account.current_tenant_id)
  292. app = self._session.scalar(stmt)
  293. # Create or update app
  294. app = self._create_or_update_app(
  295. app=app,
  296. data=data,
  297. account=account,
  298. name=pending_data.name,
  299. description=pending_data.description,
  300. icon_type=pending_data.icon_type,
  301. icon=pending_data.icon,
  302. icon_background=pending_data.icon_background,
  303. )
  304. # Delete import info from Redis
  305. redis_client.delete(redis_key)
  306. return Import(
  307. id=import_id,
  308. status=ImportStatus.COMPLETED,
  309. app_id=app.id,
  310. app_mode=app.mode,
  311. current_dsl_version=CURRENT_DSL_VERSION,
  312. imported_dsl_version=data.get("version", "0.1.0"),
  313. )
  314. except Exception as e:
  315. logger.exception("Error confirming import")
  316. return Import(
  317. id=import_id,
  318. status=ImportStatus.FAILED,
  319. error=str(e),
  320. )
  321. def check_dependencies(
  322. self,
  323. *,
  324. app_model: App,
  325. ) -> CheckDependenciesResult:
  326. """Check dependencies"""
  327. # Get dependencies from Redis
  328. redis_key = f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app_model.id}"
  329. dependencies = redis_client.get(redis_key)
  330. if not dependencies:
  331. return CheckDependenciesResult()
  332. # Extract dependencies
  333. dependencies = CheckDependenciesPendingData.model_validate_json(dependencies)
  334. # Get leaked dependencies
  335. leaked_dependencies = DependenciesAnalysisService.get_leaked_dependencies(
  336. tenant_id=app_model.tenant_id, dependencies=dependencies.dependencies
  337. )
  338. return CheckDependenciesResult(
  339. leaked_dependencies=leaked_dependencies,
  340. )
  341. def _create_or_update_app(
  342. self,
  343. *,
  344. app: Optional[App],
  345. data: dict,
  346. account: Account,
  347. name: Optional[str] = None,
  348. description: Optional[str] = None,
  349. icon_type: Optional[str] = None,
  350. icon: Optional[str] = None,
  351. icon_background: Optional[str] = None,
  352. dependencies: Optional[list[PluginDependency]] = None,
  353. ) -> App:
  354. """Create a new app or update an existing one."""
  355. app_data = data.get("app", {})
  356. app_mode = app_data.get("mode")
  357. if not app_mode:
  358. raise ValueError("loss app mode")
  359. app_mode = AppMode(app_mode)
  360. # Set icon type
  361. icon_type_value = icon_type or app_data.get("icon_type")
  362. if icon_type_value in ["emoji", "link"]:
  363. icon_type = icon_type_value
  364. else:
  365. icon_type = "emoji"
  366. icon = icon or str(app_data.get("icon", ""))
  367. if app:
  368. # Update existing app
  369. app.name = name or app_data.get("name", app.name)
  370. app.description = description or app_data.get("description", app.description)
  371. app.icon_type = icon_type
  372. app.icon = icon
  373. app.icon_background = icon_background or app_data.get("icon_background", app.icon_background)
  374. app.updated_by = account.id
  375. else:
  376. if account.current_tenant_id is None:
  377. raise ValueError("Current tenant is not set")
  378. # Create new app
  379. app = App()
  380. app.id = str(uuid4())
  381. app.tenant_id = account.current_tenant_id
  382. app.mode = app_mode.value
  383. app.name = name or app_data.get("name", "")
  384. app.description = description or app_data.get("description", "")
  385. app.icon_type = icon_type
  386. app.icon = icon
  387. app.icon_background = icon_background or app_data.get("icon_background", "#FFFFFF")
  388. app.enable_site = True
  389. app.enable_api = True
  390. app.use_icon_as_answer_icon = app_data.get("use_icon_as_answer_icon", False)
  391. app.created_by = account.id
  392. app.updated_by = account.id
  393. self._session.add(app)
  394. self._session.commit()
  395. app_was_created.send(app, account=account)
  396. # save dependencies
  397. if dependencies:
  398. redis_client.setex(
  399. f"{CHECK_DEPENDENCIES_REDIS_KEY_PREFIX}{app.id}",
  400. IMPORT_INFO_REDIS_EXPIRY,
  401. CheckDependenciesPendingData(app_id=app.id, dependencies=dependencies).model_dump_json(),
  402. )
  403. # Initialize app based on mode
  404. if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
  405. workflow_data = data.get("workflow")
  406. if not workflow_data or not isinstance(workflow_data, dict):
  407. raise ValueError("Missing workflow data for workflow/advanced chat app")
  408. environment_variables_list = workflow_data.get("environment_variables", [])
  409. environment_variables = [
  410. variable_factory.build_environment_variable_from_mapping(obj) for obj in environment_variables_list
  411. ]
  412. conversation_variables_list = workflow_data.get("conversation_variables", [])
  413. conversation_variables = [
  414. variable_factory.build_conversation_variable_from_mapping(obj) for obj in conversation_variables_list
  415. ]
  416. workflow_service = WorkflowService()
  417. current_draft_workflow = workflow_service.get_draft_workflow(app_model=app)
  418. if current_draft_workflow:
  419. unique_hash = current_draft_workflow.unique_hash
  420. else:
  421. unique_hash = None
  422. graph = workflow_data.get("graph", {})
  423. for node in graph.get("nodes", []):
  424. if node.get("data", {}).get("type", "") == NodeType.KNOWLEDGE_RETRIEVAL.value:
  425. dataset_ids = node["data"].get("dataset_ids", [])
  426. node["data"]["dataset_ids"] = [
  427. decrypted_id
  428. for dataset_id in dataset_ids
  429. if (decrypted_id := self.decrypt_dataset_id(encrypted_data=dataset_id, tenant_id=app.tenant_id))
  430. ]
  431. workflow_service.sync_draft_workflow(
  432. app_model=app,
  433. graph=workflow_data.get("graph", {}),
  434. features=workflow_data.get("features", {}),
  435. unique_hash=unique_hash,
  436. account=account,
  437. environment_variables=environment_variables,
  438. conversation_variables=conversation_variables,
  439. )
  440. elif app_mode in {AppMode.CHAT, AppMode.AGENT_CHAT, AppMode.COMPLETION}:
  441. # Initialize model config
  442. model_config = data.get("model_config")
  443. if not model_config or not isinstance(model_config, dict):
  444. raise ValueError("Missing model_config for chat/agent-chat/completion app")
  445. # Initialize or update model config
  446. if not app.app_model_config:
  447. app_model_config = AppModelConfig().from_model_config_dict(model_config)
  448. app_model_config.id = str(uuid4())
  449. app_model_config.app_id = app.id
  450. app_model_config.created_by = account.id
  451. app_model_config.updated_by = account.id
  452. app.app_model_config_id = app_model_config.id
  453. self._session.add(app_model_config)
  454. app_model_config_was_updated.send(app, app_model_config=app_model_config)
  455. else:
  456. raise ValueError("Invalid app mode")
  457. return app
  458. @classmethod
  459. def export_dsl(cls, app_model: App, include_secret: bool = False) -> str:
  460. """
  461. Export app
  462. :param app_model: App instance
  463. :param include_secret: Whether include secret variable
  464. :return:
  465. """
  466. app_mode = AppMode.value_of(app_model.mode)
  467. export_data = {
  468. "version": CURRENT_DSL_VERSION,
  469. "kind": "app",
  470. "app": {
  471. "name": app_model.name,
  472. "mode": app_model.mode,
  473. "icon": "🤖" if app_model.icon_type == "image" else app_model.icon,
  474. "icon_background": "#FFEAD5" if app_model.icon_type == "image" else app_model.icon_background,
  475. "description": app_model.description,
  476. "use_icon_as_answer_icon": app_model.use_icon_as_answer_icon,
  477. },
  478. }
  479. if app_mode in {AppMode.ADVANCED_CHAT, AppMode.WORKFLOW}:
  480. cls._append_workflow_export_data(
  481. export_data=export_data, app_model=app_model, include_secret=include_secret
  482. )
  483. else:
  484. cls._append_model_config_export_data(export_data, app_model)
  485. return yaml.dump(export_data, allow_unicode=True) # type: ignore
  486. @classmethod
  487. def _append_workflow_export_data(cls, *, export_data: dict, app_model: App, include_secret: bool) -> None:
  488. """
  489. Append workflow export data
  490. :param export_data: export data
  491. :param app_model: App instance
  492. """
  493. workflow_service = WorkflowService()
  494. workflow = workflow_service.get_draft_workflow(app_model)
  495. if not workflow:
  496. raise ValueError("Missing draft workflow configuration, please check.")
  497. workflow_dict = workflow.to_dict(include_secret=include_secret)
  498. for node in workflow_dict.get("graph", {}).get("nodes", []):
  499. if node.get("data", {}).get("type", "") == NodeType.KNOWLEDGE_RETRIEVAL.value:
  500. dataset_ids = node["data"].get("dataset_ids", [])
  501. node["data"]["dataset_ids"] = [
  502. cls.encrypt_dataset_id(dataset_id=dataset_id, tenant_id=app_model.tenant_id)
  503. for dataset_id in dataset_ids
  504. ]
  505. export_data["workflow"] = workflow_dict
  506. dependencies = cls._extract_dependencies_from_workflow(workflow)
  507. export_data["dependencies"] = [
  508. jsonable_encoder(d.model_dump())
  509. for d in DependenciesAnalysisService.generate_dependencies(
  510. tenant_id=app_model.tenant_id, dependencies=dependencies
  511. )
  512. ]
  513. @classmethod
  514. def _append_model_config_export_data(cls, export_data: dict, app_model: App) -> None:
  515. """
  516. Append model config export data
  517. :param export_data: export data
  518. :param app_model: App instance
  519. """
  520. app_model_config = app_model.app_model_config
  521. if not app_model_config:
  522. raise ValueError("Missing app configuration, please check.")
  523. export_data["model_config"] = app_model_config.to_dict()
  524. dependencies = cls._extract_dependencies_from_model_config(app_model_config.to_dict())
  525. export_data["dependencies"] = [
  526. jsonable_encoder(d.model_dump())
  527. for d in DependenciesAnalysisService.generate_dependencies(
  528. tenant_id=app_model.tenant_id, dependencies=dependencies
  529. )
  530. ]
  531. @classmethod
  532. def _extract_dependencies_from_workflow(cls, workflow: Workflow) -> list[str]:
  533. """
  534. Extract dependencies from workflow
  535. :param workflow: Workflow instance
  536. :return: dependencies list format like ["langgenius/google"]
  537. """
  538. graph = workflow.graph_dict
  539. dependencies = cls._extract_dependencies_from_workflow_graph(graph)
  540. return dependencies
  541. @classmethod
  542. def _extract_dependencies_from_workflow_graph(cls, graph: Mapping) -> list[str]:
  543. """
  544. Extract dependencies from workflow graph
  545. :param graph: Workflow graph
  546. :return: dependencies list format like ["langgenius/google"]
  547. """
  548. dependencies = []
  549. for node in graph.get("nodes", []):
  550. try:
  551. typ = node.get("data", {}).get("type")
  552. match typ:
  553. case NodeType.TOOL.value:
  554. tool_entity = ToolNodeData(**node["data"])
  555. dependencies.append(
  556. DependenciesAnalysisService.analyze_tool_dependency(tool_entity.provider_id),
  557. )
  558. case NodeType.LLM.value:
  559. llm_entity = LLMNodeData(**node["data"])
  560. dependencies.append(
  561. DependenciesAnalysisService.analyze_model_provider_dependency(llm_entity.model.provider),
  562. )
  563. case NodeType.QUESTION_CLASSIFIER.value:
  564. question_classifier_entity = QuestionClassifierNodeData(**node["data"])
  565. dependencies.append(
  566. DependenciesAnalysisService.analyze_model_provider_dependency(
  567. question_classifier_entity.model.provider
  568. ),
  569. )
  570. case NodeType.PARAMETER_EXTRACTOR.value:
  571. parameter_extractor_entity = ParameterExtractorNodeData(**node["data"])
  572. dependencies.append(
  573. DependenciesAnalysisService.analyze_model_provider_dependency(
  574. parameter_extractor_entity.model.provider
  575. ),
  576. )
  577. case NodeType.KNOWLEDGE_RETRIEVAL.value:
  578. knowledge_retrieval_entity = KnowledgeRetrievalNodeData(**node["data"])
  579. if knowledge_retrieval_entity.retrieval_mode == "multiple":
  580. if knowledge_retrieval_entity.multiple_retrieval_config:
  581. if (
  582. knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
  583. == "reranking_model"
  584. ):
  585. if knowledge_retrieval_entity.multiple_retrieval_config.reranking_model:
  586. dependencies.append(
  587. DependenciesAnalysisService.analyze_model_provider_dependency(
  588. knowledge_retrieval_entity.multiple_retrieval_config.reranking_model.provider
  589. ),
  590. )
  591. elif (
  592. knowledge_retrieval_entity.multiple_retrieval_config.reranking_mode
  593. == "weighted_score"
  594. ):
  595. if knowledge_retrieval_entity.multiple_retrieval_config.weights:
  596. vector_setting = (
  597. knowledge_retrieval_entity.multiple_retrieval_config.weights.vector_setting
  598. )
  599. dependencies.append(
  600. DependenciesAnalysisService.analyze_model_provider_dependency(
  601. vector_setting.embedding_provider_name
  602. ),
  603. )
  604. elif knowledge_retrieval_entity.retrieval_mode == "single":
  605. model_config = knowledge_retrieval_entity.single_retrieval_config
  606. if model_config:
  607. dependencies.append(
  608. DependenciesAnalysisService.analyze_model_provider_dependency(
  609. model_config.model.provider
  610. ),
  611. )
  612. case _:
  613. # TODO: Handle default case or unknown node types
  614. pass
  615. except Exception as e:
  616. logger.exception("Error extracting node dependency", exc_info=e)
  617. return dependencies
  618. @classmethod
  619. def _extract_dependencies_from_model_config(cls, model_config: Mapping) -> list[str]:
  620. """
  621. Extract dependencies from model config
  622. :param model_config: model config dict
  623. :return: dependencies list format like ["langgenius/google"]
  624. """
  625. dependencies = []
  626. try:
  627. # completion model
  628. model_dict = model_config.get("model", {})
  629. if model_dict:
  630. dependencies.append(
  631. DependenciesAnalysisService.analyze_model_provider_dependency(model_dict.get("provider", ""))
  632. )
  633. # reranking model
  634. dataset_configs = model_config.get("dataset_configs", {})
  635. if dataset_configs:
  636. for dataset_config in dataset_configs.get("datasets", {}).get("datasets", []):
  637. if dataset_config.get("reranking_model"):
  638. dependencies.append(
  639. DependenciesAnalysisService.analyze_model_provider_dependency(
  640. dataset_config.get("reranking_model", {})
  641. .get("reranking_provider_name", {})
  642. .get("provider")
  643. )
  644. )
  645. # tools
  646. agent_configs = model_config.get("agent_mode", {})
  647. if agent_configs:
  648. for agent_config in agent_configs.get("tools", []):
  649. dependencies.append(
  650. DependenciesAnalysisService.analyze_tool_dependency(agent_config.get("provider_id"))
  651. )
  652. except Exception as e:
  653. logger.exception("Error extracting model config dependency", exc_info=e)
  654. return dependencies
  655. @classmethod
  656. def get_leaked_dependencies(cls, tenant_id: str, dsl_dependencies: list[dict]) -> list[PluginDependency]:
  657. """
  658. Returns the leaked dependencies in current workspace
  659. """
  660. dependencies = [PluginDependency(**dep) for dep in dsl_dependencies]
  661. if not dependencies:
  662. return []
  663. return DependenciesAnalysisService.get_leaked_dependencies(tenant_id=tenant_id, dependencies=dependencies)
  664. @staticmethod
  665. def _generate_aes_key(tenant_id: str) -> bytes:
  666. """Generate AES key based on tenant_id"""
  667. return hashlib.sha256(tenant_id.encode()).digest()
  668. @classmethod
  669. def encrypt_dataset_id(cls, dataset_id: str, tenant_id: str) -> str:
  670. """Encrypt dataset_id using AES-CBC mode"""
  671. key = cls._generate_aes_key(tenant_id)
  672. iv = key[:16]
  673. cipher = AES.new(key, AES.MODE_CBC, iv)
  674. ct_bytes = cipher.encrypt(pad(dataset_id.encode(), AES.block_size))
  675. return base64.b64encode(ct_bytes).decode()
  676. @classmethod
  677. def decrypt_dataset_id(cls, encrypted_data: str, tenant_id: str) -> str | None:
  678. """AES decryption"""
  679. try:
  680. key = cls._generate_aes_key(tenant_id)
  681. iv = key[:16]
  682. cipher = AES.new(key, AES.MODE_CBC, iv)
  683. pt = unpad(cipher.decrypt(base64.b64decode(encrypted_data)), AES.block_size)
  684. return pt.decode()
  685. except Exception:
  686. return None