Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.

workflow_draft_variable_service.py 30KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. import dataclasses
  2. import datetime
  3. import logging
  4. from collections.abc import Mapping, Sequence
  5. from enum import StrEnum
  6. from typing import Any, ClassVar
  7. from sqlalchemy import Engine, orm, select
  8. from sqlalchemy.dialects.postgresql import insert
  9. from sqlalchemy.orm import Session
  10. from sqlalchemy.sql.expression import and_, or_
  11. from core.app.entities.app_invoke_entities import InvokeFrom
  12. from core.file.models import File
  13. from core.variables import Segment, StringSegment, Variable
  14. from core.variables.consts import MIN_SELECTORS_LENGTH
  15. from core.variables.segments import ArrayFileSegment, FileSegment
  16. from core.variables.types import SegmentType
  17. from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
  18. from core.workflow.enums import SystemVariableKey
  19. from core.workflow.nodes import NodeType
  20. from core.workflow.nodes.variable_assigner.common.helpers import get_updated_variables
  21. from core.workflow.variable_loader import VariableLoader
  22. from factories.file_factory import StorageKeyLoader
  23. from factories.variable_factory import build_segment, segment_to_variable
  24. from models import App, Conversation
  25. from models.enums import DraftVariableType
  26. from models.workflow import Workflow, WorkflowDraftVariable, WorkflowNodeExecutionModel, is_system_variable_editable
  27. _logger = logging.getLogger(__name__)
  28. @dataclasses.dataclass(frozen=True)
  29. class WorkflowDraftVariableList:
  30. variables: list[WorkflowDraftVariable]
  31. total: int | None = None
  32. class WorkflowDraftVariableError(Exception):
  33. pass
  34. class VariableResetError(WorkflowDraftVariableError):
  35. pass
  36. class UpdateNotSupportedError(WorkflowDraftVariableError):
  37. pass
  38. class DraftVarLoader(VariableLoader):
  39. # This implements the VariableLoader interface for loading draft variables.
  40. #
  41. # ref: core.workflow.variable_loader.VariableLoader
  42. # Database engine used for loading variables.
  43. _engine: Engine
  44. # Application ID for which variables are being loaded.
  45. _app_id: str
  46. _tenant_id: str
  47. _fallback_variables: Sequence[Variable]
  48. def __init__(
  49. self,
  50. engine: Engine,
  51. app_id: str,
  52. tenant_id: str,
  53. fallback_variables: Sequence[Variable] | None = None,
  54. ) -> None:
  55. self._engine = engine
  56. self._app_id = app_id
  57. self._tenant_id = tenant_id
  58. self._fallback_variables = fallback_variables or []
  59. def _selector_to_tuple(self, selector: Sequence[str]) -> tuple[str, str]:
  60. return (selector[0], selector[1])
  61. def load_variables(self, selectors: list[list[str]]) -> list[Variable]:
  62. if not selectors:
  63. return []
  64. # Map each selector (as a tuple via `_selector_to_tuple`) to its corresponding Variable instance.
  65. variable_by_selector: dict[tuple[str, str], Variable] = {}
  66. with Session(bind=self._engine, expire_on_commit=False) as session:
  67. srv = WorkflowDraftVariableService(session)
  68. draft_vars = srv.get_draft_variables_by_selectors(self._app_id, selectors)
  69. for draft_var in draft_vars:
  70. segment = draft_var.get_value()
  71. variable = segment_to_variable(
  72. segment=segment,
  73. selector=draft_var.get_selector(),
  74. id=draft_var.id,
  75. name=draft_var.name,
  76. description=draft_var.description,
  77. )
  78. selector_tuple = self._selector_to_tuple(variable.selector)
  79. variable_by_selector[selector_tuple] = variable
  80. # Important:
  81. files: list[File] = []
  82. for draft_var in draft_vars:
  83. value = draft_var.get_value()
  84. if isinstance(value, FileSegment):
  85. files.append(value.value)
  86. elif isinstance(value, ArrayFileSegment):
  87. files.extend(value.value)
  88. with Session(bind=self._engine) as session:
  89. storage_key_loader = StorageKeyLoader(session, tenant_id=self._tenant_id)
  90. storage_key_loader.load_storage_keys(files)
  91. return list(variable_by_selector.values())
  92. class WorkflowDraftVariableService:
  93. _session: Session
  94. def __init__(self, session: Session) -> None:
  95. self._session = session
  96. def get_variable(self, variable_id: str) -> WorkflowDraftVariable | None:
  97. return self._session.query(WorkflowDraftVariable).filter(WorkflowDraftVariable.id == variable_id).first()
  98. def get_draft_variables_by_selectors(
  99. self,
  100. app_id: str,
  101. selectors: Sequence[list[str]],
  102. ) -> list[WorkflowDraftVariable]:
  103. ors = []
  104. for selector in selectors:
  105. assert len(selector) >= MIN_SELECTORS_LENGTH, f"Invalid selector to get: {selector}"
  106. node_id, name = selector[:2]
  107. ors.append(and_(WorkflowDraftVariable.node_id == node_id, WorkflowDraftVariable.name == name))
  108. # NOTE(QuantumGhost): Although the number of `or` expressions may be large, as long as
  109. # each expression includes conditions on both `node_id` and `name` (which are covered by the unique index),
  110. # PostgreSQL can efficiently retrieve the results using a bitmap index scan.
  111. #
  112. # Alternatively, a `SELECT` statement could be constructed for each selector and
  113. # combined using `UNION` to fetch all rows.
  114. # Benchmarking indicates that both approaches yield comparable performance.
  115. variables = (
  116. self._session.query(WorkflowDraftVariable).where(WorkflowDraftVariable.app_id == app_id, or_(*ors)).all()
  117. )
  118. return variables
  119. def list_variables_without_values(self, app_id: str, page: int, limit: int) -> WorkflowDraftVariableList:
  120. criteria = WorkflowDraftVariable.app_id == app_id
  121. total = None
  122. query = self._session.query(WorkflowDraftVariable).filter(criteria)
  123. if page == 1:
  124. total = query.count()
  125. variables = (
  126. # Do not load the `value` field.
  127. query.options(orm.defer(WorkflowDraftVariable.value))
  128. .order_by(WorkflowDraftVariable.created_at.desc())
  129. .limit(limit)
  130. .offset((page - 1) * limit)
  131. .all()
  132. )
  133. return WorkflowDraftVariableList(variables=variables, total=total)
  134. def _list_node_variables(self, app_id: str, node_id: str) -> WorkflowDraftVariableList:
  135. criteria = (
  136. WorkflowDraftVariable.app_id == app_id,
  137. WorkflowDraftVariable.node_id == node_id,
  138. )
  139. query = self._session.query(WorkflowDraftVariable).filter(*criteria)
  140. variables = query.order_by(WorkflowDraftVariable.created_at.desc()).all()
  141. return WorkflowDraftVariableList(variables=variables)
  142. def list_node_variables(self, app_id: str, node_id: str) -> WorkflowDraftVariableList:
  143. return self._list_node_variables(app_id, node_id)
  144. def list_conversation_variables(self, app_id: str) -> WorkflowDraftVariableList:
  145. return self._list_node_variables(app_id, CONVERSATION_VARIABLE_NODE_ID)
  146. def list_system_variables(self, app_id: str) -> WorkflowDraftVariableList:
  147. return self._list_node_variables(app_id, SYSTEM_VARIABLE_NODE_ID)
  148. def get_conversation_variable(self, app_id: str, name: str) -> WorkflowDraftVariable | None:
  149. return self._get_variable(app_id=app_id, node_id=CONVERSATION_VARIABLE_NODE_ID, name=name)
  150. def get_system_variable(self, app_id: str, name: str) -> WorkflowDraftVariable | None:
  151. return self._get_variable(app_id=app_id, node_id=SYSTEM_VARIABLE_NODE_ID, name=name)
  152. def get_node_variable(self, app_id: str, node_id: str, name: str) -> WorkflowDraftVariable | None:
  153. return self._get_variable(app_id, node_id, name)
  154. def _get_variable(self, app_id: str, node_id: str, name: str) -> WorkflowDraftVariable | None:
  155. variable = (
  156. self._session.query(WorkflowDraftVariable)
  157. .where(
  158. WorkflowDraftVariable.app_id == app_id,
  159. WorkflowDraftVariable.node_id == node_id,
  160. WorkflowDraftVariable.name == name,
  161. )
  162. .first()
  163. )
  164. return variable
  165. def update_variable(
  166. self,
  167. variable: WorkflowDraftVariable,
  168. name: str | None = None,
  169. value: Segment | None = None,
  170. ) -> WorkflowDraftVariable:
  171. if not variable.editable:
  172. raise UpdateNotSupportedError(f"variable not support updating, id={variable.id}")
  173. if name is not None:
  174. variable.set_name(name)
  175. if value is not None:
  176. variable.set_value(value)
  177. variable.last_edited_at = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
  178. self._session.flush()
  179. return variable
  180. def _reset_conv_var(self, workflow: Workflow, variable: WorkflowDraftVariable) -> WorkflowDraftVariable | None:
  181. conv_var_by_name = {i.name: i for i in workflow.conversation_variables}
  182. conv_var = conv_var_by_name.get(variable.name)
  183. if conv_var is None:
  184. self._session.delete(instance=variable)
  185. self._session.flush()
  186. _logger.warning(
  187. "Conversation variable not found for draft variable, id=%s, name=%s", variable.id, variable.name
  188. )
  189. return None
  190. variable.set_value(conv_var)
  191. variable.last_edited_at = None
  192. self._session.add(variable)
  193. self._session.flush()
  194. return variable
  195. def _reset_node_var_or_sys_var(
  196. self, workflow: Workflow, variable: WorkflowDraftVariable
  197. ) -> WorkflowDraftVariable | None:
  198. # If a variable does not allow updating, it makes no sence to resetting it.
  199. if not variable.editable:
  200. return variable
  201. # No execution record for this variable, delete the variable instead.
  202. if variable.node_execution_id is None:
  203. self._session.delete(instance=variable)
  204. self._session.flush()
  205. _logger.warning("draft variable has no node_execution_id, id=%s, name=%s", variable.id, variable.name)
  206. return None
  207. query = select(WorkflowNodeExecutionModel).where(WorkflowNodeExecutionModel.id == variable.node_execution_id)
  208. node_exec = self._session.scalars(query).first()
  209. if node_exec is None:
  210. _logger.warning(
  211. "Node exectution not found for draft variable, id=%s, name=%s, node_execution_id=%s",
  212. variable.id,
  213. variable.name,
  214. variable.node_execution_id,
  215. )
  216. self._session.delete(instance=variable)
  217. self._session.flush()
  218. return None
  219. outputs_dict = node_exec.outputs_dict or {}
  220. # a sentinel value used to check the absent of the output variable key.
  221. absent = object()
  222. if variable.get_variable_type() == DraftVariableType.NODE:
  223. # Get node type for proper value extraction
  224. node_config = workflow.get_node_config_by_id(variable.node_id)
  225. node_type = workflow.get_node_type_from_node_config(node_config)
  226. # Note: Based on the implementation in `_build_from_variable_assigner_mapping`,
  227. # VariableAssignerNode (both v1 and v2) can only create conversation draft variables.
  228. # For consistency, we should simply return when processing VARIABLE_ASSIGNER nodes.
  229. #
  230. # This implementation must remain synchronized with the `_build_from_variable_assigner_mapping`
  231. # and `save` methods.
  232. if node_type == NodeType.VARIABLE_ASSIGNER:
  233. return variable
  234. output_value = outputs_dict.get(variable.name, absent)
  235. else:
  236. output_value = outputs_dict.get(f"sys.{variable.name}", absent)
  237. # We cannot use `is None` to check the existence of an output variable here as
  238. # the value of the output may be `None`.
  239. if output_value is absent:
  240. # If variable not found in execution data, delete the variable
  241. self._session.delete(instance=variable)
  242. self._session.flush()
  243. return None
  244. value_seg = WorkflowDraftVariable.build_segment_with_type(variable.value_type, output_value)
  245. # Extract variable value using unified logic
  246. variable.set_value(value_seg)
  247. variable.last_edited_at = None # Reset to indicate this is a reset operation
  248. self._session.flush()
  249. return variable
  250. def reset_variable(self, workflow: Workflow, variable: WorkflowDraftVariable) -> WorkflowDraftVariable | None:
  251. variable_type = variable.get_variable_type()
  252. if variable_type == DraftVariableType.CONVERSATION:
  253. return self._reset_conv_var(workflow, variable)
  254. else:
  255. return self._reset_node_var_or_sys_var(workflow, variable)
  256. def delete_variable(self, variable: WorkflowDraftVariable):
  257. self._session.delete(variable)
  258. def delete_workflow_variables(self, app_id: str):
  259. (
  260. self._session.query(WorkflowDraftVariable)
  261. .filter(WorkflowDraftVariable.app_id == app_id)
  262. .delete(synchronize_session=False)
  263. )
  264. def delete_node_variables(self, app_id: str, node_id: str):
  265. return self._delete_node_variables(app_id, node_id)
  266. def _delete_node_variables(self, app_id: str, node_id: str):
  267. self._session.query(WorkflowDraftVariable).where(
  268. WorkflowDraftVariable.app_id == app_id,
  269. WorkflowDraftVariable.node_id == node_id,
  270. ).delete()
  271. def _get_conversation_id_from_draft_variable(self, app_id: str) -> str | None:
  272. draft_var = self._get_variable(
  273. app_id=app_id,
  274. node_id=SYSTEM_VARIABLE_NODE_ID,
  275. name=str(SystemVariableKey.CONVERSATION_ID),
  276. )
  277. if draft_var is None:
  278. return None
  279. segment = draft_var.get_value()
  280. if not isinstance(segment, StringSegment):
  281. _logger.warning(
  282. "sys.conversation_id variable is not a string: app_id=%s, id=%s",
  283. app_id,
  284. draft_var.id,
  285. )
  286. return None
  287. return segment.value
  288. def get_or_create_conversation(
  289. self,
  290. account_id: str,
  291. app: App,
  292. workflow: Workflow,
  293. ) -> str:
  294. """
  295. get_or_create_conversation creates and returns the ID of a conversation for debugging.
  296. If a conversation already exists, as determined by the following criteria, its ID is returned:
  297. - The system variable `sys.conversation_id` exists in the draft variable table, and
  298. - A corresponding conversation record is found in the database.
  299. If no such conversation exists, a new conversation is created and its ID is returned.
  300. """
  301. conv_id = self._get_conversation_id_from_draft_variable(workflow.app_id)
  302. if conv_id is not None:
  303. conversation = (
  304. self._session.query(Conversation)
  305. .filter(
  306. Conversation.id == conv_id,
  307. Conversation.app_id == workflow.app_id,
  308. )
  309. .first()
  310. )
  311. # Only return the conversation ID if it exists and is valid (has a correspond conversation record in DB).
  312. if conversation is not None:
  313. return conv_id
  314. conversation = Conversation(
  315. app_id=workflow.app_id,
  316. app_model_config_id=app.app_model_config_id,
  317. model_provider=None,
  318. model_id="",
  319. override_model_configs=None,
  320. mode=app.mode,
  321. name="Draft Debugging Conversation",
  322. inputs={},
  323. introduction="",
  324. system_instruction="",
  325. system_instruction_tokens=0,
  326. status="normal",
  327. invoke_from=InvokeFrom.DEBUGGER.value,
  328. from_source="console",
  329. from_end_user_id=None,
  330. from_account_id=account_id,
  331. )
  332. self._session.add(conversation)
  333. self._session.flush()
  334. return conversation.id
  335. def prefill_conversation_variable_default_values(self, workflow: Workflow):
  336. """"""
  337. draft_conv_vars: list[WorkflowDraftVariable] = []
  338. for conv_var in workflow.conversation_variables:
  339. draft_var = WorkflowDraftVariable.new_conversation_variable(
  340. app_id=workflow.app_id,
  341. name=conv_var.name,
  342. value=conv_var,
  343. description=conv_var.description,
  344. )
  345. draft_conv_vars.append(draft_var)
  346. _batch_upsert_draft_varaible(
  347. self._session,
  348. draft_conv_vars,
  349. policy=_UpsertPolicy.IGNORE,
  350. )
  351. class _UpsertPolicy(StrEnum):
  352. IGNORE = "ignore"
  353. OVERWRITE = "overwrite"
  354. def _batch_upsert_draft_varaible(
  355. session: Session,
  356. draft_vars: Sequence[WorkflowDraftVariable],
  357. policy: _UpsertPolicy = _UpsertPolicy.OVERWRITE,
  358. ) -> None:
  359. if not draft_vars:
  360. return None
  361. # Although we could use SQLAlchemy ORM operations here, we choose not to for several reasons:
  362. #
  363. # 1. The variable saving process involves writing multiple rows to the
  364. # `workflow_draft_variables` table. Batch insertion significantly improves performance.
  365. # 2. Using the ORM would require either:
  366. #
  367. # a. Checking for the existence of each variable before insertion,
  368. # resulting in 2n SQL statements for n variables and potential concurrency issues.
  369. # b. Attempting insertion first, then updating if a unique index violation occurs,
  370. # which still results in n to 2n SQL statements.
  371. #
  372. # Both approaches are inefficient and suboptimal.
  373. # 3. We do not need to retrieve the results of the SQL execution or populate ORM
  374. # model instances with the returned values.
  375. # 4. Batch insertion with `ON CONFLICT DO UPDATE` allows us to insert or update all
  376. # variables in a single SQL statement, avoiding the issues above.
  377. #
  378. # For these reasons, we use the SQLAlchemy query builder and rely on dialect-specific
  379. # insert operations instead of the ORM layer.
  380. stmt = insert(WorkflowDraftVariable).values([_model_to_insertion_dict(v) for v in draft_vars])
  381. if policy == _UpsertPolicy.OVERWRITE:
  382. stmt = stmt.on_conflict_do_update(
  383. index_elements=WorkflowDraftVariable.unique_app_id_node_id_name(),
  384. set_={
  385. # Refresh creation timestamp to ensure updated variables
  386. # appear first in chronologically sorted result sets.
  387. "created_at": stmt.excluded.created_at,
  388. "updated_at": stmt.excluded.updated_at,
  389. "last_edited_at": stmt.excluded.last_edited_at,
  390. "description": stmt.excluded.description,
  391. "value_type": stmt.excluded.value_type,
  392. "value": stmt.excluded.value,
  393. "visible": stmt.excluded.visible,
  394. "editable": stmt.excluded.editable,
  395. "node_execution_id": stmt.excluded.node_execution_id,
  396. },
  397. )
  398. elif _UpsertPolicy.IGNORE:
  399. stmt = stmt.on_conflict_do_nothing(index_elements=WorkflowDraftVariable.unique_app_id_node_id_name())
  400. else:
  401. raise Exception("Invalid value for update policy.")
  402. session.execute(stmt)
  403. def _model_to_insertion_dict(model: WorkflowDraftVariable) -> dict[str, Any]:
  404. d: dict[str, Any] = {
  405. "app_id": model.app_id,
  406. "last_edited_at": None,
  407. "node_id": model.node_id,
  408. "name": model.name,
  409. "selector": model.selector,
  410. "value_type": model.value_type,
  411. "value": model.value,
  412. "node_execution_id": model.node_execution_id,
  413. }
  414. if model.visible is not None:
  415. d["visible"] = model.visible
  416. if model.editable is not None:
  417. d["editable"] = model.editable
  418. if model.created_at is not None:
  419. d["created_at"] = model.created_at
  420. if model.updated_at is not None:
  421. d["updated_at"] = model.updated_at
  422. if model.description is not None:
  423. d["description"] = model.description
  424. return d
  425. def _build_segment_for_serialized_values(v: Any) -> Segment:
  426. """
  427. Reconstructs Segment objects from serialized values, with special handling
  428. for FileSegment and ArrayFileSegment types.
  429. This function should only be used when:
  430. 1. No explicit type information is available
  431. 2. The input value is in serialized form (dict or list)
  432. It detects potential file objects in the serialized data and properly rebuilds the
  433. appropriate segment type.
  434. """
  435. return build_segment(WorkflowDraftVariable.rebuild_file_types(v))
  436. class DraftVariableSaver:
  437. # _DUMMY_OUTPUT_IDENTITY is a placeholder output for workflow nodes.
  438. # Its sole possible value is `None`.
  439. #
  440. # This is used to signal the execution of a workflow node when it has no other outputs.
  441. _DUMMY_OUTPUT_IDENTITY: ClassVar[str] = "__dummy__"
  442. _DUMMY_OUTPUT_VALUE: ClassVar[None] = None
  443. # _EXCLUDE_VARIABLE_NAMES_MAPPING maps node types and versions to variable names that
  444. # should be excluded when saving draft variables. This prevents certain internal or
  445. # technical variables from being exposed in the draft environment, particularly those
  446. # that aren't meant to be directly edited or viewed by users.
  447. _EXCLUDE_VARIABLE_NAMES_MAPPING: dict[NodeType, frozenset[str]] = {
  448. NodeType.LLM: frozenset(["finish_reason"]),
  449. NodeType.LOOP: frozenset(["loop_round"]),
  450. }
  451. # Database session used for persisting draft variables.
  452. _session: Session
  453. # The application ID associated with the draft variables.
  454. # This should match the `Workflow.app_id` of the workflow to which the current node belongs.
  455. _app_id: str
  456. # The ID of the node for which DraftVariableSaver is saving output variables.
  457. _node_id: str
  458. # The type of the current node (see NodeType).
  459. _node_type: NodeType
  460. #
  461. _node_execution_id: str
  462. # _enclosing_node_id identifies the container node that the current node belongs to.
  463. # For example, if the current node is an LLM node inside an Iteration node
  464. # or Loop node, then `_enclosing_node_id` refers to the ID of
  465. # the containing Iteration or Loop node.
  466. #
  467. # If the current node is not nested within another node, `_enclosing_node_id` is
  468. # `None`.
  469. _enclosing_node_id: str | None
  470. def __init__(
  471. self,
  472. session: Session,
  473. app_id: str,
  474. node_id: str,
  475. node_type: NodeType,
  476. node_execution_id: str,
  477. enclosing_node_id: str | None = None,
  478. ):
  479. # Important: `node_execution_id` parameter refers to the primary key (`id`) of the
  480. # WorkflowNodeExecutionModel/WorkflowNodeExecution, not their `node_execution_id`
  481. # field. These are distinct database fields with different purposes.
  482. self._session = session
  483. self._app_id = app_id
  484. self._node_id = node_id
  485. self._node_type = node_type
  486. self._node_execution_id = node_execution_id
  487. self._enclosing_node_id = enclosing_node_id
  488. def _create_dummy_output_variable(self):
  489. return WorkflowDraftVariable.new_node_variable(
  490. app_id=self._app_id,
  491. node_id=self._node_id,
  492. name=self._DUMMY_OUTPUT_IDENTITY,
  493. node_execution_id=self._node_execution_id,
  494. value=build_segment(self._DUMMY_OUTPUT_VALUE),
  495. visible=False,
  496. editable=False,
  497. )
  498. def _should_save_output_variables_for_draft(self) -> bool:
  499. if self._enclosing_node_id is not None and self._node_type != NodeType.VARIABLE_ASSIGNER:
  500. # Currently we do not save output variables for nodes inside loop or iteration.
  501. return False
  502. return True
  503. def _build_from_variable_assigner_mapping(self, process_data: Mapping[str, Any]) -> list[WorkflowDraftVariable]:
  504. draft_vars: list[WorkflowDraftVariable] = []
  505. updated_variables = get_updated_variables(process_data) or []
  506. for item in updated_variables:
  507. selector = item.selector
  508. if len(selector) < MIN_SELECTORS_LENGTH:
  509. raise Exception("selector too short")
  510. # NOTE(QuantumGhost): only the following two kinds of variable could be updated by
  511. # VariableAssigner: ConversationVariable and iteration variable.
  512. # We only save conversation variable here.
  513. if selector[0] != CONVERSATION_VARIABLE_NODE_ID:
  514. continue
  515. segment = WorkflowDraftVariable.build_segment_with_type(segment_type=item.value_type, value=item.new_value)
  516. draft_vars.append(
  517. WorkflowDraftVariable.new_conversation_variable(
  518. app_id=self._app_id,
  519. name=item.name,
  520. value=segment,
  521. )
  522. )
  523. # Add a dummy output variable to indicate that this node is executed.
  524. draft_vars.append(self._create_dummy_output_variable())
  525. return draft_vars
  526. def _build_variables_from_start_mapping(self, output: Mapping[str, Any]) -> list[WorkflowDraftVariable]:
  527. draft_vars = []
  528. has_non_sys_variables = False
  529. for name, value in output.items():
  530. value_seg = _build_segment_for_serialized_values(value)
  531. node_id, name = self._normalize_variable_for_start_node(name)
  532. # If node_id is not `sys`, it means that the variable is a user-defined input field
  533. # in `Start` node.
  534. if node_id != SYSTEM_VARIABLE_NODE_ID:
  535. draft_vars.append(
  536. WorkflowDraftVariable.new_node_variable(
  537. app_id=self._app_id,
  538. node_id=self._node_id,
  539. name=name,
  540. node_execution_id=self._node_execution_id,
  541. value=value_seg,
  542. visible=True,
  543. editable=True,
  544. )
  545. )
  546. has_non_sys_variables = True
  547. else:
  548. if name == SystemVariableKey.FILES:
  549. # Here we know the type of variable must be `array[file]`, we
  550. # just build files from the value.
  551. files = [File.model_validate(v) for v in value]
  552. if files:
  553. value_seg = WorkflowDraftVariable.build_segment_with_type(SegmentType.ARRAY_FILE, files)
  554. else:
  555. value_seg = ArrayFileSegment(value=[])
  556. draft_vars.append(
  557. WorkflowDraftVariable.new_sys_variable(
  558. app_id=self._app_id,
  559. name=name,
  560. node_execution_id=self._node_execution_id,
  561. value=value_seg,
  562. editable=self._should_variable_be_editable(node_id, name),
  563. )
  564. )
  565. if not has_non_sys_variables:
  566. draft_vars.append(self._create_dummy_output_variable())
  567. return draft_vars
  568. def _normalize_variable_for_start_node(self, name: str) -> tuple[str, str]:
  569. if not name.startswith(f"{SYSTEM_VARIABLE_NODE_ID}."):
  570. return self._node_id, name
  571. _, name_ = name.split(".", maxsplit=1)
  572. return SYSTEM_VARIABLE_NODE_ID, name_
  573. def _build_variables_from_mapping(self, output: Mapping[str, Any]) -> list[WorkflowDraftVariable]:
  574. draft_vars = []
  575. for name, value in output.items():
  576. if not self._should_variable_be_saved(name):
  577. _logger.debug(
  578. "Skip saving variable as it has been excluded by its node_type, name=%s, node_type=%s",
  579. name,
  580. self._node_type,
  581. )
  582. continue
  583. if isinstance(value, Segment):
  584. value_seg = value
  585. else:
  586. value_seg = _build_segment_for_serialized_values(value)
  587. draft_vars.append(
  588. WorkflowDraftVariable.new_node_variable(
  589. app_id=self._app_id,
  590. node_id=self._node_id,
  591. name=name,
  592. node_execution_id=self._node_execution_id,
  593. value=value_seg,
  594. visible=self._should_variable_be_visible(self._node_id, self._node_type, name),
  595. )
  596. )
  597. return draft_vars
  598. def save(
  599. self,
  600. process_data: Mapping[str, Any] | None = None,
  601. outputs: Mapping[str, Any] | None = None,
  602. ):
  603. draft_vars: list[WorkflowDraftVariable] = []
  604. if outputs is None:
  605. outputs = {}
  606. if process_data is None:
  607. process_data = {}
  608. if not self._should_save_output_variables_for_draft():
  609. return
  610. if self._node_type == NodeType.VARIABLE_ASSIGNER:
  611. draft_vars = self._build_from_variable_assigner_mapping(process_data=process_data)
  612. elif self._node_type == NodeType.START:
  613. draft_vars = self._build_variables_from_start_mapping(outputs)
  614. else:
  615. draft_vars = self._build_variables_from_mapping(outputs)
  616. _batch_upsert_draft_varaible(self._session, draft_vars)
  617. @staticmethod
  618. def _should_variable_be_editable(node_id: str, name: str) -> bool:
  619. if node_id in (CONVERSATION_VARIABLE_NODE_ID, ENVIRONMENT_VARIABLE_NODE_ID):
  620. return False
  621. if node_id == SYSTEM_VARIABLE_NODE_ID and not is_system_variable_editable(name):
  622. return False
  623. return True
  624. @staticmethod
  625. def _should_variable_be_visible(node_id: str, node_type: NodeType, name: str) -> bool:
  626. if node_type in NodeType.IF_ELSE:
  627. return False
  628. if node_id == SYSTEM_VARIABLE_NODE_ID and not is_system_variable_editable(name):
  629. return False
  630. return True
  631. def _should_variable_be_saved(self, name: str) -> bool:
  632. exclude_var_names = self._EXCLUDE_VARIABLE_NAMES_MAPPING.get(self._node_type)
  633. if exclude_var_names is None:
  634. return True
  635. return name not in exclude_var_names