You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

workflow.py 57KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541
  1. import json
  2. import logging
  3. from collections.abc import Mapping, Sequence
  4. from datetime import datetime
  5. from enum import Enum, StrEnum
  6. from typing import TYPE_CHECKING, Any, Optional, Union
  7. from uuid import uuid4
  8. import sqlalchemy as sa
  9. from sqlalchemy import DateTime, Select, exists, orm, select
  10. from core.file.constants import maybe_file_object
  11. from core.file.models import File
  12. from core.variables import utils as variable_utils
  13. from core.variables.variables import FloatVariable, IntegerVariable, StringVariable
  14. from core.workflow.constants import CONVERSATION_VARIABLE_NODE_ID, SYSTEM_VARIABLE_NODE_ID
  15. from core.workflow.nodes.enums import NodeType
  16. from extensions.ext_storage import Storage
  17. from factories.variable_factory import TypeMismatchError, build_segment_with_type
  18. from libs.datetime_utils import naive_utc_now
  19. from libs.uuid_utils import uuidv7
  20. from ._workflow_exc import NodeNotFoundError, WorkflowDataError
  21. if TYPE_CHECKING:
  22. from models.model import AppMode, UploadFile
  23. from sqlalchemy import Index, PrimaryKeyConstraint, String, UniqueConstraint, func
  24. from sqlalchemy.orm import Mapped, declared_attr, mapped_column
  25. from constants import DEFAULT_FILE_NUMBER_LIMITS, HIDDEN_VALUE
  26. from core.helper import encrypter
  27. from core.variables import SecretVariable, Segment, SegmentType, Variable
  28. from factories import variable_factory
  29. from libs import helper
  30. from .account import Account
  31. from .base import Base
  32. from .engine import db
  33. from .enums import CreatorUserRole, DraftVariableType, ExecutionOffLoadType
  34. from .types import EnumText, StringUUID
  35. logger = logging.getLogger(__name__)
  36. class WorkflowType(Enum):
  37. """
  38. Workflow Type Enum
  39. """
  40. WORKFLOW = "workflow"
  41. CHAT = "chat"
  42. @classmethod
  43. def value_of(cls, value: str) -> "WorkflowType":
  44. """
  45. Get value of given mode.
  46. :param value: mode value
  47. :return: mode
  48. """
  49. for mode in cls:
  50. if mode.value == value:
  51. return mode
  52. raise ValueError(f"invalid workflow type value {value}")
  53. @classmethod
  54. def from_app_mode(cls, app_mode: Union[str, "AppMode"]) -> "WorkflowType":
  55. """
  56. Get workflow type from app mode.
  57. :param app_mode: app mode
  58. :return: workflow type
  59. """
  60. from models.model import AppMode
  61. app_mode = app_mode if isinstance(app_mode, AppMode) else AppMode.value_of(app_mode)
  62. return cls.WORKFLOW if app_mode == AppMode.WORKFLOW else cls.CHAT
  63. class _InvalidGraphDefinitionError(Exception):
  64. pass
  65. class Workflow(Base):
  66. """
  67. Workflow, for `Workflow App` and `Chat App workflow mode`.
  68. Attributes:
  69. - id (uuid) Workflow ID, pk
  70. - tenant_id (uuid) Workspace ID
  71. - app_id (uuid) App ID
  72. - type (string) Workflow type
  73. `workflow` for `Workflow App`
  74. `chat` for `Chat App workflow mode`
  75. - version (string) Version
  76. `draft` for draft version (only one for each app), other for version number (redundant)
  77. - graph (text) Workflow canvas configuration (JSON)
  78. The entire canvas configuration JSON, including Node, Edge, and other configurations
  79. - nodes (array[object]) Node list, see Node Schema
  80. - edges (array[object]) Edge list, see Edge Schema
  81. - created_by (uuid) Creator ID
  82. - created_at (timestamp) Creation time
  83. - updated_by (uuid) `optional` Last updater ID
  84. - updated_at (timestamp) `optional` Last update time
  85. """
  86. __tablename__ = "workflows"
  87. __table_args__ = (
  88. sa.PrimaryKeyConstraint("id", name="workflow_pkey"),
  89. sa.Index("workflow_version_idx", "tenant_id", "app_id", "version"),
  90. )
  91. id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
  92. tenant_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
  93. app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
  94. type: Mapped[str] = mapped_column(String(255), nullable=False)
  95. version: Mapped[str] = mapped_column(String(255), nullable=False)
  96. marked_name: Mapped[str] = mapped_column(default="", server_default="")
  97. marked_comment: Mapped[str] = mapped_column(default="", server_default="")
  98. graph: Mapped[str] = mapped_column(sa.Text)
  99. _features: Mapped[str] = mapped_column("features", sa.TEXT)
  100. created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
  101. created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
  102. updated_by: Mapped[Optional[str]] = mapped_column(StringUUID)
  103. updated_at: Mapped[datetime] = mapped_column(
  104. DateTime,
  105. nullable=False,
  106. default=naive_utc_now(),
  107. server_onupdate=func.current_timestamp(),
  108. )
  109. _environment_variables: Mapped[str] = mapped_column(
  110. "environment_variables", sa.Text, nullable=False, server_default="{}"
  111. )
  112. _conversation_variables: Mapped[str] = mapped_column(
  113. "conversation_variables", sa.Text, nullable=False, server_default="{}"
  114. )
  115. VERSION_DRAFT = "draft"
  116. @classmethod
  117. def new(
  118. cls,
  119. *,
  120. tenant_id: str,
  121. app_id: str,
  122. type: str,
  123. version: str,
  124. graph: str,
  125. features: str,
  126. created_by: str,
  127. environment_variables: Sequence[Variable],
  128. conversation_variables: Sequence[Variable],
  129. marked_name: str = "",
  130. marked_comment: str = "",
  131. ) -> "Workflow":
  132. workflow = Workflow()
  133. workflow.id = str(uuid4())
  134. workflow.tenant_id = tenant_id
  135. workflow.app_id = app_id
  136. workflow.type = type
  137. workflow.version = version
  138. workflow.graph = graph
  139. workflow.features = features
  140. workflow.created_by = created_by
  141. workflow.environment_variables = environment_variables or []
  142. workflow.conversation_variables = conversation_variables or []
  143. workflow.marked_name = marked_name
  144. workflow.marked_comment = marked_comment
  145. workflow.created_at = naive_utc_now()
  146. workflow.updated_at = workflow.created_at
  147. return workflow
  148. @property
  149. def created_by_account(self):
  150. return db.session.get(Account, self.created_by)
  151. @property
  152. def updated_by_account(self):
  153. return db.session.get(Account, self.updated_by) if self.updated_by else None
  154. @property
  155. def graph_dict(self) -> Mapping[str, Any]:
  156. # TODO(QuantumGhost): Consider caching `graph_dict` to avoid repeated JSON decoding.
  157. #
  158. # Using `functools.cached_property` could help, but some code in the codebase may
  159. # modify the returned dict, which can cause issues elsewhere.
  160. #
  161. # For example, changing this property to a cached property led to errors like the
  162. # following when single stepping an `Iteration` node:
  163. #
  164. # Root node id 1748401971780start not found in the graph
  165. #
  166. # There is currently no standard way to make a dict deeply immutable in Python,
  167. # and tracking modifications to the returned dict is difficult. For now, we leave
  168. # the code as-is to avoid these issues.
  169. #
  170. # Currently, the following functions / methods would mutate the returned dict:
  171. #
  172. # - `_get_graph_and_variable_pool_of_single_iteration`.
  173. # - `_get_graph_and_variable_pool_of_single_loop`.
  174. return json.loads(self.graph) if self.graph else {}
  175. def get_node_config_by_id(self, node_id: str) -> Mapping[str, Any]:
  176. """Extract a node configuration from the workflow graph by node ID.
  177. A node configuration is a dictionary containing the node's properties, including
  178. the node's id, title, and its data as a dict.
  179. """
  180. workflow_graph = self.graph_dict
  181. if not workflow_graph:
  182. raise WorkflowDataError(f"workflow graph not found, workflow_id={self.id}")
  183. nodes = workflow_graph.get("nodes")
  184. if not nodes:
  185. raise WorkflowDataError("nodes not found in workflow graph")
  186. try:
  187. node_config = next(filter(lambda node: node["id"] == node_id, nodes))
  188. except StopIteration:
  189. raise NodeNotFoundError(node_id)
  190. assert isinstance(node_config, dict)
  191. return node_config
  192. @staticmethod
  193. def get_node_type_from_node_config(node_config: Mapping[str, Any]) -> NodeType:
  194. """Extract type of a node from the node configuration returned by `get_node_config_by_id`."""
  195. node_config_data = node_config.get("data", {})
  196. # Get node class
  197. node_type = NodeType(node_config_data.get("type"))
  198. return node_type
  199. @staticmethod
  200. def get_enclosing_node_type_and_id(node_config: Mapping[str, Any]) -> tuple[NodeType, str] | None:
  201. in_loop = node_config.get("isInLoop", False)
  202. in_iteration = node_config.get("isInIteration", False)
  203. if in_loop:
  204. loop_id = node_config.get("loop_id")
  205. if loop_id is None:
  206. raise _InvalidGraphDefinitionError("invalid graph")
  207. return NodeType.LOOP, loop_id
  208. elif in_iteration:
  209. iteration_id = node_config.get("iteration_id")
  210. if iteration_id is None:
  211. raise _InvalidGraphDefinitionError("invalid graph")
  212. return NodeType.ITERATION, iteration_id
  213. else:
  214. return None
  215. @property
  216. def features(self) -> str:
  217. """
  218. Convert old features structure to new features structure.
  219. """
  220. if not self._features:
  221. return self._features
  222. features = json.loads(self._features)
  223. if features.get("file_upload", {}).get("image", {}).get("enabled", False):
  224. image_enabled = True
  225. image_number_limits = int(features["file_upload"]["image"].get("number_limits", DEFAULT_FILE_NUMBER_LIMITS))
  226. image_transfer_methods = features["file_upload"]["image"].get(
  227. "transfer_methods", ["remote_url", "local_file"]
  228. )
  229. features["file_upload"]["enabled"] = image_enabled
  230. features["file_upload"]["number_limits"] = image_number_limits
  231. features["file_upload"]["allowed_file_upload_methods"] = image_transfer_methods
  232. features["file_upload"]["allowed_file_types"] = features["file_upload"].get("allowed_file_types", ["image"])
  233. features["file_upload"]["allowed_file_extensions"] = features["file_upload"].get(
  234. "allowed_file_extensions", []
  235. )
  236. del features["file_upload"]["image"]
  237. self._features = json.dumps(features)
  238. return self._features
  239. @features.setter
  240. def features(self, value: str) -> None:
  241. self._features = value
  242. @property
  243. def features_dict(self) -> dict[str, Any]:
  244. return json.loads(self.features) if self.features else {}
  245. def user_input_form(self, to_old_structure: bool = False) -> list:
  246. # get start node from graph
  247. if not self.graph:
  248. return []
  249. graph_dict = self.graph_dict
  250. if "nodes" not in graph_dict:
  251. return []
  252. start_node = next((node for node in graph_dict["nodes"] if node["data"]["type"] == "start"), None)
  253. if not start_node:
  254. return []
  255. # get user_input_form from start node
  256. variables: list[Any] = start_node.get("data", {}).get("variables", [])
  257. if to_old_structure:
  258. old_structure_variables = []
  259. for variable in variables:
  260. old_structure_variables.append({variable["type"]: variable})
  261. return old_structure_variables
  262. return variables
  263. @property
  264. def unique_hash(self) -> str:
  265. """
  266. Get hash of workflow.
  267. :return: hash
  268. """
  269. entity = {"graph": self.graph_dict, "features": self.features_dict}
  270. return helper.generate_text_hash(json.dumps(entity, sort_keys=True))
  271. @property
  272. def tool_published(self) -> bool:
  273. """
  274. DEPRECATED: This property is not accurate for determining if a workflow is published as a tool.
  275. It only checks if there's a WorkflowToolProvider for the app, not if this specific workflow version
  276. is the one being used by the tool.
  277. For accurate checking, use a direct query with tenant_id, app_id, and version.
  278. """
  279. from models.tools import WorkflowToolProvider
  280. stmt = select(
  281. exists().where(
  282. WorkflowToolProvider.tenant_id == self.tenant_id,
  283. WorkflowToolProvider.app_id == self.app_id,
  284. )
  285. )
  286. return db.session.execute(stmt).scalar_one()
  287. @property
  288. def environment_variables(self) -> Sequence[StringVariable | IntegerVariable | FloatVariable | SecretVariable]:
  289. # TODO: find some way to init `self._environment_variables` when instance created.
  290. if self._environment_variables is None:
  291. self._environment_variables = "{}"
  292. # Use workflow.tenant_id to avoid relying on request user in background threads
  293. tenant_id = self.tenant_id
  294. if not tenant_id:
  295. return []
  296. environment_variables_dict: dict[str, Any] = json.loads(self._environment_variables)
  297. results = [
  298. variable_factory.build_environment_variable_from_mapping(v) for v in environment_variables_dict.values()
  299. ]
  300. # decrypt secret variables value
  301. def decrypt_func(var):
  302. if isinstance(var, SecretVariable):
  303. return var.model_copy(update={"value": encrypter.decrypt_token(tenant_id=tenant_id, token=var.value)})
  304. elif isinstance(var, (StringVariable, IntegerVariable, FloatVariable)):
  305. return var
  306. else:
  307. raise AssertionError("this statement should be unreachable.")
  308. decrypted_results: list[SecretVariable | StringVariable | IntegerVariable | FloatVariable] = list(
  309. map(decrypt_func, results)
  310. )
  311. return decrypted_results
  312. @environment_variables.setter
  313. def environment_variables(self, value: Sequence[Variable]):
  314. if not value:
  315. self._environment_variables = "{}"
  316. return
  317. # Use workflow.tenant_id to avoid relying on request user in background threads
  318. tenant_id = self.tenant_id
  319. if not tenant_id:
  320. self._environment_variables = "{}"
  321. return
  322. value = list(value)
  323. if any(var for var in value if not var.id):
  324. raise ValueError("environment variable require a unique id")
  325. # Compare inputs and origin variables,
  326. # if the value is HIDDEN_VALUE, use the origin variable value (only update `name`).
  327. origin_variables_dictionary = {var.id: var for var in self.environment_variables}
  328. for i, variable in enumerate(value):
  329. if variable.id in origin_variables_dictionary and variable.value == HIDDEN_VALUE:
  330. value[i] = origin_variables_dictionary[variable.id].model_copy(update={"name": variable.name})
  331. # encrypt secret variables value
  332. def encrypt_func(var):
  333. if isinstance(var, SecretVariable):
  334. return var.model_copy(update={"value": encrypter.encrypt_token(tenant_id=tenant_id, token=var.value)})
  335. else:
  336. return var
  337. encrypted_vars = list(map(encrypt_func, value))
  338. environment_variables_json = json.dumps(
  339. {var.name: var.model_dump() for var in encrypted_vars},
  340. ensure_ascii=False,
  341. )
  342. self._environment_variables = environment_variables_json
  343. def to_dict(self, *, include_secret: bool = False) -> Mapping[str, Any]:
  344. environment_variables = list(self.environment_variables)
  345. environment_variables = [
  346. v if not isinstance(v, SecretVariable) or include_secret else v.model_copy(update={"value": ""})
  347. for v in environment_variables
  348. ]
  349. result = {
  350. "graph": self.graph_dict,
  351. "features": self.features_dict,
  352. "environment_variables": [var.model_dump(mode="json") for var in environment_variables],
  353. "conversation_variables": [var.model_dump(mode="json") for var in self.conversation_variables],
  354. }
  355. return result
  356. @property
  357. def conversation_variables(self) -> Sequence[Variable]:
  358. # TODO: find some way to init `self._conversation_variables` when instance created.
  359. if self._conversation_variables is None:
  360. self._conversation_variables = "{}"
  361. variables_dict: dict[str, Any] = json.loads(self._conversation_variables)
  362. results = [variable_factory.build_conversation_variable_from_mapping(v) for v in variables_dict.values()]
  363. return results
  364. @conversation_variables.setter
  365. def conversation_variables(self, value: Sequence[Variable]) -> None:
  366. self._conversation_variables = json.dumps(
  367. {var.name: var.model_dump() for var in value},
  368. ensure_ascii=False,
  369. )
  370. @staticmethod
  371. def version_from_datetime(d: datetime) -> str:
  372. return str(d)
  373. class WorkflowRun(Base):
  374. """
  375. Workflow Run
  376. Attributes:
  377. - id (uuid) Run ID
  378. - tenant_id (uuid) Workspace ID
  379. - app_id (uuid) App ID
  380. - workflow_id (uuid) Workflow ID
  381. - type (string) Workflow type
  382. - triggered_from (string) Trigger source
  383. `debugging` for canvas debugging
  384. `app-run` for (published) app execution
  385. - version (string) Version
  386. - graph (text) Workflow canvas configuration (JSON)
  387. - inputs (text) Input parameters
  388. - status (string) Execution status, `running` / `succeeded` / `failed` / `stopped`
  389. - outputs (text) `optional` Output content
  390. - error (string) `optional` Error reason
  391. - elapsed_time (float) `optional` Time consumption (s)
  392. - total_tokens (int) `optional` Total tokens used
  393. - total_steps (int) Total steps (redundant), default 0
  394. - created_by_role (string) Creator role
  395. - `account` Console account
  396. - `end_user` End user
  397. - created_by (uuid) Runner ID
  398. - created_at (timestamp) Run time
  399. - finished_at (timestamp) End time
  400. """
  401. __tablename__ = "workflow_runs"
  402. __table_args__ = (
  403. sa.PrimaryKeyConstraint("id", name="workflow_run_pkey"),
  404. sa.Index("workflow_run_triggerd_from_idx", "tenant_id", "app_id", "triggered_from"),
  405. )
  406. id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
  407. tenant_id: Mapped[str] = mapped_column(StringUUID)
  408. app_id: Mapped[str] = mapped_column(StringUUID)
  409. workflow_id: Mapped[str] = mapped_column(StringUUID)
  410. type: Mapped[str] = mapped_column(String(255))
  411. triggered_from: Mapped[str] = mapped_column(String(255))
  412. version: Mapped[str] = mapped_column(String(255))
  413. graph: Mapped[Optional[str]] = mapped_column(sa.Text)
  414. inputs: Mapped[Optional[str]] = mapped_column(sa.Text)
  415. status: Mapped[str] = mapped_column(String(255)) # running, succeeded, failed, stopped, partial-succeeded
  416. outputs: Mapped[Optional[str]] = mapped_column(sa.Text, default="{}")
  417. error: Mapped[Optional[str]] = mapped_column(sa.Text)
  418. elapsed_time: Mapped[float] = mapped_column(sa.Float, nullable=False, server_default=sa.text("0"))
  419. total_tokens: Mapped[int] = mapped_column(sa.BigInteger, server_default=sa.text("0"))
  420. total_steps: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
  421. created_by_role: Mapped[str] = mapped_column(String(255)) # account, end_user
  422. created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
  423. created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
  424. finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
  425. exceptions_count: Mapped[int] = mapped_column(sa.Integer, server_default=sa.text("0"), nullable=True)
  426. @property
  427. def created_by_account(self):
  428. created_by_role = CreatorUserRole(self.created_by_role)
  429. return db.session.get(Account, self.created_by) if created_by_role == CreatorUserRole.ACCOUNT else None
  430. @property
  431. def created_by_end_user(self):
  432. from models.model import EndUser
  433. created_by_role = CreatorUserRole(self.created_by_role)
  434. return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
  435. @property
  436. def graph_dict(self) -> Mapping[str, Any]:
  437. return json.loads(self.graph) if self.graph else {}
  438. @property
  439. def inputs_dict(self) -> Mapping[str, Any]:
  440. return json.loads(self.inputs) if self.inputs else {}
  441. @property
  442. def outputs_dict(self) -> Mapping[str, Any]:
  443. return json.loads(self.outputs) if self.outputs else {}
  444. @property
  445. def message(self):
  446. from models.model import Message
  447. return (
  448. db.session.query(Message).where(Message.app_id == self.app_id, Message.workflow_run_id == self.id).first()
  449. )
  450. @property
  451. def workflow(self):
  452. return db.session.query(Workflow).where(Workflow.id == self.workflow_id).first()
  453. def to_dict(self):
  454. return {
  455. "id": self.id,
  456. "tenant_id": self.tenant_id,
  457. "app_id": self.app_id,
  458. "workflow_id": self.workflow_id,
  459. "type": self.type,
  460. "triggered_from": self.triggered_from,
  461. "version": self.version,
  462. "graph": self.graph_dict,
  463. "inputs": self.inputs_dict,
  464. "status": self.status,
  465. "outputs": self.outputs_dict,
  466. "error": self.error,
  467. "elapsed_time": self.elapsed_time,
  468. "total_tokens": self.total_tokens,
  469. "total_steps": self.total_steps,
  470. "created_by_role": self.created_by_role,
  471. "created_by": self.created_by,
  472. "created_at": self.created_at,
  473. "finished_at": self.finished_at,
  474. "exceptions_count": self.exceptions_count,
  475. }
  476. @classmethod
  477. def from_dict(cls, data: dict) -> "WorkflowRun":
  478. return cls(
  479. id=data.get("id"),
  480. tenant_id=data.get("tenant_id"),
  481. app_id=data.get("app_id"),
  482. workflow_id=data.get("workflow_id"),
  483. type=data.get("type"),
  484. triggered_from=data.get("triggered_from"),
  485. version=data.get("version"),
  486. graph=json.dumps(data.get("graph")),
  487. inputs=json.dumps(data.get("inputs")),
  488. status=data.get("status"),
  489. outputs=json.dumps(data.get("outputs")),
  490. error=data.get("error"),
  491. elapsed_time=data.get("elapsed_time"),
  492. total_tokens=data.get("total_tokens"),
  493. total_steps=data.get("total_steps"),
  494. created_by_role=data.get("created_by_role"),
  495. created_by=data.get("created_by"),
  496. created_at=data.get("created_at"),
  497. finished_at=data.get("finished_at"),
  498. exceptions_count=data.get("exceptions_count"),
  499. )
  500. class WorkflowNodeExecutionTriggeredFrom(StrEnum):
  501. """
  502. Workflow Node Execution Triggered From Enum
  503. """
  504. SINGLE_STEP = "single-step"
  505. WORKFLOW_RUN = "workflow-run"
  506. class WorkflowNodeExecutionModel(Base): # This model is expected to have `offload_data` preloaded in most cases.
  507. """
  508. Workflow Node Execution
  509. - id (uuid) Execution ID
  510. - tenant_id (uuid) Workspace ID
  511. - app_id (uuid) App ID
  512. - workflow_id (uuid) Workflow ID
  513. - triggered_from (string) Trigger source
  514. `single-step` for single-step debugging
  515. `workflow-run` for workflow execution (debugging / user execution)
  516. - workflow_run_id (uuid) `optional` Workflow run ID
  517. Null for single-step debugging.
  518. - index (int) Execution sequence number, used for displaying Tracing Node order
  519. - predecessor_node_id (string) `optional` Predecessor node ID, used for displaying execution path
  520. - node_id (string) Node ID
  521. - node_type (string) Node type, such as `start`
  522. - title (string) Node title
  523. - inputs (json) All predecessor node variable content used in the node
  524. - process_data (json) Node process data
  525. - outputs (json) `optional` Node output variables
  526. - status (string) Execution status, `running` / `succeeded` / `failed`
  527. - error (string) `optional` Error reason
  528. - elapsed_time (float) `optional` Time consumption (s)
  529. - execution_metadata (text) Metadata
  530. - total_tokens (int) `optional` Total tokens used
  531. - total_price (decimal) `optional` Total cost
  532. - currency (string) `optional` Currency, such as USD / RMB
  533. - created_at (timestamp) Run time
  534. - created_by_role (string) Creator role
  535. - `account` Console account
  536. - `end_user` End user
  537. - created_by (uuid) Runner ID
  538. - finished_at (timestamp) End time
  539. """
  540. __tablename__ = "workflow_node_executions"
  541. @declared_attr
  542. def __table_args__(cls): # noqa
  543. return (
  544. PrimaryKeyConstraint("id", name="workflow_node_execution_pkey"),
  545. Index(
  546. "workflow_node_execution_workflow_run_idx",
  547. "tenant_id",
  548. "app_id",
  549. "workflow_id",
  550. "triggered_from",
  551. "workflow_run_id",
  552. ),
  553. Index(
  554. "workflow_node_execution_node_run_idx",
  555. "tenant_id",
  556. "app_id",
  557. "workflow_id",
  558. "triggered_from",
  559. "node_id",
  560. ),
  561. Index(
  562. "workflow_node_execution_id_idx",
  563. "tenant_id",
  564. "app_id",
  565. "workflow_id",
  566. "triggered_from",
  567. "node_execution_id",
  568. ),
  569. Index(
  570. # The first argument is the index name,
  571. # which we leave as `None`` to allow auto-generation by the ORM.
  572. None,
  573. cls.tenant_id,
  574. cls.workflow_id,
  575. cls.node_id,
  576. # MyPy may flag the following line because it doesn't recognize that
  577. # the `declared_attr` decorator passes the receiving class as the first
  578. # argument to this method, allowing us to reference class attributes.
  579. cls.created_at.desc(), # type: ignore
  580. ),
  581. )
  582. id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
  583. tenant_id: Mapped[str] = mapped_column(StringUUID)
  584. app_id: Mapped[str] = mapped_column(StringUUID)
  585. workflow_id: Mapped[str] = mapped_column(StringUUID)
  586. triggered_from: Mapped[str] = mapped_column(String(255))
  587. workflow_run_id: Mapped[Optional[str]] = mapped_column(StringUUID)
  588. index: Mapped[int] = mapped_column(sa.Integer)
  589. predecessor_node_id: Mapped[Optional[str]] = mapped_column(String(255))
  590. node_execution_id: Mapped[Optional[str]] = mapped_column(String(255))
  591. node_id: Mapped[str] = mapped_column(String(255))
  592. node_type: Mapped[str] = mapped_column(String(255))
  593. title: Mapped[str] = mapped_column(String(255))
  594. inputs: Mapped[Optional[str]] = mapped_column(sa.Text)
  595. process_data: Mapped[Optional[str]] = mapped_column(sa.Text)
  596. outputs: Mapped[Optional[str]] = mapped_column(sa.Text)
  597. status: Mapped[str] = mapped_column(String(255))
  598. error: Mapped[Optional[str]] = mapped_column(sa.Text)
  599. elapsed_time: Mapped[float] = mapped_column(sa.Float, server_default=sa.text("0"))
  600. execution_metadata: Mapped[Optional[str]] = mapped_column(sa.Text)
  601. created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.current_timestamp())
  602. created_by_role: Mapped[str] = mapped_column(String(255))
  603. created_by: Mapped[str] = mapped_column(StringUUID)
  604. finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime)
  605. offload_data: Mapped[list["WorkflowNodeExecutionOffload"]] = orm.relationship(
  606. "WorkflowNodeExecutionOffload",
  607. primaryjoin="WorkflowNodeExecutionModel.id == foreign(WorkflowNodeExecutionOffload.node_execution_id)",
  608. uselist=True,
  609. lazy="raise",
  610. back_populates="execution",
  611. )
  612. @staticmethod
  613. def preload_offload_data(
  614. query: Select[tuple["WorkflowNodeExecutionModel"]] | orm.Query["WorkflowNodeExecutionModel"],
  615. ):
  616. return query.options(orm.selectinload(WorkflowNodeExecutionModel.offload_data))
  617. @staticmethod
  618. def preload_offload_data_and_files(
  619. query: Select[tuple["WorkflowNodeExecutionModel"]] | orm.Query["WorkflowNodeExecutionModel"],
  620. ):
  621. return query.options(
  622. orm.selectinload(WorkflowNodeExecutionModel.offload_data).options(
  623. # Using `joinedload` instead of `selectinload` to minimize database roundtrips,
  624. # as `selectinload` would require separate queries for `inputs_file` and `outputs_file`.
  625. orm.selectinload(WorkflowNodeExecutionOffload.file),
  626. )
  627. )
  628. @property
  629. def created_by_account(self):
  630. created_by_role = CreatorUserRole(self.created_by_role)
  631. # TODO(-LAN-): Avoid using db.session.get() here.
  632. return db.session.get(Account, self.created_by) if created_by_role == CreatorUserRole.ACCOUNT else None
  633. @property
  634. def created_by_end_user(self):
  635. from models.model import EndUser
  636. created_by_role = CreatorUserRole(self.created_by_role)
  637. # TODO(-LAN-): Avoid using db.session.get() here.
  638. return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
  639. @property
  640. def inputs_dict(self):
  641. return json.loads(self.inputs) if self.inputs else None
  642. @property
  643. def outputs_dict(self) -> dict[str, Any] | None:
  644. return json.loads(self.outputs) if self.outputs else None
  645. @property
  646. def process_data_dict(self):
  647. return json.loads(self.process_data) if self.process_data else None
  648. @property
  649. def execution_metadata_dict(self) -> dict[str, Any]:
  650. # When the metadata is unset, we return an empty dictionary instead of `None`.
  651. # This approach streamlines the logic for the caller, making it easier to handle
  652. # cases where metadata is absent.
  653. return json.loads(self.execution_metadata) if self.execution_metadata else {}
  654. @property
  655. def extras(self):
  656. from core.tools.tool_manager import ToolManager
  657. extras = {}
  658. if self.execution_metadata_dict:
  659. from core.workflow.nodes import NodeType
  660. if self.node_type == NodeType.TOOL.value and "tool_info" in self.execution_metadata_dict:
  661. tool_info = self.execution_metadata_dict["tool_info"]
  662. extras["icon"] = ToolManager.get_tool_icon(
  663. tenant_id=self.tenant_id,
  664. provider_type=tool_info["provider_type"],
  665. provider_id=tool_info["provider_id"],
  666. )
  667. return extras
  668. def _get_offload_by_type(self, type_: ExecutionOffLoadType) -> Optional["WorkflowNodeExecutionOffload"]:
  669. return next(iter([i for i in self.offload_data if i.type_ == type_]), None)
  670. @property
  671. def inputs_truncated(self) -> bool:
  672. """Check if inputs were truncated (offloaded to external storage)."""
  673. return self._get_offload_by_type(ExecutionOffLoadType.INPUTS) is not None
  674. @property
  675. def outputs_truncated(self) -> bool:
  676. """Check if outputs were truncated (offloaded to external storage)."""
  677. return self._get_offload_by_type(ExecutionOffLoadType.OUTPUTS) is not None
  678. @property
  679. def process_data_truncated(self) -> bool:
  680. """Check if process_data were truncated (offloaded to external storage)."""
  681. return self._get_offload_by_type(ExecutionOffLoadType.PROCESS_DATA) is not None
  682. @staticmethod
  683. def _load_full_content(session: orm.Session, file_id: str, storage: Storage):
  684. from .model import UploadFile
  685. stmt = sa.select(UploadFile).where(UploadFile.id == file_id)
  686. file = session.scalars(stmt).first()
  687. assert file is not None, f"UploadFile with id {file_id} should exist but not"
  688. content = storage.load(file.key)
  689. return json.loads(content)
  690. def load_full_inputs(self, session: orm.Session, storage: Storage) -> Mapping[str, Any] | None:
  691. offload = self._get_offload_by_type(ExecutionOffLoadType.INPUTS)
  692. if offload is None:
  693. return self.inputs_dict
  694. return self._load_full_content(session, offload.file_id, storage)
  695. def load_full_outputs(self, session: orm.Session, storage: Storage) -> Mapping[str, Any] | None:
  696. offload: WorkflowNodeExecutionOffload | None = self._get_offload_by_type(ExecutionOffLoadType.OUTPUTS)
  697. if offload is None:
  698. return self.outputs_dict
  699. return self._load_full_content(session, offload.file_id, storage)
  700. def load_full_process_data(self, session: orm.Session, storage: Storage) -> Mapping[str, Any] | None:
  701. offload: WorkflowNodeExecutionOffload | None = self._get_offload_by_type(ExecutionOffLoadType.PROCESS_DATA)
  702. if offload is None:
  703. return self.process_data_dict
  704. return self._load_full_content(session, offload.file_id, storage)
  705. class WorkflowNodeExecutionOffload(Base):
  706. __tablename__ = "workflow_node_execution_offload"
  707. __table_args__ = (
  708. UniqueConstraint(
  709. "node_execution_id",
  710. "type",
  711. # Treat `NULL` as distinct for this unique index, so
  712. # we can have mutitple records with `NULL` node_exeution_id, simplify garbage collection process.
  713. postgresql_nulls_not_distinct=False,
  714. ),
  715. )
  716. _HASH_COL_SIZE = 64
  717. id: Mapped[str] = mapped_column(
  718. StringUUID,
  719. primary_key=True,
  720. server_default=sa.text("uuidv7()"),
  721. )
  722. created_at: Mapped[datetime] = mapped_column(
  723. DateTime, default=naive_utc_now, server_default=func.current_timestamp()
  724. )
  725. tenant_id: Mapped[str] = mapped_column(StringUUID)
  726. app_id: Mapped[str] = mapped_column(StringUUID)
  727. # `node_execution_id` indicates the `WorkflowNodeExecutionModel` associated with this offload record.
  728. # A value of `None` signifies that this offload record is not linked to any execution record
  729. # and should be considered for garbage collection.
  730. node_execution_id: Mapped[str | None] = mapped_column(StringUUID, nullable=True)
  731. type_: Mapped[ExecutionOffLoadType] = mapped_column(EnumText(ExecutionOffLoadType), name="type", nullable=False)
  732. # Design Decision: Combining inputs and outputs into a single object was considered to reduce I/O
  733. # operations. However, due to the current design of `WorkflowNodeExecutionRepository`,
  734. # the `save` method is called at two distinct times:
  735. #
  736. # - When the node starts execution: the `inputs` field exists, but the `outputs` field is absent
  737. # - When the node completes execution (either succeeded or failed): the `outputs` field becomes available
  738. #
  739. # It's difficult to correlate these two successive calls to `save` for combined storage.
  740. # Converting the `WorkflowNodeExecutionRepository` to buffer the first `save` call and flush
  741. # when execution completes was also considered, but this would make the execution state unobservable
  742. # until completion, significantly damaging the observability of workflow execution.
  743. #
  744. # Given these constraints, `inputs` and `outputs` are stored separately to maintain real-time
  745. # observability and system reliability.
  746. # `file_id` references to the offloaded storage object containing the data.
  747. file_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
  748. execution: Mapped[WorkflowNodeExecutionModel] = orm.relationship(
  749. foreign_keys=[node_execution_id],
  750. lazy="raise",
  751. uselist=False,
  752. primaryjoin="WorkflowNodeExecutionOffload.node_execution_id == WorkflowNodeExecutionModel.id",
  753. back_populates="offload_data",
  754. )
  755. file: Mapped[Optional["UploadFile"]] = orm.relationship(
  756. foreign_keys=[file_id],
  757. lazy="raise",
  758. uselist=False,
  759. primaryjoin="WorkflowNodeExecutionOffload.file_id == UploadFile.id",
  760. )
  761. class WorkflowAppLogCreatedFrom(Enum):
  762. """
  763. Workflow App Log Created From Enum
  764. """
  765. SERVICE_API = "service-api"
  766. WEB_APP = "web-app"
  767. INSTALLED_APP = "installed-app"
  768. @classmethod
  769. def value_of(cls, value: str) -> "WorkflowAppLogCreatedFrom":
  770. """
  771. Get value of given mode.
  772. :param value: mode value
  773. :return: mode
  774. """
  775. for mode in cls:
  776. if mode.value == value:
  777. return mode
  778. raise ValueError(f"invalid workflow app log created from value {value}")
  779. class WorkflowAppLog(Base):
  780. """
  781. Workflow App execution log, excluding workflow debugging records.
  782. Attributes:
  783. - id (uuid) run ID
  784. - tenant_id (uuid) Workspace ID
  785. - app_id (uuid) App ID
  786. - workflow_id (uuid) Associated Workflow ID
  787. - workflow_run_id (uuid) Associated Workflow Run ID
  788. - created_from (string) Creation source
  789. `service-api` App Execution OpenAPI
  790. `web-app` WebApp
  791. `installed-app` Installed App
  792. - created_by_role (string) Creator role
  793. - `account` Console account
  794. - `end_user` End user
  795. - created_by (uuid) Creator ID, depends on the user table according to created_by_role
  796. - created_at (timestamp) Creation time
  797. """
  798. __tablename__ = "workflow_app_logs"
  799. __table_args__ = (
  800. sa.PrimaryKeyConstraint("id", name="workflow_app_log_pkey"),
  801. sa.Index("workflow_app_log_app_idx", "tenant_id", "app_id"),
  802. )
  803. id: Mapped[str] = mapped_column(StringUUID, server_default=sa.text("uuid_generate_v4()"))
  804. tenant_id: Mapped[str] = mapped_column(StringUUID)
  805. app_id: Mapped[str] = mapped_column(StringUUID)
  806. workflow_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
  807. workflow_run_id: Mapped[str] = mapped_column(StringUUID)
  808. created_from: Mapped[str] = mapped_column(String(255), nullable=False)
  809. created_by_role: Mapped[str] = mapped_column(String(255), nullable=False)
  810. created_by: Mapped[str] = mapped_column(StringUUID, nullable=False)
  811. created_at: Mapped[datetime] = mapped_column(DateTime, nullable=False, server_default=func.current_timestamp())
  812. @property
  813. def workflow_run(self):
  814. return db.session.get(WorkflowRun, self.workflow_run_id)
  815. @property
  816. def created_by_account(self):
  817. created_by_role = CreatorUserRole(self.created_by_role)
  818. return db.session.get(Account, self.created_by) if created_by_role == CreatorUserRole.ACCOUNT else None
  819. @property
  820. def created_by_end_user(self):
  821. from models.model import EndUser
  822. created_by_role = CreatorUserRole(self.created_by_role)
  823. return db.session.get(EndUser, self.created_by) if created_by_role == CreatorUserRole.END_USER else None
  824. def to_dict(self):
  825. return {
  826. "id": self.id,
  827. "tenant_id": self.tenant_id,
  828. "app_id": self.app_id,
  829. "workflow_id": self.workflow_id,
  830. "workflow_run_id": self.workflow_run_id,
  831. "created_from": self.created_from,
  832. "created_by_role": self.created_by_role,
  833. "created_by": self.created_by,
  834. "created_at": self.created_at,
  835. }
  836. class ConversationVariable(Base):
  837. __tablename__ = "workflow_conversation_variables"
  838. id: Mapped[str] = mapped_column(StringUUID, primary_key=True)
  839. conversation_id: Mapped[str] = mapped_column(StringUUID, nullable=False, primary_key=True, index=True)
  840. app_id: Mapped[str] = mapped_column(StringUUID, nullable=False, index=True)
  841. data: Mapped[str] = mapped_column(sa.Text, nullable=False)
  842. created_at: Mapped[datetime] = mapped_column(
  843. DateTime, nullable=False, server_default=func.current_timestamp(), index=True
  844. )
  845. updated_at: Mapped[datetime] = mapped_column(
  846. DateTime, nullable=False, server_default=func.current_timestamp(), onupdate=func.current_timestamp()
  847. )
  848. def __init__(self, *, id: str, app_id: str, conversation_id: str, data: str) -> None:
  849. self.id = id
  850. self.app_id = app_id
  851. self.conversation_id = conversation_id
  852. self.data = data
  853. @classmethod
  854. def from_variable(cls, *, app_id: str, conversation_id: str, variable: Variable) -> "ConversationVariable":
  855. obj = cls(
  856. id=variable.id,
  857. app_id=app_id,
  858. conversation_id=conversation_id,
  859. data=variable.model_dump_json(),
  860. )
  861. return obj
  862. def to_variable(self) -> Variable:
  863. mapping = json.loads(self.data)
  864. return variable_factory.build_conversation_variable_from_mapping(mapping)
  865. # Only `sys.query` and `sys.files` could be modified.
  866. _EDITABLE_SYSTEM_VARIABLE = frozenset(["query", "files"])
  867. def _naive_utc_datetime():
  868. return naive_utc_now()
  869. class WorkflowDraftVariable(Base):
  870. """`WorkflowDraftVariable` record variables and outputs generated during
  871. debugging workflow or chatflow.
  872. IMPORTANT: This model maintains multiple invariant rules that must be preserved.
  873. Do not instantiate this class directly with the constructor.
  874. Instead, use the factory methods (`new_conversation_variable`, `new_sys_variable`,
  875. `new_node_variable`) defined below to ensure all invariants are properly maintained.
  876. """
  877. @staticmethod
  878. def unique_app_id_node_id_name() -> list[str]:
  879. return [
  880. "app_id",
  881. "node_id",
  882. "name",
  883. ]
  884. __tablename__ = "workflow_draft_variables"
  885. __table_args__ = (
  886. UniqueConstraint(*unique_app_id_node_id_name()),
  887. Index("workflow_draft_variable_file_id_idx", "file_id"),
  888. )
  889. # Required for instance variable annotation.
  890. __allow_unmapped__ = True
  891. # id is the unique identifier of a draft variable.
  892. id: Mapped[str] = mapped_column(StringUUID, primary_key=True, server_default=sa.text("uuid_generate_v4()"))
  893. created_at: Mapped[datetime] = mapped_column(
  894. DateTime,
  895. nullable=False,
  896. default=_naive_utc_datetime,
  897. server_default=func.current_timestamp(),
  898. )
  899. updated_at: Mapped[datetime] = mapped_column(
  900. DateTime,
  901. nullable=False,
  902. default=_naive_utc_datetime,
  903. server_default=func.current_timestamp(),
  904. onupdate=func.current_timestamp(),
  905. )
  906. # "`app_id` maps to the `id` field in the `model.App` model."
  907. app_id: Mapped[str] = mapped_column(StringUUID, nullable=False)
  908. # `last_edited_at` records when the value of a given draft variable
  909. # is edited.
  910. #
  911. # If it's not edited after creation, its value is `None`.
  912. last_edited_at: Mapped[datetime | None] = mapped_column(
  913. DateTime,
  914. nullable=True,
  915. default=None,
  916. )
  917. # The `node_id` field is special.
  918. #
  919. # If the variable is a conversation variable or a system variable, then the value of `node_id`
  920. # is `conversation` or `sys`, respective.
  921. #
  922. # Otherwise, if the variable is a variable belonging to a specific node, the value of `_node_id` is
  923. # the identity of correspond node in graph definition. An example of node id is `"1745769620734"`.
  924. #
  925. # However, there's one caveat. The id of the first "Answer" node in chatflow is "answer". (Other
  926. # "Answer" node conform the rules above.)
  927. node_id: Mapped[str] = mapped_column(sa.String(255), nullable=False, name="node_id")
  928. # From `VARIABLE_PATTERN`, we may conclude that the length of a top level variable is less than
  929. # 80 chars.
  930. #
  931. # ref: api/core/workflow/entities/variable_pool.py:18
  932. name: Mapped[str] = mapped_column(sa.String(255), nullable=False)
  933. description: Mapped[str] = mapped_column(
  934. sa.String(255),
  935. default="",
  936. nullable=False,
  937. )
  938. selector: Mapped[str] = mapped_column(sa.String(255), nullable=False, name="selector")
  939. # The data type of this variable's value
  940. #
  941. # If the variable is offloaded, `value_type` represents the type of the truncated value,
  942. # which may differ from the original value's type. Typically, they are the same,
  943. # but in cases where the structurally truncated value still exceeds the size limit,
  944. # text slicing is applied, and the `value_type` is converted to `STRING`.
  945. value_type: Mapped[SegmentType] = mapped_column(EnumText(SegmentType, length=20))
  946. # The variable's value serialized as a JSON string
  947. #
  948. # If the variable is offloaded, `value` contains a truncated version, not the full original value.
  949. value: Mapped[str] = mapped_column(sa.Text, nullable=False, name="value")
  950. # Controls whether the variable should be displayed in the variable inspection panel
  951. visible: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=True)
  952. # Determines whether this variable can be modified by users
  953. editable: Mapped[bool] = mapped_column(sa.Boolean, nullable=False, default=False)
  954. # The `node_execution_id` field identifies the workflow node execution that created this variable.
  955. # It corresponds to the `id` field in the `WorkflowNodeExecutionModel` model.
  956. #
  957. # This field is not `None` for system variables and node variables, and is `None`
  958. # for conversation variables.
  959. node_execution_id: Mapped[str | None] = mapped_column(
  960. StringUUID,
  961. nullable=True,
  962. default=None,
  963. )
  964. # Reference to WorkflowDraftVariableFile for offloaded large variables
  965. #
  966. # Indicates whether the current draft variable is offloaded.
  967. # If not offloaded, this field will be None.
  968. file_id: Mapped[str | None] = mapped_column(
  969. StringUUID,
  970. nullable=True,
  971. default=None,
  972. comment="Reference to WorkflowDraftVariableFile if variable is offloaded to external storage",
  973. )
  974. is_default_value: Mapped[bool] = mapped_column(
  975. sa.Boolean,
  976. nullable=False,
  977. default=False,
  978. comment=(
  979. "Indicates whether the current value is the default for a conversation variable. "
  980. "Always `FALSE` for other types of variables."
  981. ),
  982. )
  983. # Relationship to WorkflowDraftVariableFile
  984. variable_file: Mapped[Optional["WorkflowDraftVariableFile"]] = orm.relationship(
  985. foreign_keys=[file_id],
  986. lazy="raise",
  987. uselist=False,
  988. primaryjoin="WorkflowDraftVariableFile.id == WorkflowDraftVariable.file_id",
  989. )
  990. # Cache for deserialized value
  991. #
  992. # NOTE(QuantumGhost): This field serves two purposes:
  993. #
  994. # 1. Caches deserialized values to reduce repeated parsing costs
  995. # 2. Allows modification of the deserialized value after retrieval,
  996. # particularly important for `File`` variables which require database
  997. # lookups to obtain storage_key and other metadata
  998. #
  999. # Use double underscore prefix for better encapsulation,
  1000. # making this attribute harder to access from outside the class.
  1001. __value: Segment | None
  1002. def __init__(self, *args, **kwargs):
  1003. """
  1004. The constructor of `WorkflowDraftVariable` is not intended for
  1005. direct use outside this file. Its solo purpose is setup private state
  1006. used by the model instance.
  1007. Please use the factory methods
  1008. (`new_conversation_variable`, `new_sys_variable`, `new_node_variable`)
  1009. defined below to create instances of this class.
  1010. """
  1011. super().__init__(*args, **kwargs)
  1012. self.__value = None
  1013. @orm.reconstructor
  1014. def _init_on_load(self):
  1015. self.__value = None
  1016. def get_selector(self) -> list[str]:
  1017. selector = json.loads(self.selector)
  1018. if not isinstance(selector, list):
  1019. logger.error(
  1020. "invalid selector loaded from database, type=%s, value=%s",
  1021. type(selector),
  1022. self.selector,
  1023. )
  1024. raise ValueError("invalid selector.")
  1025. return selector
  1026. def _set_selector(self, value: list[str]):
  1027. self.selector = json.dumps(value)
  1028. def _loads_value(self) -> Segment:
  1029. value = json.loads(self.value)
  1030. return self.build_segment_with_type(self.value_type, value)
  1031. @staticmethod
  1032. def rebuild_file_types(value: Any) -> Any:
  1033. # NOTE(QuantumGhost): Temporary workaround for structured data handling.
  1034. # By this point, `output` has been converted to dict by
  1035. # `WorkflowEntry.handle_special_values`, so we need to
  1036. # reconstruct File objects from their serialized form
  1037. # to maintain proper variable saving behavior.
  1038. #
  1039. # Ideally, we should work with structured data objects directly
  1040. # rather than their serialized forms.
  1041. # However, multiple components in the codebase depend on
  1042. # `WorkflowEntry.handle_special_values`, making a comprehensive migration challenging.
  1043. if isinstance(value, dict):
  1044. if not maybe_file_object(value):
  1045. return value
  1046. return File.model_validate(value)
  1047. elif isinstance(value, list) and value:
  1048. first = value[0]
  1049. if not maybe_file_object(first):
  1050. return value
  1051. return [File.model_validate(i) for i in value]
  1052. else:
  1053. return value
  1054. @classmethod
  1055. def build_segment_with_type(cls, segment_type: SegmentType, value: Any) -> Segment:
  1056. # Extends `variable_factory.build_segment_with_type` functionality by
  1057. # reconstructing `FileSegment`` or `ArrayFileSegment`` objects from
  1058. # their serialized dictionary or list representations, respectively.
  1059. if segment_type == SegmentType.FILE:
  1060. if isinstance(value, File):
  1061. return build_segment_with_type(segment_type, value)
  1062. elif isinstance(value, dict):
  1063. file = cls.rebuild_file_types(value)
  1064. return build_segment_with_type(segment_type, file)
  1065. else:
  1066. raise TypeMismatchError(f"expected dict or File for FileSegment, got {type(value)}")
  1067. if segment_type == SegmentType.ARRAY_FILE:
  1068. if not isinstance(value, list):
  1069. raise TypeMismatchError(f"expected list for ArrayFileSegment, got {type(value)}")
  1070. file_list = cls.rebuild_file_types(value)
  1071. return build_segment_with_type(segment_type=segment_type, value=file_list)
  1072. return build_segment_with_type(segment_type=segment_type, value=value)
  1073. def get_value(self) -> Segment:
  1074. """Decode the serialized value into its corresponding `Segment` object.
  1075. This method caches the result, so repeated calls will return the same
  1076. object instance without re-parsing the serialized data.
  1077. If you need to modify the returned `Segment`, use `value.model_copy()`
  1078. to create a copy first to avoid affecting the cached instance.
  1079. For more information about the caching mechanism, see the documentation
  1080. of the `__value` field.
  1081. Returns:
  1082. Segment: The deserialized value as a Segment object.
  1083. """
  1084. if self.__value is not None:
  1085. return self.__value
  1086. value = self._loads_value()
  1087. self.__value = value
  1088. return value
  1089. def set_name(self, name: str):
  1090. self.name = name
  1091. self._set_selector([self.node_id, name])
  1092. def set_value(self, value: Segment):
  1093. """Updates the `value` and corresponding `value_type` fields in the database model.
  1094. This method also stores the provided Segment object in the deserialized cache
  1095. without creating a copy, allowing for efficient value access.
  1096. Args:
  1097. value: The Segment object to store as the variable's value.
  1098. """
  1099. self.__value = value
  1100. self.value = variable_utils.dumps_with_segments(value)
  1101. self.value_type = value.value_type
  1102. def get_node_id(self) -> str | None:
  1103. if self.get_variable_type() == DraftVariableType.NODE:
  1104. return self.node_id
  1105. else:
  1106. return None
  1107. def get_variable_type(self) -> DraftVariableType:
  1108. match self.node_id:
  1109. case DraftVariableType.CONVERSATION:
  1110. return DraftVariableType.CONVERSATION
  1111. case DraftVariableType.SYS:
  1112. return DraftVariableType.SYS
  1113. case _:
  1114. return DraftVariableType.NODE
  1115. def is_truncated(self) -> bool:
  1116. return self.file_id is not None
  1117. @classmethod
  1118. def _new(
  1119. cls,
  1120. *,
  1121. app_id: str,
  1122. node_id: str,
  1123. name: str,
  1124. value: Segment,
  1125. node_execution_id: str | None,
  1126. description: str = "",
  1127. file_id: str | None = None,
  1128. ) -> "WorkflowDraftVariable":
  1129. variable = WorkflowDraftVariable()
  1130. variable.created_at = _naive_utc_datetime()
  1131. variable.updated_at = _naive_utc_datetime()
  1132. variable.description = description
  1133. variable.app_id = app_id
  1134. variable.node_id = node_id
  1135. variable.name = name
  1136. variable.set_value(value)
  1137. variable.file_id = file_id
  1138. variable._set_selector(list(variable_utils.to_selector(node_id, name)))
  1139. variable.node_execution_id = node_execution_id
  1140. return variable
  1141. @classmethod
  1142. def new_conversation_variable(
  1143. cls,
  1144. *,
  1145. app_id: str,
  1146. name: str,
  1147. value: Segment,
  1148. description: str = "",
  1149. ) -> "WorkflowDraftVariable":
  1150. variable = cls._new(
  1151. app_id=app_id,
  1152. node_id=CONVERSATION_VARIABLE_NODE_ID,
  1153. name=name,
  1154. value=value,
  1155. description=description,
  1156. node_execution_id=None,
  1157. )
  1158. variable.editable = True
  1159. return variable
  1160. @classmethod
  1161. def new_sys_variable(
  1162. cls,
  1163. *,
  1164. app_id: str,
  1165. name: str,
  1166. value: Segment,
  1167. node_execution_id: str,
  1168. editable: bool = False,
  1169. ) -> "WorkflowDraftVariable":
  1170. variable = cls._new(
  1171. app_id=app_id,
  1172. node_id=SYSTEM_VARIABLE_NODE_ID,
  1173. name=name,
  1174. node_execution_id=node_execution_id,
  1175. value=value,
  1176. )
  1177. variable.editable = editable
  1178. return variable
  1179. @classmethod
  1180. def new_node_variable(
  1181. cls,
  1182. *,
  1183. app_id: str,
  1184. node_id: str,
  1185. name: str,
  1186. value: Segment,
  1187. node_execution_id: str,
  1188. visible: bool = True,
  1189. editable: bool = True,
  1190. file_id: str | None = None,
  1191. ) -> "WorkflowDraftVariable":
  1192. variable = cls._new(
  1193. app_id=app_id,
  1194. node_id=node_id,
  1195. name=name,
  1196. node_execution_id=node_execution_id,
  1197. value=value,
  1198. file_id=file_id,
  1199. )
  1200. variable.visible = visible
  1201. variable.editable = editable
  1202. return variable
  1203. @property
  1204. def edited(self):
  1205. return self.last_edited_at is not None
  1206. class WorkflowDraftVariableFile(Base):
  1207. """Stores metadata about files associated with large workflow draft variables.
  1208. This model acts as an intermediary between WorkflowDraftVariable and UploadFile,
  1209. allowing for proper cleanup of orphaned files when variables are updated or deleted.
  1210. The MIME type of the stored content is recorded in `UploadFile.mime_type`.
  1211. Possible values are 'application/json' for JSON types other than plain text,
  1212. and 'text/plain' for JSON strings.
  1213. """
  1214. __tablename__ = "workflow_draft_variable_files"
  1215. # Primary key
  1216. id: Mapped[str] = mapped_column(
  1217. StringUUID,
  1218. primary_key=True,
  1219. default=uuidv7,
  1220. server_default=sa.text("uuidv7()"),
  1221. )
  1222. created_at: Mapped[datetime] = mapped_column(
  1223. DateTime,
  1224. nullable=False,
  1225. default=_naive_utc_datetime,
  1226. server_default=func.current_timestamp(),
  1227. )
  1228. tenant_id: Mapped[str] = mapped_column(
  1229. StringUUID,
  1230. nullable=False,
  1231. comment="The tenant to which the WorkflowDraftVariableFile belongs, referencing Tenant.id",
  1232. )
  1233. app_id: Mapped[str] = mapped_column(
  1234. StringUUID,
  1235. nullable=False,
  1236. comment="The application to which the WorkflowDraftVariableFile belongs, referencing App.id",
  1237. )
  1238. user_id: Mapped[str] = mapped_column(
  1239. StringUUID,
  1240. nullable=False,
  1241. comment="The owner to of the WorkflowDraftVariableFile, referencing Account.id",
  1242. )
  1243. # Reference to the `UploadFile.id` field
  1244. upload_file_id: Mapped[str] = mapped_column(
  1245. StringUUID,
  1246. nullable=False,
  1247. comment="Reference to UploadFile containing the large variable data",
  1248. )
  1249. # -------------- metadata about the variable content --------------
  1250. # The `size` is already recorded in UploadFiles. It is duplicated here to avoid an additional database lookup.
  1251. size: Mapped[int | None] = mapped_column(
  1252. sa.BigInteger,
  1253. nullable=False,
  1254. comment="Size of the original variable content in bytes",
  1255. )
  1256. length: Mapped[Optional[int]] = mapped_column(
  1257. sa.Integer,
  1258. nullable=True,
  1259. comment=(
  1260. "Length of the original variable content. For array and array-like types, "
  1261. "this represents the number of elements. For object types, it indicates the number of keys. "
  1262. "For other types, the value is NULL."
  1263. ),
  1264. )
  1265. # The `value_type` field records the type of the original value.
  1266. value_type: Mapped[SegmentType] = mapped_column(
  1267. EnumText(SegmentType, length=20),
  1268. nullable=False,
  1269. )
  1270. # Relationship to UploadFile
  1271. upload_file: Mapped["UploadFile"] = orm.relationship(
  1272. foreign_keys=[upload_file_id],
  1273. lazy="raise",
  1274. uselist=False,
  1275. primaryjoin="WorkflowDraftVariableFile.upload_file_id == UploadFile.id",
  1276. )
  1277. def is_system_variable_editable(name: str) -> bool:
  1278. return name in _EDITABLE_SYSTEM_VARIABLE