Signed-off-by: -LAN- <laipz8200@outlook.com>tags/2.0.0-beta.1
| workflow graph execution events. | workflow graph execution events. | ||||
| """ | """ | ||||
| from .event_collector import EventCollector | |||||
| from .event_emitter import EventEmitter | |||||
| from .event_handlers import EventHandlerRegistry | |||||
| from .event_handlers import EventHandler | |||||
| from .event_manager import EventManager | |||||
| __all__ = [ | __all__ = [ | ||||
| "EventCollector", | |||||
| "EventEmitter", | |||||
| "EventHandlerRegistry", | |||||
| "EventHandler", | |||||
| "EventManager", | |||||
| ] | ] |
| """ | |||||
| Event emitter for yielding events to external consumers. | |||||
| """ | |||||
| import threading | |||||
| import time | |||||
| from collections.abc import Generator | |||||
| from typing import final | |||||
| from core.workflow.graph_events import GraphEngineEvent | |||||
| from .event_collector import EventCollector | |||||
| @final | |||||
| class EventEmitter: | |||||
| """ | |||||
| Emits collected events as a generator for external consumption. | |||||
| This provides a generator interface for yielding events as they're | |||||
| collected, with proper synchronization for multi-threaded access. | |||||
| """ | |||||
| def __init__(self, event_collector: EventCollector) -> None: | |||||
| """ | |||||
| Initialize the event emitter. | |||||
| Args: | |||||
| event_collector: The collector to emit events from | |||||
| """ | |||||
| self._event_collector = event_collector | |||||
| self._execution_complete = threading.Event() | |||||
| def mark_complete(self) -> None: | |||||
| """Mark execution as complete to stop the generator.""" | |||||
| self._execution_complete.set() | |||||
| def emit_events(self) -> Generator[GraphEngineEvent, None, None]: | |||||
| """ | |||||
| Generator that yields events as they're collected. | |||||
| Yields: | |||||
| GraphEngineEvent instances as they're processed | |||||
| """ | |||||
| yielded_count = 0 | |||||
| while not self._execution_complete.is_set() or yielded_count < self._event_collector.event_count(): | |||||
| # Get new events since last yield | |||||
| new_events = self._event_collector.get_new_events(yielded_count) | |||||
| # Yield any new events | |||||
| for event in new_events: | |||||
| yield event | |||||
| yielded_count += 1 | |||||
| # Small sleep to avoid busy waiting | |||||
| if not self._execution_complete.is_set() and not new_events: | |||||
| time.sleep(0.001) |
| from ..error_handling import ErrorHandler | from ..error_handling import ErrorHandler | ||||
| from ..graph_traversal import EdgeProcessor | from ..graph_traversal import EdgeProcessor | ||||
| from ..state_management import UnifiedStateManager | from ..state_management import UnifiedStateManager | ||||
| from .event_collector import EventCollector | |||||
| from .event_manager import EventManager | |||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| @final | @final | ||||
| class EventHandlerRegistry: | |||||
| class EventHandler: | |||||
| """ | """ | ||||
| Registry of event handlers for different event types. | Registry of event handlers for different event types. | ||||
| graph_runtime_state: GraphRuntimeState, | graph_runtime_state: GraphRuntimeState, | ||||
| graph_execution: GraphExecution, | graph_execution: GraphExecution, | ||||
| response_coordinator: ResponseStreamCoordinator, | response_coordinator: ResponseStreamCoordinator, | ||||
| event_collector: "EventCollector", | |||||
| event_collector: "EventManager", | |||||
| edge_processor: "EdgeProcessor", | edge_processor: "EdgeProcessor", | ||||
| state_manager: "UnifiedStateManager", | state_manager: "UnifiedStateManager", | ||||
| error_handler: "ErrorHandler", | error_handler: "ErrorHandler", | ||||
| graph_runtime_state: Runtime state with variable pool | graph_runtime_state: Runtime state with variable pool | ||||
| graph_execution: Graph execution aggregate | graph_execution: Graph execution aggregate | ||||
| response_coordinator: Response stream coordinator | response_coordinator: Response stream coordinator | ||||
| event_collector: Event collector for collecting events | |||||
| event_collector: Event manager for collecting events | |||||
| edge_processor: Edge processor for edge traversal | edge_processor: Edge processor for edge traversal | ||||
| state_manager: Unified state manager | state_manager: Unified state manager | ||||
| error_handler: Error handler | error_handler: Error handler |
| """ | """ | ||||
| Event collector for buffering and managing events. | |||||
| Unified event manager for collecting and emitting events. | |||||
| """ | """ | ||||
| import threading | import threading | ||||
| import time | |||||
| from collections.abc import Generator | |||||
| from typing import final | from typing import final | ||||
| from core.workflow.graph_events import GraphEngineEvent | from core.workflow.graph_events import GraphEngineEvent | ||||
| @final | @final | ||||
| class EventCollector: | |||||
| class EventManager: | |||||
| """ | """ | ||||
| Collects and buffers events for later retrieval. | |||||
| Unified event manager that collects, buffers, and emits events. | |||||
| This provides thread-safe event collection with support for | |||||
| notifying layers about events as they're collected. | |||||
| This class combines event collection with event emission, providing | |||||
| thread-safe event management with support for notifying layers and | |||||
| streaming events to external consumers. | |||||
| """ | """ | ||||
| def __init__(self) -> None: | def __init__(self) -> None: | ||||
| """Initialize the event collector.""" | |||||
| """Initialize the event manager.""" | |||||
| self._events: list[GraphEngineEvent] = [] | self._events: list[GraphEngineEvent] = [] | ||||
| self._lock = ReadWriteLock() | self._lock = ReadWriteLock() | ||||
| self._layers: list[Layer] = [] | self._layers: list[Layer] = [] | ||||
| self._execution_complete = threading.Event() | |||||
| def set_layers(self, layers: list[Layer]) -> None: | def set_layers(self, layers: list[Layer]) -> None: | ||||
| """ | """ | ||||
| self._events.append(event) | self._events.append(event) | ||||
| self._notify_layers(event) | self._notify_layers(event) | ||||
| def get_events(self) -> list[GraphEngineEvent]: | |||||
| """ | |||||
| Get all collected events. | |||||
| Returns: | |||||
| List of collected events | |||||
| """ | |||||
| with self._lock.read_lock(): | |||||
| return list(self._events) | |||||
| def get_new_events(self, start_index: int) -> list[GraphEngineEvent]: | |||||
| def _get_new_events(self, start_index: int) -> list[GraphEngineEvent]: | |||||
| """ | """ | ||||
| Get new events starting from a specific index. | Get new events starting from a specific index. | ||||
| with self._lock.read_lock(): | with self._lock.read_lock(): | ||||
| return list(self._events[start_index:]) | return list(self._events[start_index:]) | ||||
| def event_count(self) -> int: | |||||
| def _event_count(self) -> int: | |||||
| """ | """ | ||||
| Get the current count of collected events. | Get the current count of collected events. | ||||
| with self._lock.read_lock(): | with self._lock.read_lock(): | ||||
| return len(self._events) | return len(self._events) | ||||
| def clear(self) -> None: | |||||
| """Clear all collected events.""" | |||||
| with self._lock.write_lock(): | |||||
| self._events.clear() | |||||
| def mark_complete(self) -> None: | |||||
| """Mark execution as complete to stop the event emission generator.""" | |||||
| self._execution_complete.set() | |||||
| def emit_events(self) -> Generator[GraphEngineEvent, None, None]: | |||||
| """ | |||||
| Generator that yields events as they're collected. | |||||
| Yields: | |||||
| GraphEngineEvent instances as they're processed | |||||
| """ | |||||
| yielded_count = 0 | |||||
| while not self._execution_complete.is_set() or yielded_count < self._event_count(): | |||||
| # Get new events since last yield | |||||
| new_events = self._get_new_events(yielded_count) | |||||
| # Yield any new events | |||||
| for event in new_events: | |||||
| yield event | |||||
| yielded_count += 1 | |||||
| # Small sleep to avoid busy waiting | |||||
| if not self._execution_complete.is_set() and not new_events: | |||||
| time.sleep(0.001) | |||||
| def _notify_layers(self, event: GraphEngineEvent) -> None: | def _notify_layers(self, event: GraphEngineEvent) -> None: | ||||
| """ | """ |
| from .domain import ExecutionContext, GraphExecution | from .domain import ExecutionContext, GraphExecution | ||||
| from .entities.commands import AbortCommand | from .entities.commands import AbortCommand | ||||
| from .error_handling import ErrorHandler | from .error_handling import ErrorHandler | ||||
| from .event_management import EventCollector, EventEmitter, EventHandlerRegistry | |||||
| from .event_management import EventHandler, EventManager | |||||
| from .graph_traversal import EdgeProcessor, SkipPropagator | from .graph_traversal import EdgeProcessor, SkipPropagator | ||||
| from .layers.base import Layer | from .layers.base import Layer | ||||
| from .orchestration import Dispatcher, ExecutionCoordinator | from .orchestration import Dispatcher, ExecutionCoordinator | ||||
| ) | ) | ||||
| # === Event Management === | # === Event Management === | ||||
| # Event collector aggregates events from all subsystems | |||||
| self._event_collector = EventCollector() | |||||
| # Event emitter streams collected events to consumers | |||||
| self._event_emitter = EventEmitter(self._event_collector) | |||||
| # Event manager handles both collection and emission of events | |||||
| self._event_manager = EventManager() | |||||
| # === Error Handling === | # === Error Handling === | ||||
| # Centralized error handler for graph execution errors | # Centralized error handler for graph execution errors | ||||
| # === Event Handler Registry === | # === Event Handler Registry === | ||||
| # Central registry for handling all node execution events | # Central registry for handling all node execution events | ||||
| self._event_handler_registry = EventHandlerRegistry( | |||||
| self._event_handler_registry = EventHandler( | |||||
| graph=self._graph, | graph=self._graph, | ||||
| graph_runtime_state=self._graph_runtime_state, | graph_runtime_state=self._graph_runtime_state, | ||||
| graph_execution=self._graph_execution, | graph_execution=self._graph_execution, | ||||
| response_coordinator=self._response_coordinator, | response_coordinator=self._response_coordinator, | ||||
| event_collector=self._event_collector, | |||||
| event_collector=self._event_manager, | |||||
| edge_processor=self._edge_processor, | edge_processor=self._edge_processor, | ||||
| state_manager=self._state_manager, | state_manager=self._state_manager, | ||||
| error_handler=self._error_handler, | error_handler=self._error_handler, | ||||
| graph_execution=self._graph_execution, | graph_execution=self._graph_execution, | ||||
| state_manager=self._state_manager, | state_manager=self._state_manager, | ||||
| event_handler=self._event_handler_registry, | event_handler=self._event_handler_registry, | ||||
| event_collector=self._event_collector, | |||||
| event_collector=self._event_manager, | |||||
| command_processor=self._command_processor, | command_processor=self._command_processor, | ||||
| worker_pool=self._worker_pool, | worker_pool=self._worker_pool, | ||||
| ) | ) | ||||
| self._dispatcher = Dispatcher( | self._dispatcher = Dispatcher( | ||||
| event_queue=self._event_queue, | event_queue=self._event_queue, | ||||
| event_handler=self._event_handler_registry, | event_handler=self._event_handler_registry, | ||||
| event_collector=self._event_collector, | |||||
| event_collector=self._event_manager, | |||||
| execution_coordinator=self._execution_coordinator, | execution_coordinator=self._execution_coordinator, | ||||
| max_execution_time=self._execution_context.max_execution_time, | max_execution_time=self._execution_context.max_execution_time, | ||||
| event_emitter=self._event_emitter, | |||||
| event_emitter=self._event_manager, | |||||
| ) | ) | ||||
| # === Extensibility === | # === Extensibility === | ||||
| self._start_execution() | self._start_execution() | ||||
| # Yield events as they occur | # Yield events as they occur | ||||
| yield from self._event_emitter.emit_events() | |||||
| yield from self._event_manager.emit_events() | |||||
| # Handle completion | # Handle completion | ||||
| if self._graph_execution.aborted: | if self._graph_execution.aborted: | ||||
| def _initialize_layers(self) -> None: | def _initialize_layers(self) -> None: | ||||
| """Initialize layers with context.""" | """Initialize layers with context.""" | ||||
| self._event_collector.set_layers(self._layers) | |||||
| self._event_manager.set_layers(self._layers) | |||||
| for layer in self._layers: | for layer in self._layers: | ||||
| try: | try: | ||||
| layer.initialize(self._graph_runtime_state, self._command_channel) | layer.initialize(self._graph_runtime_state, self._command_channel) |
| from core.workflow.graph_events.base import GraphNodeEventBase | from core.workflow.graph_events.base import GraphNodeEventBase | ||||
| from ..event_management import EventCollector, EventEmitter | |||||
| from ..event_management import EventManager | |||||
| from .execution_coordinator import ExecutionCoordinator | from .execution_coordinator import ExecutionCoordinator | ||||
| if TYPE_CHECKING: | if TYPE_CHECKING: | ||||
| from ..event_management import EventHandlerRegistry | |||||
| from ..event_management import EventHandler | |||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| event_queue: queue.Queue[GraphNodeEventBase], | event_queue: queue.Queue[GraphNodeEventBase], | ||||
| event_handler: "EventHandlerRegistry", | |||||
| event_collector: EventCollector, | |||||
| event_handler: "EventHandler", | |||||
| event_collector: EventManager, | |||||
| execution_coordinator: ExecutionCoordinator, | execution_coordinator: ExecutionCoordinator, | ||||
| max_execution_time: int, | max_execution_time: int, | ||||
| event_emitter: EventEmitter | None = None, | |||||
| event_emitter: EventManager | None = None, | |||||
| ) -> None: | ) -> None: | ||||
| """ | """ | ||||
| Initialize the dispatcher. | Initialize the dispatcher. | ||||
| Args: | Args: | ||||
| event_queue: Queue of events from workers | event_queue: Queue of events from workers | ||||
| event_handler: Event handler registry for processing events | event_handler: Event handler registry for processing events | ||||
| event_collector: Event collector for collecting unhandled events | |||||
| event_collector: Event manager for collecting unhandled events | |||||
| execution_coordinator: Coordinator for execution flow | execution_coordinator: Coordinator for execution flow | ||||
| max_execution_time: Maximum execution time in seconds | max_execution_time: Maximum execution time in seconds | ||||
| event_emitter: Optional event emitter to signal completion | |||||
| event_emitter: Optional event manager to signal completion | |||||
| """ | """ | ||||
| self._event_queue = event_queue | self._event_queue = event_queue | ||||
| self._event_handler = event_handler | self._event_handler = event_handler |
| from ..command_processing import CommandProcessor | from ..command_processing import CommandProcessor | ||||
| from ..domain import GraphExecution | from ..domain import GraphExecution | ||||
| from ..event_management import EventCollector | |||||
| from ..event_management import EventManager | |||||
| from ..state_management import UnifiedStateManager | from ..state_management import UnifiedStateManager | ||||
| from ..worker_management import SimpleWorkerPool | from ..worker_management import SimpleWorkerPool | ||||
| if TYPE_CHECKING: | if TYPE_CHECKING: | ||||
| from ..event_management import EventHandlerRegistry | |||||
| from ..event_management import EventHandler | |||||
| @final | @final | ||||
| self, | self, | ||||
| graph_execution: GraphExecution, | graph_execution: GraphExecution, | ||||
| state_manager: UnifiedStateManager, | state_manager: UnifiedStateManager, | ||||
| event_handler: "EventHandlerRegistry", | |||||
| event_collector: EventCollector, | |||||
| event_handler: "EventHandler", | |||||
| event_collector: EventManager, | |||||
| command_processor: CommandProcessor, | command_processor: CommandProcessor, | ||||
| worker_pool: SimpleWorkerPool, | worker_pool: SimpleWorkerPool, | ||||
| ) -> None: | ) -> None: | ||||
| graph_execution: Graph execution aggregate | graph_execution: Graph execution aggregate | ||||
| state_manager: Unified state manager | state_manager: Unified state manager | ||||
| event_handler: Event handler registry for processing events | event_handler: Event handler registry for processing events | ||||
| event_collector: Event collector for collecting events | |||||
| event_collector: Event manager for collecting events | |||||
| command_processor: Processor for commands | command_processor: Processor for commands | ||||
| worker_pool: Pool of workers | worker_pool: Pool of workers | ||||
| """ | """ |