初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+23
@@ -0,0 +1,23 @@
|
||||
"""Workflow execution runtime (scheduler, context, event sink)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.workflows.runtime.context import (
|
||||
CancelToken,
|
||||
RunContext,
|
||||
WorkflowPaused,
|
||||
WorkflowRuntimeDeps,
|
||||
)
|
||||
from deerflow.workflows.runtime.engine import EngineResult, WorkflowEngine
|
||||
from deerflow.workflows.runtime.sink import PersistingWorkflowEventSink, envelope_from_row
|
||||
|
||||
__all__ = [
|
||||
"CancelToken",
|
||||
"EngineResult",
|
||||
"PersistingWorkflowEventSink",
|
||||
"RunContext",
|
||||
"WorkflowEngine",
|
||||
"WorkflowPaused",
|
||||
"WorkflowRuntimeDeps",
|
||||
"envelope_from_row",
|
||||
]
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""Constrained Python execution for the ``code`` node.
|
||||
|
||||
The snippet runs in a **separate interpreter process** with:
|
||||
|
||||
* a scrubbed environment (no API keys, tokens, or DSNs inherited),
|
||||
* a fresh temporary working directory that is deleted afterwards,
|
||||
* an address-space cap and no core dumps (POSIX ``setrlimit``),
|
||||
* sockets disabled unless the node explicitly allows network,
|
||||
* a hard wall-clock timeout with process-group kill,
|
||||
* bounded stdout/stderr capture.
|
||||
|
||||
This is process isolation, not a container: a determined snippet can still read
|
||||
world-readable files on the host. That is why ``workflows.code.enabled``
|
||||
defaults to ``false`` — turn it on only where the container sandbox or an OS
|
||||
level jail already confines the backend process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_RUNNER_TEMPLATE = """
|
||||
import json, os, sys
|
||||
|
||||
_ALLOW_NETWORK = {allow_network}
|
||||
if not _ALLOW_NETWORK:
|
||||
import socket
|
||||
|
||||
def _blocked(*_args, **_kwargs):
|
||||
raise PermissionError("network access is disabled for this code node")
|
||||
|
||||
socket.socket = _blocked
|
||||
socket.create_connection = _blocked
|
||||
socket.getaddrinfo = _blocked
|
||||
|
||||
try:
|
||||
import resource
|
||||
|
||||
_limit = {memory_bytes}
|
||||
resource.setrlimit(resource.RLIMIT_AS, (_limit, _limit))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
inputs = json.loads(os.environ.get("WORKFLOW_NODE_INPUT") or "{{}}")
|
||||
output = {{}}
|
||||
_globals = {{"__name__": "__workflow_code__", "inputs": inputs, "output": output}}
|
||||
|
||||
with open({source_path!r}, "r", encoding="utf-8") as handle:
|
||||
_source = handle.read()
|
||||
|
||||
exec(compile(_source, "<workflow-code>", "exec"), _globals)
|
||||
|
||||
_result = _globals.get("output")
|
||||
if not isinstance(_result, dict):
|
||||
_result = {{"value": _result}}
|
||||
sys.stdout.write("\\n__WORKFLOW_RESULT__" + json.dumps(_result, ensure_ascii=False, default=str))
|
||||
"""
|
||||
|
||||
|
||||
def _safe_env() -> dict[str, str]:
|
||||
env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"LANG": os.environ.get("LANG", "C.UTF-8"),
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"HOME": "",
|
||||
}
|
||||
if sys.platform == "win32":
|
||||
for key in ("SYSTEMROOT", "COMSPEC", "TEMP", "TMP"):
|
||||
if os.environ.get(key):
|
||||
env[key] = os.environ[key]
|
||||
return env
|
||||
|
||||
|
||||
async def run_python_snippet(
|
||||
*,
|
||||
source: str,
|
||||
inputs: dict[str, Any],
|
||||
timeout_seconds: int,
|
||||
max_output_chars: int,
|
||||
memory_limit_mb: int,
|
||||
allow_network: bool,
|
||||
) -> dict[str, Any]:
|
||||
workdir = Path(tempfile.mkdtemp(prefix="wf-code-"))
|
||||
try:
|
||||
source_path = workdir / "snippet.py"
|
||||
source_path.write_text(source, encoding="utf-8")
|
||||
runner_path = workdir / "runner.py"
|
||||
runner_path.write_text(
|
||||
_RUNNER_TEMPLATE.format(
|
||||
allow_network=bool(allow_network),
|
||||
memory_bytes=max(64, int(memory_limit_mb)) * 1024 * 1024,
|
||||
source_path=str(source_path),
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
env = _safe_env()
|
||||
env["WORKFLOW_NODE_INPUT"] = json.dumps(inputs, ensure_ascii=False, default=str)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-S",
|
||||
str(runner_path),
|
||||
cwd=str(workdir),
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
start_new_session=sys.platform != "win32",
|
||||
)
|
||||
try:
|
||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
_terminate(process)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"代码节点执行超时({timeout_seconds}s)",
|
||||
details={"timeoutSeconds": timeout_seconds},
|
||||
) from exc
|
||||
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace")
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace")[:max_output_chars]
|
||||
if process.returncode != 0:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"代码节点执行失败",
|
||||
details={"exitCode": process.returncode, "stderr": stderr[-2000:]},
|
||||
)
|
||||
|
||||
marker = "__WORKFLOW_RESULT__"
|
||||
index = stdout.rfind(marker)
|
||||
result: dict[str, Any] = {}
|
||||
logs = stdout
|
||||
if index >= 0:
|
||||
logs = stdout[:index]
|
||||
try:
|
||||
parsed = json.loads(stdout[index + len(marker) :])
|
||||
result = parsed if isinstance(parsed, dict) else {"value": parsed}
|
||||
except json.JSONDecodeError:
|
||||
result = {}
|
||||
return {
|
||||
"result": result,
|
||||
"stdout": logs[:max_output_chars],
|
||||
"stderr": stderr,
|
||||
}
|
||||
finally:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _terminate(process: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
if sys.platform != "win32":
|
||||
os.killpg(os.getpgid(process.pid), 9)
|
||||
else:
|
||||
process.kill()
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
logger.debug("code node process already gone", exc_info=True)
|
||||
|
||||
|
||||
__all__ = ["run_python_snippet"]
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
"""Run-scoped execution context shared by the engine and node executors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.events import WorkflowEventType
|
||||
from deerflow.workflows.schemas import ArtifactRef, WorkflowGraph, WorkflowNode
|
||||
|
||||
EmitFn = Callable[..., Awaitable[Any]]
|
||||
|
||||
|
||||
def _public_artifact_payload(ref: ArtifactRef) -> dict[str, Any]:
|
||||
"""Wire payload for ``artifact.created``.
|
||||
|
||||
The sandbox/host ``path`` stays internal: clients address artifacts by id
|
||||
through the run-scoped content endpoints, never by filesystem path."""
|
||||
from deerflow.workflows.events import artifact_kind
|
||||
|
||||
return {
|
||||
"artifactId": ref.artifact_id,
|
||||
"name": ref.name,
|
||||
"kind": artifact_kind(ref.mime_type),
|
||||
"mimeType": ref.mime_type,
|
||||
"sizeBytes": ref.size_bytes,
|
||||
"preview": ref.preview,
|
||||
}
|
||||
|
||||
|
||||
class WorkflowPaused(Exception): # noqa: N818 - control-flow signal, not an error
|
||||
"""Raised by ``human_input`` to suspend the run until a resume arrives."""
|
||||
|
||||
def __init__(self, node_id: str, pending_input: dict[str, Any]) -> None:
|
||||
super().__init__(f"workflow paused at {node_id}")
|
||||
self.node_id = node_id
|
||||
self.pending_input = pending_input
|
||||
|
||||
|
||||
class CancelToken:
|
||||
"""Cooperative cancellation shared between the executor and node code."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._event = asyncio.Event()
|
||||
self._reason = "cancelled"
|
||||
|
||||
def cancel(self, reason: str = "cancelled") -> None:
|
||||
self._reason = reason
|
||||
self._event.set()
|
||||
|
||||
@property
|
||||
def cancelled(self) -> bool:
|
||||
return self._event.is_set()
|
||||
|
||||
async def wait(self) -> None:
|
||||
await self._event.wait()
|
||||
|
||||
def raise_if_cancelled(self, node_id: str | None = None) -> None:
|
||||
if self._event.is_set():
|
||||
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowRuntimeDeps:
|
||||
"""Injected capabilities. The harness never imports the app layer, so every
|
||||
integration point (agents, HTTP egress, SQL, sandbox, subworkflows) arrives
|
||||
as an async callable supplied by the caller. ``None`` means "not available",
|
||||
which node executors turn into a typed ``WORKFLOW_RESOURCE_MISSING``."""
|
||||
|
||||
run_agent: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_http: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_sql: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_code: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_subworkflow: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
# Starts (or reattaches to) a durable Deep Research report-writing job.
|
||||
# The app adapter owns its session/job/event stores; workflow nodes only
|
||||
# receive the final report plus safe progress callbacks.
|
||||
run_deep_research: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
save_artifact: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
resolve_credential: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
|
||||
resolve_data_source: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunContext:
|
||||
"""Everything a node executor is allowed to see."""
|
||||
|
||||
run_id: str
|
||||
workflow_id: str
|
||||
version_id: str
|
||||
owner_id: str
|
||||
graph: WorkflowGraph
|
||||
inputs: dict[str, Any]
|
||||
deps: WorkflowRuntimeDeps
|
||||
cancel: CancelToken
|
||||
emit: EmitFn
|
||||
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
loop_state: dict[str, int] = field(default_factory=dict)
|
||||
resume_payload: dict[str, Any] | None = None
|
||||
node_run_ids: dict[str, str] = field(default_factory=dict)
|
||||
current_node_id: str | None = None
|
||||
env: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
"""Whitelisted tree visible to expressions. Credentials never appear."""
|
||||
return {
|
||||
"inputs": self.inputs,
|
||||
"nodes": self.node_results,
|
||||
"run": {"id": self.run_id, "workflowId": self.workflow_id, "versionId": self.version_id},
|
||||
"loop": dict(self.loop_state),
|
||||
"env": self.env,
|
||||
}
|
||||
|
||||
async def emit_event(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
) -> None:
|
||||
node = node_id or self.current_node_id
|
||||
await self.emit(
|
||||
event_type,
|
||||
data=data or {},
|
||||
node_id=node,
|
||||
node_run_id=self.node_run_ids.get(node) if node else None,
|
||||
)
|
||||
|
||||
async def register_artifact(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
*,
|
||||
name: str,
|
||||
mime_type: str = "application/octet-stream",
|
||||
path: str = "",
|
||||
size_bytes: int | None = None,
|
||||
preview: str | None = None,
|
||||
) -> ArtifactRef | None:
|
||||
"""Persist an artifact reference and emit ``artifact.created``."""
|
||||
if self.deps.save_artifact is None:
|
||||
return None
|
||||
row = await self.deps.save_artifact(
|
||||
run_id=self.run_id,
|
||||
node_id=node.id,
|
||||
name=name,
|
||||
mime_type=mime_type,
|
||||
path=path,
|
||||
size_bytes=size_bytes,
|
||||
preview=preview,
|
||||
)
|
||||
ref = ArtifactRef(
|
||||
artifactId=str(row.get("id") or ""),
|
||||
name=str(row.get("name") or name),
|
||||
mimeType=str(row.get("mime_type") or mime_type),
|
||||
path=str(row.get("path") or path),
|
||||
sizeBytes=row.get("size_bytes"),
|
||||
preview=row.get("preview"),
|
||||
)
|
||||
await self.emit_event("artifact.created", data=_public_artifact_payload(ref), node_id=node.id)
|
||||
return ref
|
||||
|
||||
|
||||
__all__ = ["CancelToken", "RunContext", "WorkflowPaused", "WorkflowRuntimeDeps"]
|
||||
+531
@@ -0,0 +1,531 @@
|
||||
"""Persisted DAG scheduler for workflow runs.
|
||||
|
||||
Scheduling model
|
||||
----------------
|
||||
Every edge carries a state: ``pending`` → ``active`` (its source chose this
|
||||
port) or ``pruned`` (the source chose a different port). A node is *ready* when
|
||||
none of its non-back incoming edges is ``pending`` and at least one is
|
||||
``active``; a node whose incoming edges all resolved to ``pruned`` is *skipped*
|
||||
and prunes its own outgoing edges in turn. That single rule gives us fan-out
|
||||
parallelism, condition branching, and merge joins without a topological
|
||||
pre-pass, and it terminates because each superstep resolves at least one edge.
|
||||
|
||||
Loops are the one cycle we allow: the back edge (``loop`` node, port
|
||||
``continue``) is excluded from readiness gating, and activating it clears the
|
||||
body's results so the next iteration re-runs them. Every re-entry is capped by
|
||||
``settings.max_loop_iterations``.
|
||||
|
||||
Durability: node results are written to the run store as they complete, so a
|
||||
resumed or re-leased run replays only what is missing. ``human_input`` raises
|
||||
:class:`WorkflowPaused`, which the caller turns into ``awaiting_input``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.ports import ACTIVE_PORTS, LOOP_CONTINUE_PORT
|
||||
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowEdge, WorkflowGraph, WorkflowNode
|
||||
|
||||
|
||||
class ExecutorRegistry(Protocol):
|
||||
"""Structural view of ``deerflow.workflows.nodes.NodeRegistry``.
|
||||
|
||||
Declared here rather than imported so the scheduler has no dependency on
|
||||
the node package (which depends on the runtime context).
|
||||
"""
|
||||
|
||||
def get(self, node_type: str) -> Any: ...
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PENDING = "pending"
|
||||
ACTIVE = "active"
|
||||
PRUNED = "pruned"
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineResult:
|
||||
output: dict[str, Any] = field(default_factory=dict)
|
||||
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
loop_state: dict[str, int] = field(default_factory=dict)
|
||||
steps: int = 0
|
||||
|
||||
|
||||
def _is_back_edge(edge: WorkflowEdge, nodes: dict[str, WorkflowNode]) -> bool:
|
||||
source = nodes.get(edge.source)
|
||||
return bool(source and source.type == "loop" and (edge.source_port or "") == LOOP_CONTINUE_PORT)
|
||||
|
||||
|
||||
def _node_started_event_data(node: WorkflowNode, node_attempt: int) -> dict[str, Any]:
|
||||
"""Safe presentation metadata for workflow clients.
|
||||
|
||||
This stays intentionally small: UI clients need stable agent identity and
|
||||
an optional explicit parallel group, but never prompt templates, tools,
|
||||
credentials, or the full node config. A missing group is valid; clients
|
||||
may still use their backwards-compatible name-based presentation.
|
||||
"""
|
||||
|
||||
data: dict[str, Any] = {
|
||||
"nodeType": node.type,
|
||||
"nodeKind": node.type,
|
||||
"name": node.name,
|
||||
"attempt": node_attempt,
|
||||
}
|
||||
group_id = node.config.get("parallelGroupId") or node.config.get("parallel_group_id")
|
||||
if isinstance(group_id, str) and group_id.strip():
|
||||
data["parallelGroupId"] = group_id.strip()
|
||||
round_index = node.config.get("roundIndex")
|
||||
if round_index is None:
|
||||
round_index = node.config.get("round_index")
|
||||
if isinstance(round_index, int) and round_index >= 0:
|
||||
data["roundIndex"] = round_index
|
||||
|
||||
if node.type == "agent":
|
||||
agent_id = node.config.get("agentId") or node.config.get("agent_id")
|
||||
if isinstance(agent_id, str) and agent_id.strip():
|
||||
agent: dict[str, str] = {
|
||||
"agentId": agent_id.strip(),
|
||||
"name": str(
|
||||
node.config.get("agentName")
|
||||
or node.config.get("agent_name")
|
||||
or node.name
|
||||
or agent_id
|
||||
),
|
||||
}
|
||||
avatar_url = node.config.get("agentAvatarUrl") or node.config.get("agent_avatar_url")
|
||||
if isinstance(avatar_url, str) and avatar_url.strip():
|
||||
agent["avatarUrl"] = avatar_url.strip()
|
||||
data["agent"] = agent
|
||||
return data
|
||||
|
||||
|
||||
class WorkflowEngine:
|
||||
def __init__(self, registry: ExecutorRegistry, *, run_store: WorkflowRunStore | None = None) -> None:
|
||||
self._registry = registry
|
||||
self._store = run_store
|
||||
|
||||
async def run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
*,
|
||||
preloaded: dict[str, dict[str, Any]] | None = None,
|
||||
loop_state: dict[str, int] | None = None,
|
||||
attempt: int = 1,
|
||||
) -> EngineResult:
|
||||
graph: WorkflowGraph = ctx.graph
|
||||
nodes = graph.node_map()
|
||||
outgoing: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
|
||||
incoming: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
|
||||
for edge in graph.edges:
|
||||
outgoing[edge.source].append(edge)
|
||||
incoming[edge.target].append(edge)
|
||||
|
||||
edge_state: dict[str, str] = {edge.id: PENDING for edge in graph.edges}
|
||||
done: set[str] = set()
|
||||
skipped: set[str] = set()
|
||||
|
||||
ctx.loop_state.update(loop_state or {})
|
||||
if preloaded:
|
||||
ctx.node_results.update(preloaded)
|
||||
done |= set(preloaded)
|
||||
# Re-derive edge states from the ports each replayed node selected.
|
||||
for node_id in list(done):
|
||||
self._apply_ports(
|
||||
node_id,
|
||||
ctx.node_results.get(node_id, {}).get("metadata") or {},
|
||||
outgoing,
|
||||
edge_state,
|
||||
)
|
||||
|
||||
semaphore = asyncio.Semaphore(max(1, graph.settings.max_parallelism))
|
||||
steps = 0
|
||||
max_steps = max(1, graph.settings.max_steps)
|
||||
|
||||
while True:
|
||||
ctx.cancel.raise_if_cancelled()
|
||||
self._propagate_skips(nodes, incoming, outgoing, edge_state, done, skipped)
|
||||
ready = [node_id for node_id in nodes if node_id not in done and node_id not in skipped and self._readiness(node_id, incoming, edge_state, nodes) == "ready"]
|
||||
if not ready:
|
||||
break
|
||||
|
||||
steps += len(ready)
|
||||
if steps > max_steps:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
f"运行步数超过上限 {max_steps}",
|
||||
details={"maxSteps": max_steps},
|
||||
)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(self._run_one(nodes[node_id], ctx, semaphore, attempt) for node_id in ready),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
paused: WorkflowPaused | None = None
|
||||
failure: BaseException | None = None
|
||||
for node_id, outcome in zip(ready, results, strict=True):
|
||||
if isinstance(outcome, WorkflowPaused):
|
||||
paused = paused or outcome
|
||||
continue
|
||||
if isinstance(outcome, BaseException):
|
||||
failure = failure or outcome
|
||||
continue
|
||||
ctx.node_results[node_id] = outcome
|
||||
done.add(node_id)
|
||||
metadata = outcome.get("metadata") or {}
|
||||
self._apply_ports(node_id, metadata, outgoing, edge_state)
|
||||
if nodes[node_id].type == "loop":
|
||||
await self._maybe_reset_loop_body(nodes[node_id], ctx, nodes, outgoing, edge_state, done, skipped)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
if paused is not None:
|
||||
raise paused
|
||||
|
||||
return EngineResult(
|
||||
output=self._collect_output(nodes, ctx),
|
||||
node_results=dict(ctx.node_results),
|
||||
loop_state=dict(ctx.loop_state),
|
||||
steps=steps,
|
||||
)
|
||||
|
||||
# ── scheduling helpers ──────────────────────────────────────────────
|
||||
def _readiness(
|
||||
self,
|
||||
node_id: str,
|
||||
incoming: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
nodes: dict[str, WorkflowNode],
|
||||
) -> str:
|
||||
edges = incoming.get(node_id) or []
|
||||
if not edges:
|
||||
return "ready"
|
||||
back = [e for e in edges if _is_back_edge(e, nodes)]
|
||||
if any(edge_state[e.id] == ACTIVE for e in back):
|
||||
return "ready"
|
||||
normal = [e for e in edges if not _is_back_edge(e, nodes)]
|
||||
if not normal:
|
||||
return "blocked"
|
||||
if any(edge_state[e.id] == PENDING for e in normal):
|
||||
return "blocked"
|
||||
if any(edge_state[e.id] == ACTIVE for e in normal):
|
||||
return "ready"
|
||||
return "skip"
|
||||
|
||||
def _apply_ports(
|
||||
self,
|
||||
node_id: str,
|
||||
metadata: dict[str, Any],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
) -> None:
|
||||
active_ports = metadata.get(ACTIVE_PORTS)
|
||||
allowed = {str(p) for p in active_ports} if isinstance(active_ports, list) else None
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
port = edge.source_port or ""
|
||||
if allowed is None or not port or port in allowed:
|
||||
edge_state[edge.id] = ACTIVE
|
||||
else:
|
||||
edge_state[edge.id] = PRUNED
|
||||
|
||||
def _propagate_skips(
|
||||
self,
|
||||
nodes: dict[str, WorkflowNode],
|
||||
incoming: dict[str, list[WorkflowEdge]],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
done: set[str],
|
||||
skipped: set[str],
|
||||
) -> None:
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node_id in nodes:
|
||||
if node_id in done or node_id in skipped:
|
||||
continue
|
||||
if self._readiness(node_id, incoming, edge_state, nodes) != "skip":
|
||||
continue
|
||||
skipped.add(node_id)
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
edge_state[edge.id] = PRUNED
|
||||
changed = True
|
||||
|
||||
async def _maybe_reset_loop_body(
|
||||
self,
|
||||
loop_node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
nodes: dict[str, WorkflowNode],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
done: set[str],
|
||||
skipped: set[str],
|
||||
) -> None:
|
||||
back_edges = [e for e in outgoing.get(loop_node.id) or [] if _is_back_edge(e, nodes) and edge_state[e.id] == ACTIVE]
|
||||
if not back_edges:
|
||||
return
|
||||
body_entry = str(loop_node.config.get("bodyEntry") or loop_node.config.get("body_entry") or "")
|
||||
if not body_entry or body_entry not in nodes:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
"循环节点的 bodyEntry 不存在",
|
||||
node_id=loop_node.id,
|
||||
details={"bodyEntry": body_entry},
|
||||
)
|
||||
body = self._loop_body(body_entry, loop_node.id, outgoing, nodes)
|
||||
for node_id in body:
|
||||
done.discard(node_id)
|
||||
skipped.discard(node_id)
|
||||
ctx.node_results.pop(node_id, None)
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
if not _is_back_edge(edge, nodes):
|
||||
edge_state[edge.id] = PENDING
|
||||
# The back edge stays ACTIVE so the body entry is reachable again; the
|
||||
# loop node prunes it itself once it selects the ``done`` port.
|
||||
if self._store is not None and body:
|
||||
try:
|
||||
await self._store.invalidate_node_runs(ctx.run_id, sorted(body))
|
||||
except Exception: # noqa: BLE001 - bookkeeping must not kill the run
|
||||
logger.warning("failed to invalidate node runs for loop reset", exc_info=True)
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={
|
||||
"phase": "loop.iteration",
|
||||
"iteration": ctx.loop_state.get(loop_node.id, 0),
|
||||
"bodyEntry": body_entry,
|
||||
},
|
||||
node_id=loop_node.id,
|
||||
)
|
||||
|
||||
def _loop_body(
|
||||
self,
|
||||
body_entry: str,
|
||||
loop_id: str,
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
nodes: dict[str, WorkflowNode],
|
||||
) -> set[str]:
|
||||
"""Nodes reachable from ``body_entry`` up to (and including) the loop node."""
|
||||
body: set[str] = set()
|
||||
stack = [body_entry]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if current in body:
|
||||
continue
|
||||
body.add(current)
|
||||
if current == loop_id:
|
||||
continue
|
||||
for edge in outgoing.get(current) or []:
|
||||
if not _is_back_edge(edge, nodes):
|
||||
stack.append(edge.target)
|
||||
return body
|
||||
|
||||
# ── node execution ──────────────────────────────────────────────────
|
||||
async def _run_one(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
semaphore: asyncio.Semaphore,
|
||||
attempt: int,
|
||||
) -> dict[str, Any]:
|
||||
async with semaphore:
|
||||
ctx.cancel.raise_if_cancelled(node.id)
|
||||
retry_cfg = node.config.get("retry") if isinstance(node.config.get("retry"), dict) else {}
|
||||
max_attempts = max(1, min(5, int(retry_cfg.get("maxAttempts") or 1)))
|
||||
backoff = max(0.0, float(retry_cfg.get("backoffSeconds") or 0.5))
|
||||
on_error = str(node.config.get("onError") or "fail")
|
||||
timeout = self._node_timeout(node, ctx)
|
||||
|
||||
last_error: WorkflowError | None = None
|
||||
for node_attempt in range(1, max_attempts + 1):
|
||||
node_run = await self._begin_node_run(ctx, node, attempt, node_attempt)
|
||||
started = time.monotonic()
|
||||
await ctx.emit_event(
|
||||
"node.started",
|
||||
data=_node_started_event_data(node, node_attempt),
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
executor = self._registry.get(node.type)
|
||||
result: NodeResult = await asyncio.wait_for(executor.execute(node, ctx), timeout=timeout)
|
||||
except WorkflowPaused:
|
||||
await self._finish_node_run(ctx, node_run, status="queued", output=None)
|
||||
raise
|
||||
except TimeoutError as exc:
|
||||
last_error = WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"节点执行超时({timeout}s)",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"timeoutSeconds": timeout},
|
||||
)
|
||||
await self._fail_node_run(ctx, node_run, last_error, started)
|
||||
if node_attempt < max_attempts:
|
||||
await asyncio.sleep(backoff * node_attempt)
|
||||
continue
|
||||
if on_error == "continue":
|
||||
return self._error_result(last_error)
|
||||
raise last_error from exc
|
||||
except WorkflowError as exc:
|
||||
last_error = exc
|
||||
exc.node_id = exc.node_id or node.id
|
||||
await self._fail_node_run(ctx, node_run, exc, started)
|
||||
if exc.retryable and node_attempt < max_attempts:
|
||||
await asyncio.sleep(backoff * node_attempt)
|
||||
continue
|
||||
if on_error == "continue":
|
||||
return self._error_result(exc)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - unexpected node bug
|
||||
logger.exception("workflow node %s crashed", node.id)
|
||||
last_error = WorkflowError(
|
||||
"WORKFLOW_INTERNAL",
|
||||
"节点执行发生内部错误",
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
)
|
||||
await self._fail_node_run(ctx, node_run, last_error, started)
|
||||
if on_error == "continue":
|
||||
return self._error_result(last_error)
|
||||
raise last_error from exc
|
||||
|
||||
payload = result.model_dump(by_alias=False)
|
||||
duration = int((time.monotonic() - started) * 1000)
|
||||
await self._finish_node_run(ctx, node_run, status="completed", output=payload, duration_ms=duration)
|
||||
await ctx.emit_event(
|
||||
"node.completed",
|
||||
data={
|
||||
"nodeType": node.type,
|
||||
"durationMs": duration,
|
||||
"data": payload.get("data"),
|
||||
"warnings": payload.get("warnings") or [],
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
return payload
|
||||
|
||||
raise last_error or WorkflowError("WORKFLOW_INTERNAL", "节点执行失败", node_id=node.id)
|
||||
|
||||
def _node_timeout(self, node: WorkflowNode, ctx: RunContext) -> int:
|
||||
configured = node.config.get("timeoutSeconds") or node.config.get("timeout_seconds")
|
||||
graph_limit = ctx.graph.settings.node_timeout_seconds
|
||||
try:
|
||||
value = int(configured) if configured else graph_limit
|
||||
except (TypeError, ValueError):
|
||||
value = graph_limit
|
||||
return max(1, min(value, graph_limit))
|
||||
|
||||
def _error_result(self, error: WorkflowError) -> dict[str, Any]:
|
||||
return {
|
||||
"data": {},
|
||||
"messages": [],
|
||||
"artifacts": [],
|
||||
"metadata": {"failed": True, "error": error.to_body().model_dump(by_alias=True)},
|
||||
"warnings": [error.message],
|
||||
}
|
||||
|
||||
async def _begin_node_run(self, ctx: RunContext, node: WorkflowNode, run_attempt: int, node_attempt: int) -> dict[str, Any] | None:
|
||||
if self._store is None:
|
||||
return None
|
||||
try:
|
||||
row = await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node.id,
|
||||
"node_type": node.type,
|
||||
"status": "running",
|
||||
"attempt": node_attempt,
|
||||
"iteration": int(ctx.loop_state.get(node.id, 0)),
|
||||
"started_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run start for %s", node.id, exc_info=True)
|
||||
return None
|
||||
ctx.node_run_ids[node.id] = str(row.get("id") or "")
|
||||
return row
|
||||
|
||||
async def _finish_node_run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
node_run: dict[str, Any] | None,
|
||||
*,
|
||||
status: str,
|
||||
output: dict[str, Any] | None,
|
||||
duration_ms: int | None = None,
|
||||
) -> None:
|
||||
if self._store is None or node_run is None:
|
||||
return
|
||||
try:
|
||||
await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node_run["node_id"],
|
||||
"node_type": node_run["node_type"],
|
||||
"attempt": node_run["attempt"],
|
||||
"iteration": node_run["iteration"],
|
||||
"status": status,
|
||||
"output": output,
|
||||
"duration_ms": duration_ms,
|
||||
"finished_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run finish", exc_info=True)
|
||||
|
||||
async def _fail_node_run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
node_run: dict[str, Any] | None,
|
||||
error: WorkflowError,
|
||||
started: float,
|
||||
) -> None:
|
||||
duration = int((time.monotonic() - started) * 1000)
|
||||
body = error.to_body().model_dump(by_alias=True)
|
||||
if self._store is not None and node_run is not None:
|
||||
try:
|
||||
await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node_run["node_id"],
|
||||
"node_type": node_run["node_type"],
|
||||
"attempt": node_run["attempt"],
|
||||
"iteration": node_run["iteration"],
|
||||
"status": "failed",
|
||||
"error": body,
|
||||
"duration_ms": duration,
|
||||
"finished_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run failure", exc_info=True)
|
||||
await ctx.emit_event(
|
||||
"node.failed",
|
||||
data={"error": body, "durationMs": duration},
|
||||
node_id=error.node_id,
|
||||
)
|
||||
|
||||
def _collect_output(self, nodes: dict[str, WorkflowNode], ctx: RunContext) -> dict[str, Any]:
|
||||
output: dict[str, Any] = {}
|
||||
for node_id, node in nodes.items():
|
||||
if node.type != "output":
|
||||
continue
|
||||
result = ctx.node_results.get(node_id)
|
||||
if isinstance(result, dict) and isinstance(result.get("data"), dict):
|
||||
output.update(result["data"])
|
||||
return output
|
||||
|
||||
|
||||
__all__ = ["EngineResult", "WorkflowEngine"]
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Persist-then-publish event sink for workflow runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.workflows.events import WorkflowEventEnvelope, WorkflowEventType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
|
||||
|
||||
|
||||
class PersistingWorkflowEventSink:
|
||||
"""Assigns ``seq`` in the database first, then fans out to live listeners.
|
||||
|
||||
A persistence failure never aborts the run, but the event is **dropped
|
||||
rather than published**: SSE clients tail by cursor and rebuild from the
|
||||
durable log, so a live-only frame with a made-up seq would render once and
|
||||
then vanish on reconnect (a "ghost event"). Callers can see the drop via
|
||||
``degraded_count`` / the ``eventPersistenceDegraded`` marker on the returned
|
||||
envelope; the stream endpoint synthesises a closing frame from the run row
|
||||
when a terminal event could not be logged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkflowEventStore,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
publish: PublishFn | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._run_id = run_id
|
||||
self._workflow_id = workflow_id
|
||||
self._version_id = version_id
|
||||
self._publish = publish
|
||||
self.degraded_count = 0
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> WorkflowEventEnvelope:
|
||||
payload = data or {}
|
||||
seq = 0
|
||||
degraded = False
|
||||
try:
|
||||
row = await self._store.append(
|
||||
run_id=self._run_id,
|
||||
workflow_id=self._workflow_id,
|
||||
version_id=self._version_id,
|
||||
event_type=event_type,
|
||||
payload=payload,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
)
|
||||
seq = int(row.get("seq") or 0)
|
||||
except Exception as exc: # noqa: BLE001 - the run must survive a log outage
|
||||
logger.warning(
|
||||
"workflow event not persisted (dropped, not published): run=%s event=%s node=%s error=%s:%s",
|
||||
self._run_id,
|
||||
event_type,
|
||||
node_id,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
self.degraded_count += 1
|
||||
degraded = True
|
||||
|
||||
envelope = WorkflowEventEnvelope(
|
||||
runId=self._run_id,
|
||||
workflowId=self._workflow_id,
|
||||
versionId=self._version_id,
|
||||
seq=seq,
|
||||
event=event_type,
|
||||
nodeId=node_id,
|
||||
nodeRunId=node_run_id,
|
||||
timestamp=datetime.now(UTC),
|
||||
data={**payload, "eventPersistenceDegraded": True} if degraded else payload,
|
||||
)
|
||||
if degraded:
|
||||
return envelope
|
||||
if self._publish is not None:
|
||||
try:
|
||||
await self._publish(envelope.to_sse_dict())
|
||||
except Exception: # noqa: BLE001 - live fan-out is best effort
|
||||
logger.debug("workflow live publish failed", exc_info=True)
|
||||
return envelope
|
||||
|
||||
|
||||
def envelope_from_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Turn a stored event row into the SSE wire object (replay path)."""
|
||||
return {
|
||||
"schemaVersion": "1.0",
|
||||
"runId": row.get("run_id"),
|
||||
"workflowId": row.get("workflow_id"),
|
||||
"versionId": row.get("version_id"),
|
||||
"seq": int(row.get("seq") or 0),
|
||||
"event": row.get("event_type"),
|
||||
"nodeId": row.get("node_id"),
|
||||
"nodeRunId": row.get("node_run_id"),
|
||||
"timestamp": row.get("created_at"),
|
||||
"data": row.get("payload") or {},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["PersistingWorkflowEventSink", "envelope_from_row"]
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
"""Read-only query execution for the ``sql_read`` node.
|
||||
|
||||
Engines are cached per DSN with a tiny pool because a workflow typically hits
|
||||
the same reporting database many times in one run. Values are bound as
|
||||
parameters and every cell is truncated before it can reach an event payload.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engines: dict[str, AsyncEngine] = {}
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def _get_engine(dsn: str) -> AsyncEngine:
|
||||
async with _lock:
|
||||
engine = _engines.get(dsn)
|
||||
if engine is None:
|
||||
engine = create_async_engine(
|
||||
dsn,
|
||||
pool_pre_ping=True,
|
||||
pool_size=2,
|
||||
max_overflow=2,
|
||||
pool_recycle=1800,
|
||||
)
|
||||
_engines[dsn] = engine
|
||||
return engine
|
||||
|
||||
|
||||
async def dispose_engines() -> None:
|
||||
async with _lock:
|
||||
for engine in _engines.values():
|
||||
await engine.dispose()
|
||||
_engines.clear()
|
||||
|
||||
|
||||
def _coerce(value: Any, max_chars: int) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (bytes, bytearray, memoryview)):
|
||||
return f"<binary {len(bytes(value))} bytes>"
|
||||
text_value = str(value)
|
||||
return text_value if len(text_value) <= max_chars else text_value[:max_chars] + "…"
|
||||
|
||||
|
||||
async def execute_read_query(
|
||||
*,
|
||||
dsn: str,
|
||||
statement: str,
|
||||
parameters: dict[str, Any],
|
||||
max_rows: int,
|
||||
timeout_seconds: int,
|
||||
max_cell_chars: int = 4000,
|
||||
) -> dict[str, Any]:
|
||||
engine = await _get_engine(dsn)
|
||||
|
||||
async def _run() -> dict[str, Any]:
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text(statement), parameters)
|
||||
columns = list(result.keys())
|
||||
rows = result.fetchmany(max_rows)
|
||||
truncated = bool(result.fetchone())
|
||||
return {
|
||||
"columns": columns,
|
||||
"rows": [{col: _coerce(val, max_cell_chars) for col, val in zip(columns, row, strict=False)} for row in rows],
|
||||
"rowCount": len(rows),
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_run(), timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"数据库查询超时({timeout_seconds}s)",
|
||||
retryable=True,
|
||||
details={"timeoutSeconds": timeout_seconds},
|
||||
) from exc
|
||||
except WorkflowError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - never leak DSN/credentials upward
|
||||
logger.warning("workflow sql_read failed", exc_info=True)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"数据库查询执行失败",
|
||||
retryable=True,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
|
||||
__all__ = ["dispose_engines", "execute_read_query"]
|
||||
Reference in new issue
Block a user