代码提交

This commit is contained in:
董宏凯 committed 2026-08-30 01:25:20 +08:00
1 parent 4ac7bc21cd
commit 17cf849361
110 files changed
+17007 -518

No files matched your search

@@ -0,0 +1,64 @@
"""Workflow Studio protocol package (phase 0 — frozen contracts).
Execution, persistence, and Gateway routers land in later phases.
This package must never import ``app.*``.
"""
from deerflow.workflows.errors import (
ALL_WORKFLOW_ERROR_CODES,
WorkflowError,
WorkflowErrorBody,
WorkflowErrorCode,
)
from deerflow.workflows.events import (
ALL_WORKFLOW_EVENT_TYPES,
TERMINAL_EVENT_TYPES,
WorkflowEventEnvelope,
WorkflowEventSink,
WorkflowEventType,
)
from deerflow.workflows.schemas import (
ALL_NODE_TYPES,
ALL_RUN_STATUSES,
SCHEMA_VERSION,
TERMINAL_RUN_STATUSES,
ArtifactRef,
NodeResult,
NodeType,
ResumeRunRequest,
RunStatus,
StartRunRequest,
WorkflowEdge,
WorkflowGraph,
WorkflowGraphSettings,
WorkflowNode,
)
from deerflow.workflows.validator import assert_publishable, validate_workflow_graph
__all__ = [
"ALL_NODE_TYPES",
"ALL_RUN_STATUSES",
"ALL_WORKFLOW_ERROR_CODES",
"ALL_WORKFLOW_EVENT_TYPES",
"SCHEMA_VERSION",
"TERMINAL_EVENT_TYPES",
"TERMINAL_RUN_STATUSES",
"ArtifactRef",
"NodeResult",
"NodeType",
"ResumeRunRequest",
"RunStatus",
"StartRunRequest",
"WorkflowEdge",
"WorkflowError",
"WorkflowErrorBody",
"WorkflowErrorCode",
"WorkflowEventEnvelope",
"WorkflowEventSink",
"WorkflowEventType",
"WorkflowGraph",
"WorkflowGraphSettings",
"WorkflowNode",
"assert_publishable",
"validate_workflow_graph",
]
@@ -0,0 +1,150 @@
"""Stable workflow error codes and exception types (protocol v1).
User-visible ``message`` must never include stack traces, SQL passwords,
full response bodies, or internal host paths. Full exceptions go to
controlled server logs only.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
WorkflowErrorCode = Literal[
# graph / publish
"WORKFLOW_SCHEMA_INVALID",
"WORKFLOW_GRAPH_CYCLE",
"WORKFLOW_NODE_NOT_FOUND",
"WORKFLOW_EDGE_INVALID",
"WORKFLOW_EXPRESSION_INVALID",
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
"WORKFLOW_RESOURCE_MISSING",
"WORKFLOW_RESOURCE_FORBIDDEN",
"WORKFLOW_LIMIT_EXCEEDED",
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"WORKFLOW_DRAFT_CONFLICT",
"WORKFLOW_VERSION_NOT_FOUND",
# run lifecycle
"WORKFLOW_RUN_NOT_FOUND",
"WORKFLOW_RUN_CONFLICT",
"WORKFLOW_RUN_NOT_RESUMABLE",
"WORKFLOW_RESUME_TOKEN_INVALID",
"WORKFLOW_CANCELLED",
"WORKFLOW_TIMEOUT",
"WORKFLOW_MAX_STEPS",
"WORKFLOW_MAX_LOOP_ITERATIONS",
"WORKFLOW_BUDGET_EXCEEDED",
# node categories
"WORKFLOW_INPUT_INVALID",
"WORKFLOW_AGENT_FAILED",
"WORKFLOW_SKILL_FAILED",
"WORKFLOW_HTTP_POLICY_DENIED",
"WORKFLOW_HTTP_FAILED",
"WORKFLOW_SQL_POLICY_DENIED",
"WORKFLOW_SQL_FAILED",
"WORKFLOW_CODE_SANDBOX_FAILED",
"WORKFLOW_TRANSFORM_FAILED",
"WORKFLOW_CONDITION_FAILED",
"WORKFLOW_MERGE_FAILED",
"WORKFLOW_HUMAN_INPUT_INVALID",
"WORKFLOW_SUBWORKFLOW_FAILED",
"WORKFLOW_LOOP_FAILED",
# auth / embed
"WORKFLOW_FORBIDDEN",
"WORKFLOW_EMBED_TICKET_INVALID",
"WORKFLOW_EMBED_ORIGIN_DENIED",
# generic
"WORKFLOW_INTERNAL",
]
ALL_WORKFLOW_ERROR_CODES: tuple[WorkflowErrorCode, ...] = (
"WORKFLOW_SCHEMA_INVALID",
"WORKFLOW_GRAPH_CYCLE",
"WORKFLOW_NODE_NOT_FOUND",
"WORKFLOW_EDGE_INVALID",
"WORKFLOW_EXPRESSION_INVALID",
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
"WORKFLOW_RESOURCE_MISSING",
"WORKFLOW_RESOURCE_FORBIDDEN",
"WORKFLOW_LIMIT_EXCEEDED",
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"WORKFLOW_DRAFT_CONFLICT",
"WORKFLOW_VERSION_NOT_FOUND",
"WORKFLOW_RUN_NOT_FOUND",
"WORKFLOW_RUN_CONFLICT",
"WORKFLOW_RUN_NOT_RESUMABLE",
"WORKFLOW_RESUME_TOKEN_INVALID",
"WORKFLOW_CANCELLED",
"WORKFLOW_TIMEOUT",
"WORKFLOW_MAX_STEPS",
"WORKFLOW_MAX_LOOP_ITERATIONS",
"WORKFLOW_BUDGET_EXCEEDED",
"WORKFLOW_INPUT_INVALID",
"WORKFLOW_AGENT_FAILED",
"WORKFLOW_SKILL_FAILED",
"WORKFLOW_HTTP_POLICY_DENIED",
"WORKFLOW_HTTP_FAILED",
"WORKFLOW_SQL_POLICY_DENIED",
"WORKFLOW_SQL_FAILED",
"WORKFLOW_CODE_SANDBOX_FAILED",
"WORKFLOW_TRANSFORM_FAILED",
"WORKFLOW_CONDITION_FAILED",
"WORKFLOW_MERGE_FAILED",
"WORKFLOW_HUMAN_INPUT_INVALID",
"WORKFLOW_SUBWORKFLOW_FAILED",
"WORKFLOW_LOOP_FAILED",
"WORKFLOW_FORBIDDEN",
"WORKFLOW_EMBED_TICKET_INVALID",
"WORKFLOW_EMBED_ORIGIN_DENIED",
"WORKFLOW_INTERNAL",
)
class WorkflowErrorBody(BaseModel):
"""Wire-format error body shared by API responses and ``run.failed`` events."""
code: WorkflowErrorCode
message: str
retryable: bool = False
node_id: str | None = Field(default=None, alias="nodeId")
details: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowError(Exception):
"""Raised by validators / node executors; convertible to :class:`WorkflowErrorBody`."""
def __init__(
self,
code: WorkflowErrorCode,
message: str,
*,
retryable: bool = False,
node_id: str | None = None,
details: dict[str, Any] | None = None,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.retryable = retryable
self.node_id = node_id
self.details = details or {}
def to_body(self) -> WorkflowErrorBody:
return WorkflowErrorBody(
code=self.code,
message=self.message,
retryable=self.retryable,
node_id=self.node_id,
details=self.details,
)
__all__ = [
"ALL_WORKFLOW_ERROR_CODES",
"WorkflowError",
"WorkflowErrorBody",
"WorkflowErrorCode",
]
@@ -0,0 +1,158 @@
"""Workflow SSE event protocol v1.0.
Invariant: **persist first, then publish**. ``seq`` is monotonic per run.
Heartbeat comments (``: heartbeat``) are live-only and never stored.
Canonical ``data`` payloads (frozen contract — REST replay and SSE must agree):
node.tool.started {"name": "web_search", "toolCallId": "call_001",
"title": "联网检索", "args": {"path": "..."}}
node.tool.finished {"name": "web_search", "toolCallId": "call_001",
"status": "succeeded" | "failed", "summary": "获得 12 条结果"}
run.awaiting_input {"resumeToken": "one-time-token", "prompt": "请确认是否发布",
"formSchema": {...}, "actions": ["approve", "reject"]}
artifact.created {"artifactId": "artifact_001", "name": "weekly-report.docx",
"kind": "file" | "markdown" | "json" | "table",
"mimeType": "...", "sizeBytes": 1234, "preview": "..."}
node.started {"nodeType": ..., "name": ..., "attempt": 1,
"agent": {...}, "parallelGroupId": "...", "roundIndex": 2}
Rules: tool identity is ``name`` (never ``toolName``); outcomes are ``status``
(never ``ok``); human-input descriptors are flattened (never nested under
``pendingInput``); artifacts never carry filesystem paths; agent nodes expose
display metadata only (no system prompts, skill whitelists or credentials).
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any, Literal, Protocol, runtime_checkable
from pydantic import BaseModel, Field
SCHEMA_VERSION = "1.0"
WorkflowEventType = Literal[
"run.created",
"run.queued",
"run.started",
"node.queued",
"node.started",
"node.progress",
"node.output.delta",
"node.tool.started",
"node.tool.finished",
"artifact.created",
"node.completed",
"node.failed",
"run.awaiting_input",
"run.resumed",
"run.cancel_requested",
"run.cancelled",
"run.completed",
"run.failed",
]
ALL_WORKFLOW_EVENT_TYPES: tuple[WorkflowEventType, ...] = (
"run.created",
"run.queued",
"run.started",
"node.queued",
"node.started",
"node.progress",
"node.output.delta",
"node.tool.started",
"node.tool.finished",
"artifact.created",
"node.completed",
"node.failed",
"run.awaiting_input",
"run.resumed",
"run.cancel_requested",
"run.cancelled",
"run.completed",
"run.failed",
)
TERMINAL_EVENT_TYPES: frozenset[WorkflowEventType] = frozenset({"run.completed", "run.failed", "run.cancelled"})
_KIND_BY_MIME: tuple[tuple[tuple[str, ...], str], ...] = (
(("text/markdown", "text/x-markdown"), "markdown"),
(("application/json", "text/json", "application/javascript"), "json"),
(("text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "table"),
)
def artifact_kind(mime_type: str) -> str:
"""Coarse wire-level artifact category: file | markdown | json | table."""
lowered = (mime_type or "").lower()
for prefixes, kind in _KIND_BY_MIME:
if any(lowered.startswith(prefix) for prefix in prefixes):
return kind
return "file"
class WorkflowEventEnvelope(BaseModel):
"""Common envelope for every persisted / SSE workflow event."""
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
run_id: str = Field(alias="runId")
workflow_id: str = Field(alias="workflowId")
version_id: str = Field(alias="versionId")
seq: int = 0
event_type: WorkflowEventType = Field(alias="event")
node_id: str | None = Field(default=None, alias="nodeId")
node_run_id: str | None = Field(default=None, alias="nodeRunId")
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
data: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
def to_sse_dict(self) -> dict[str, Any]:
"""JSON object for the SSE ``data:`` line (camelCase wire fields)."""
return {
"schemaVersion": self.schema_version,
"runId": self.run_id,
"workflowId": self.workflow_id,
"versionId": self.version_id,
"seq": self.seq,
"event": self.event_type,
"nodeId": self.node_id,
"nodeRunId": self.node_run_id,
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
"data": self.data,
}
def to_sse_frame(self) -> str:
"""Full SSE frame including ``id`` / ``event`` / ``data`` lines."""
import json
payload = json.dumps(self.to_sse_dict(), ensure_ascii=False, separators=(",", ":"))
return f"id: {self.seq}\nevent: {self.event_type}\ndata: {payload}\n\n"
@runtime_checkable
class WorkflowEventSink(Protocol):
"""Persist-then-publish sink used by the future executor."""
async def emit(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> WorkflowEventEnvelope:
"""Persist event (assign ``seq``), then fan out to live subscribers."""
...
__all__ = [
"ALL_WORKFLOW_EVENT_TYPES",
"SCHEMA_VERSION",
"TERMINAL_EVENT_TYPES",
"WorkflowEventEnvelope",
"WorkflowEventSink",
"WorkflowEventType",
"artifact_kind",
]
@@ -0,0 +1,264 @@
"""Safe expression / template evaluation for workflow node configs.
Two surfaces, both intentionally tiny:
* ``render_template("… {{ nodes.a.data.title }} …", state)`` — string
interpolation over a whitelisted state tree.
* ``evaluate_condition("nodes.a.data.score > 0.5", state)`` — boolean
predicate over the same tree.
Neither uses :func:`eval`. The condition grammar is a single comparison or a
chain joined by ``and`` / ``or``; operands are either a state path or a JSON
literal. Anything else raises ``WORKFLOW_EXPRESSION_INVALID`` so a malformed
graph fails loudly at publish time instead of silently at run time.
"""
from __future__ import annotations
import json
import re
from typing import Any
from deerflow.workflows.errors import WorkflowError
_TEMPLATE_RE = re.compile(r"\{\{\s*([^{}]+?)\s*\}\}")
_ROOT_KEYS = ("inputs", "nodes", "run", "loop", "env")
_PATH_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\])*$")
_COMPARATORS = ("==", "!=", ">=", "<=", ">", "<", " in ", " contains ")
_MAX_RENDER_CHARS = 200_000
def _invalid(expr: str, reason: str) -> WorkflowError:
return WorkflowError(
"WORKFLOW_EXPRESSION_INVALID",
f"表达式不合法:{reason}",
details={"expression": expr[:200]},
)
def resolve_path(path: str, state: dict[str, Any]) -> Any:
"""Read ``a.b[0].c`` from the state tree. Missing segments yield ``None``."""
expr = path.strip()
if not _PATH_RE.match(expr):
raise _invalid(expr, "路径格式不合法")
root = expr.split(".", 1)[0].split("[", 1)[0]
if root not in _ROOT_KEYS:
raise _invalid(expr, f"根节点必须是 {'/'.join(_ROOT_KEYS)} 之一")
current: Any = state
for segment in re.findall(r"[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\]", expr):
if current is None:
return None
if segment.startswith("["):
index = int(segment[1:-1])
if not isinstance(current, (list, tuple)) or index >= len(current):
return None
current = current[index]
elif isinstance(current, dict):
current = current.get(segment)
else:
return None
return current
def _stringify(value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, (int, float)):
return str(value)
return json.dumps(value, ensure_ascii=False, default=str)
def render_template(template: str, state: dict[str, Any]) -> str:
"""Replace every ``{{ path }}`` with its stringified state value."""
if not template:
return ""
def _sub(match: re.Match[str]) -> str:
return _stringify(resolve_path(match.group(1), state))
rendered = _TEMPLATE_RE.sub(_sub, template)
if len(rendered) > _MAX_RENDER_CHARS:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
"模板渲染结果超出长度上限",
details={"limit": _MAX_RENDER_CHARS},
)
return rendered
def render_value(value: Any, state: dict[str, Any]) -> Any:
"""Recursively render templates inside dict / list / str structures.
A string that is exactly one ``{{ path }}`` keeps the resolved value's
native type instead of being stringified.
"""
if isinstance(value, str):
match = _TEMPLATE_RE.fullmatch(value.strip())
if match:
return resolve_path(match.group(1), state)
return render_template(value, state)
if isinstance(value, dict):
return {k: render_value(v, state) for k, v in value.items()}
if isinstance(value, list):
return [render_value(v, state) for v in value]
return value
def _operand(token: str, state: dict[str, Any]) -> Any:
text = token.strip()
if not text:
raise _invalid(token, "操作数为空")
match = _TEMPLATE_RE.fullmatch(text)
if match:
return resolve_path(match.group(1), state)
lowered = text.lower()
if lowered in ("true", "false"):
return lowered == "true"
if lowered in ("null", "none"):
return None
if (text[0], text[-1]) in (('"', '"'), ("'", "'")) and len(text) >= 2:
return text[1:-1]
try:
return json.loads(text)
except json.JSONDecodeError:
pass
if _PATH_RE.match(text) and text.split(".", 1)[0].split("[", 1)[0] in _ROOT_KEYS:
return resolve_path(text, state)
return text
def _truthy(value: Any) -> bool:
if isinstance(value, str):
return value.strip().lower() not in ("", "false", "0", "null", "none")
return bool(value)
def _compare(left: Any, op: str, right: Any) -> bool:
if op == "==":
return left == right
if op == "!=":
return left != right
if op == "in":
try:
return left in right # type: ignore[operator]
except TypeError:
return False
if op == "contains":
try:
return right in left # type: ignore[operator]
except TypeError:
return False
try:
if op == ">":
return left > right
if op == "<":
return left < right
if op == ">=":
return left >= right
if op == "<=":
return left <= right
except TypeError:
return False
raise _invalid(op, f"不支持的比较运算符 {op}")
def _split_top_level(expr: str, keyword: str) -> list[str] | None:
"""Split on ``keyword`` outside quotes / braces; None when absent."""
parts: list[str] = []
depth = 0
quote: str | None = None
buffer: list[str] = []
index = 0
pattern = f" {keyword} "
while index < len(expr):
char = expr[index]
if quote:
buffer.append(char)
if char == quote:
quote = None
index += 1
continue
if char in ('"', "'"):
quote = char
buffer.append(char)
index += 1
continue
if char in "[{(":
depth += 1
elif char in "]})":
depth -= 1
if depth == 0 and expr[index : index + len(pattern)] == pattern:
parts.append("".join(buffer))
buffer = []
index += len(pattern)
continue
buffer.append(char)
index += 1
if not parts:
return None
parts.append("".join(buffer))
return parts
def evaluate_condition(expression: str, state: dict[str, Any]) -> bool:
"""Evaluate a boolean predicate without ``eval``."""
expr = (expression or "").strip()
if not expr:
return False
or_parts = _split_top_level(expr, "or")
if or_parts:
return any(evaluate_condition(part, state) for part in or_parts)
and_parts = _split_top_level(expr, "and")
if and_parts:
return all(evaluate_condition(part, state) for part in and_parts)
if expr.startswith("not "):
return not evaluate_condition(expr[4:], state)
if expr.startswith("(") and expr.endswith(")"):
return evaluate_condition(expr[1:-1], state)
for comparator in _COMPARATORS:
op = comparator.strip()
index = expr.find(comparator)
if index <= 0:
continue
left = _operand(expr[:index], state)
right = _operand(expr[index + len(comparator) :], state)
return _compare(left, op, right)
return _truthy(_operand(expr, state))
_NODES_IDENT_RE = re.compile(r"(?:\{\{\s*)?nodes\.([A-Za-z_][A-Za-z0-9_\-]*)")
def collect_referenced_nodes(value: Any) -> set[str]:
"""Node ids referenced through ``nodes.<id>`` anywhere in a config tree.
Covers both ``{{ nodes.x.data }}`` templates and bare condition paths
such as ``nodes.x.data.score > 0.5``.
"""
found: set[str] = set()
if isinstance(value, str):
found.update(_NODES_IDENT_RE.findall(value))
elif isinstance(value, dict):
for item in value.values():
found |= collect_referenced_nodes(item)
elif isinstance(value, list):
for item in value:
found |= collect_referenced_nodes(item)
return found
__all__ = [
"collect_referenced_nodes",
"evaluate_condition",
"render_template",
"render_value",
"resolve_path",
]
@@ -0,0 +1,65 @@
"""Node executors for the workflow runtime."""
from __future__ import annotations
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.nodes.agents import (
AgentNodeExecutor,
SkillNodeExecutor,
SubworkflowNodeExecutor,
)
from deerflow.workflows.nodes.base import ACTIVE_PORTS, NodeExecutor, NodeRegistry
from deerflow.workflows.nodes.core import (
ConditionNodeExecutor,
LoopNodeExecutor,
MergeNodeExecutor,
OutputNodeExecutor,
StartNodeExecutor,
TransformNodeExecutor,
)
from deerflow.workflows.nodes.human import HumanInputNodeExecutor
from deerflow.workflows.nodes.integrations import (
CodeNodeExecutor,
HttpNodeExecutor,
SqlReadNodeExecutor,
)
__all__ = [
"ACTIVE_PORTS",
"AgentNodeExecutor",
"CodeNodeExecutor",
"ConditionNodeExecutor",
"HttpNodeExecutor",
"HumanInputNodeExecutor",
"LoopNodeExecutor",
"MergeNodeExecutor",
"NodeExecutor",
"NodeRegistry",
"OutputNodeExecutor",
"SkillNodeExecutor",
"SqlReadNodeExecutor",
"StartNodeExecutor",
"SubworkflowNodeExecutor",
"TransformNodeExecutor",
"build_default_registry",
]
def build_default_registry(config: WorkflowConfig | None = None) -> NodeRegistry:
"""Registry with every executable node type wired up."""
cfg = config or WorkflowConfig()
registry = NodeRegistry()
registry.register("start", StartNodeExecutor())
registry.register("output", OutputNodeExecutor())
registry.register("transform", TransformNodeExecutor())
registry.register("condition", ConditionNodeExecutor())
registry.register("merge", MergeNodeExecutor())
registry.register("loop", LoopNodeExecutor())
registry.register("human_input", HumanInputNodeExecutor())
registry.register("http", HttpNodeExecutor(cfg))
registry.register("sql_read", SqlReadNodeExecutor(cfg))
registry.register("code", CodeNodeExecutor(cfg))
registry.register("agent", AgentNodeExecutor())
registry.register("skill", SkillNodeExecutor())
registry.register("subworkflow", SubworkflowNodeExecutor())
return registry
@@ -0,0 +1,361 @@
"""LLM-backed nodes: ``agent``, ``skill``, ``subworkflow``.
These are the only nodes that call the DeerFlow LangGraph lead agent. The agent
invocation itself is injected (``deps.run_agent``) so the harness keeps its
import firewall; this module owns prompt rendering, streaming fan-out, JSON
response coercion, and error typing.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template, render_value
from deerflow.workflows.nodes.base import require_dep
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schemas import ArtifactRef, NodeResult, WorkflowNode
logger = logging.getLogger(__name__)
_JSON_BLOCK = re.compile(r"```(?:json)?\s*(.+?)\s*```", re.DOTALL)
_DELTA_FLUSH_SECONDS = 0.05
_DELTA_FLUSH_CHARS = 1024
class _DeltaCoalescer:
"""Merge model tokens into 50ms / 1KB event payloads (§9.5)."""
def __init__(self, ctx: RunContext, node_id: str) -> None:
self._ctx = ctx
self._node_id = node_id
self._parts: list[str] = []
self._chars = 0
self._task: asyncio.Task | None = None
self._lock = asyncio.Lock()
async def push(self, chunk: str) -> None:
if not chunk:
return
async with self._lock:
self._parts.append(chunk)
self._chars += len(chunk)
if self._chars >= _DELTA_FLUSH_CHARS:
await self._flush_unlocked()
return
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._delayed())
async def _delayed(self) -> None:
await asyncio.sleep(_DELTA_FLUSH_SECONDS)
async with self._lock:
await self._flush_unlocked()
async def _flush_unlocked(self) -> None:
if not self._parts:
return
text = "".join(self._parts)
self._parts.clear()
self._chars = 0
await self._ctx.emit_event(
"node.output.delta",
data={"channel": "answer", "delta": text, "text": text},
node_id=self._node_id,
)
async def flush(self) -> None:
task = self._task
if task is not None and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async with self._lock:
await self._flush_unlocked()
def _extract_json(text: str) -> Any:
candidate = text.strip()
match = _JSON_BLOCK.search(candidate)
if match:
candidate = match.group(1).strip()
else:
start = min((i for i in (candidate.find("{"), candidate.find("[")) if i >= 0), default=-1)
if start > 0:
candidate = candidate[start:]
try:
return json.loads(candidate)
except json.JSONDecodeError:
return None
_SAFE_ARG_STRING_CHARS = 160
_SAFE_ARG_MAX_KEYS = 12
def _safe_tool_args(args: Any) -> dict[str, Any]:
"""Minimal safe projection of tool-call args for the event log.
Scalars only (strings truncated), no nested containers, bounded key count —
enough for the canvas to label the step without leaking full payloads such
as entire documents, prompts or credentials that a tool may carry."""
if not isinstance(args, dict):
return {}
out: dict[str, Any] = {}
for key, value in list(args.items())[:_SAFE_ARG_MAX_KEYS]:
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
continue
if isinstance(value, str):
value = value[:_SAFE_ARG_STRING_CHARS]
out[str(key)[:64]] = value
return out
class _AgentInvocation:
"""Shared plumbing for ``agent`` and ``skill`` (agent_skill mode)."""
async def _invoke(
self,
node: WorkflowNode,
ctx: RunContext,
*,
agent_id: str,
prompt: str,
skill_names: list[str],
response_mode: str,
response_schema: dict[str, Any],
thread_mode: str,
failure_code: str,
) -> NodeResult:
run_agent = require_dep(ctx.deps.run_agent, node=node, what="智能体执行")
coalescer = _DeltaCoalescer(ctx, node.id)
async def on_delta(chunk: str) -> None:
await coalescer.push(chunk)
async def on_tool(event: dict[str, Any]) -> None:
phase = str(event.get("phase") or "started")
name = str(event.get("name") or "")
call_id = str(event.get("id") or "")
if phase == "started":
await ctx.emit_event(
"node.tool.started",
data={
"name": name,
"toolCallId": call_id,
"title": str(event.get("title") or name),
"args": _safe_tool_args(event.get("args")),
},
node_id=node.id,
)
return
ok = event.get("ok")
await ctx.emit_event(
"node.tool.finished",
data={
"name": name,
"toolCallId": call_id,
"status": "succeeded" if ok in (True, None) else "failed",
"summary": event.get("summary"),
},
node_id=node.id,
)
try:
try:
outcome = await run_agent(
run_id=ctx.run_id,
node_id=node.id,
owner_id=ctx.owner_id,
agent_id=agent_id,
prompt=prompt,
skill_names=skill_names,
thread_mode=thread_mode,
cancel=ctx.cancel,
on_delta=on_delta,
on_tool=on_tool,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
except Exception as exc: # noqa: BLE001 - agent failures are expected
logger.warning("workflow agent node %s failed", node.id, exc_info=True)
raise WorkflowError(
failure_code,
"智能体执行失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
finally:
await coalescer.flush()
text = str(outcome.get("text") or "")
data: dict[str, Any] = {"text": text}
if response_mode == "json":
payload = outcome.get("json")
if payload is None:
payload = _extract_json(text)
if payload is None:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"智能体未返回可解析的 JSON",
retryable=True,
node_id=node.id,
details={"preview": text[:500]},
)
required = response_schema.get("required") if isinstance(response_schema, dict) else None
if isinstance(required, list) and isinstance(payload, dict):
missing = [str(k) for k in required if payload.get(str(k)) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
f"智能体返回缺少字段:{'、'.join(missing)}",
retryable=True,
node_id=node.id,
details={"missing": missing},
)
data["json"] = payload
artifacts: list[ArtifactRef] = []
for raw in outcome.get("artifacts") or []:
if not isinstance(raw, dict) or not raw.get("name"):
continue
ref = await ctx.register_artifact(
node,
name=str(raw.get("name")),
mime_type=str(raw.get("mime_type") or raw.get("mimeType") or "application/octet-stream"),
path=str(raw.get("path") or ""),
size_bytes=raw.get("size_bytes") or raw.get("sizeBytes"),
preview=raw.get("preview"),
)
if ref is not None:
artifacts.append(ref)
return NodeResult(
data=data,
messages=[m for m in (outcome.get("messages") or []) if isinstance(m, dict)],
artifacts=artifacts,
metadata={
"agentId": agent_id,
"threadId": outcome.get("thread_id"),
"usage": outcome.get("usage") or {},
},
)
class AgentNodeExecutor(_AgentInvocation):
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
if not agent_id:
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点未选择智能体", node_id=node.id)
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
prompt = render_template(template, ctx.state()).strip()
if not prompt:
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点提示词为空", node_id=node.id)
schema = node.config.get("responseSchema") or node.config.get("response_schema") or {}
return await self._invoke(
node,
ctx,
agent_id=agent_id,
prompt=prompt,
skill_names=[str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])],
response_mode=str(node.config.get("responseMode") or node.config.get("response_mode") or "text"),
response_schema=schema if isinstance(schema, dict) else {},
thread_mode=str(node.config.get("threadMode") or node.config.get("thread_mode") or "isolated_per_node"),
failure_code="WORKFLOW_AGENT_FAILED",
)
class SkillNodeExecutor(_AgentInvocation):
"""``agent_skill`` runs the agent with a restricted skill set.
``callable_skill`` (direct script invocation) is not executable yet."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
mode = str(node.config.get("mode") or "agent_skill")
if mode != "agent_skill":
raise WorkflowError(
"WORKFLOW_SKILL_FAILED",
"callable_skill 模式暂未开放执行,请改用 agent_skill",
node_id=node.id,
details={"mode": mode},
)
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
skill_names = [str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])]
if not agent_id or not skill_names:
raise WorkflowError("WORKFLOW_SKILL_FAILED", "技能节点需要同时指定智能体与技能", node_id=node.id)
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
prompt = render_template(template, ctx.state()).strip() or f"请使用技能 {skill_names[0]} 完成任务。"
return await self._invoke(
node,
ctx,
agent_id=agent_id,
prompt=prompt,
skill_names=skill_names,
response_mode="text",
response_schema={},
thread_mode="isolated_per_node",
failure_code="WORKFLOW_SKILL_FAILED",
)
class SubworkflowNodeExecutor:
"""Runs a published child workflow synchronously and folds in its output."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
run_subworkflow = require_dep(ctx.deps.run_subworkflow, node=node, what="子工作流执行")
workflow_id = str(node.config.get("workflowId") or node.config.get("workflow_id") or "")
version_id = str(node.config.get("versionId") or node.config.get("version_id") or "")
if not workflow_id or not version_id:
raise WorkflowError("WORKFLOW_SUBWORKFLOW_FAILED", "子工作流节点缺少 workflowId/versionId", node_id=node.id)
mapping = node.config.get("inputMapping") or node.config.get("input_mapping") or {}
state = ctx.state()
inputs = {str(key): render_value(template, state) for key, template in mapping.items()} if isinstance(mapping, dict) else {}
await ctx.emit_event(
"node.progress",
data={"phase": "subworkflow.start", "workflowId": workflow_id, "versionId": version_id},
node_id=node.id,
)
try:
outcome = await run_subworkflow(
parent_run_id=ctx.run_id,
node_id=node.id,
owner_id=ctx.owner_id,
workflow_id=workflow_id,
version_id=version_id,
inputs=inputs,
cancel=ctx.cancel,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
except Exception as exc: # noqa: BLE001
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
"子工作流执行失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
status = str(outcome.get("status") or "completed")
if status != "completed":
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
f"子工作流未成功完成({status})",
node_id=node.id,
details={"childRunId": outcome.get("run_id"), "status": status},
)
output = outcome.get("output")
return NodeResult(
data=output if isinstance(output, dict) else {"value": output},
metadata={"childRunId": outcome.get("run_id"), "workflowId": workflow_id},
)
__all__ = ["AgentNodeExecutor", "SkillNodeExecutor", "SubworkflowNodeExecutor"]
@@ -0,0 +1,52 @@
"""Node executor protocol and registry."""
from __future__ import annotations
from typing import Any, Protocol, runtime_checkable
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.ports import ACTIVE_PORTS
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schemas import NodeResult, NodeType, WorkflowNode
@runtime_checkable
class NodeExecutor(Protocol):
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult: ...
class NodeRegistry:
"""Maps node type → executor. Unknown types fail with a typed error."""
def __init__(self) -> None:
self._executors: dict[str, NodeExecutor] = {}
def register(self, node_type: NodeType | str, executor: NodeExecutor) -> None:
self._executors[str(node_type)] = executor
def get(self, node_type: str) -> NodeExecutor:
executor = self._executors.get(node_type)
if executor is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
f"节点类型暂不支持执行:{node_type}",
details={"nodeType": node_type},
)
return executor
def supported(self) -> list[str]:
return sorted(self._executors)
def require_dep(value: Any, *, node: WorkflowNode, what: str) -> Any:
if value is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
f"运行环境缺少{what}能力,无法执行该节点",
node_id=node.id,
details={"capability": what},
)
return value
__all__ = ["ACTIVE_PORTS", "NodeExecutor", "NodeRegistry", "require_dep"]
@@ -0,0 +1,238 @@
"""Deterministic control / data nodes: start, output, transform, condition, merge, loop."""
from __future__ import annotations
import json
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import evaluate_condition, render_template, render_value
from deerflow.workflows.nodes.base import ACTIVE_PORTS
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schema_validation import schema_issues
from deerflow.workflows.schemas import NodeResult, WorkflowNode
DEFAULT_PORT = "else"
LOOP_CONTINUE_PORT = "continue"
LOOP_DONE_PORT = "done"
def _required_keys(schema: dict[str, Any]) -> list[str]:
required = schema.get("required")
if isinstance(required, list):
return [str(k) for k in required]
return []
class StartNodeExecutor:
"""Validates run inputs against the graph input schema and exposes them."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
schema = ctx.graph.input_schema or {}
missing = [key for key in _required_keys(schema) if ctx.inputs.get(key) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_INPUT_INVALID",
f"缺少必填输入:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
# Full JSON Schema pass (types / enums / nested objects) beyond the
# friendly missing-key check above.
issues = schema_issues(ctx.inputs, schema)
if issues:
raise WorkflowError(
"WORKFLOW_INPUT_INVALID",
"输入不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
node_id=node.id,
details={"errors": issues},
)
declared = node.config.get("fields")
data = dict(ctx.inputs)
if isinstance(declared, list) and declared:
names = {str(f.get("name")) for f in declared if isinstance(f, dict) and f.get("name")}
data = {k: v for k, v in data.items() if k in names} or data
return NodeResult(data=data)
class OutputNodeExecutor:
"""Projects the final run output. Empty mapping = merge all upstream data."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
mapping = node.config.get("mapping") or {}
state = ctx.state()
if isinstance(mapping, dict) and mapping:
data = {str(key): render_value(template, state) for key, template in mapping.items()}
else:
data = {}
for result in ctx.node_results.values():
payload = result.get("data")
if isinstance(payload, dict):
data.update(payload)
schema = ctx.graph.output_schema or {}
missing = [key for key in _required_keys(schema) if data.get(key) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
f"输出缺少必填字段:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
issues = schema_issues(data, schema)
if issues:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"输出不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
node_id=node.id,
details={"errors": issues},
)
return NodeResult(data=data, metadata={"isRunOutput": True})
class TransformNodeExecutor:
"""Pure data reshaping. Supported ops: set / pick / omit / json_parse / join."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
state = ctx.state()
data: dict[str, Any] = {}
template = node.config.get("template")
if isinstance(template, str) and template:
data["text"] = render_template(template, state)
for raw in node.config.get("operations") or []:
if not isinstance(raw, dict):
continue
op = str(raw.get("op") or "set")
target = str(raw.get("target") or raw.get("name") or "value")
try:
if op == "set":
data[target] = render_value(raw.get("value"), state)
elif op == "pick":
source = render_value(raw.get("source"), state)
keys = [str(k) for k in (raw.get("keys") or [])]
data[target] = {k: source.get(k) for k in keys} if isinstance(source, dict) else None
elif op == "omit":
source = render_value(raw.get("source"), state)
keys = {str(k) for k in (raw.get("keys") or [])}
data[target] = {k: v for k, v in source.items() if k not in keys} if isinstance(source, dict) else None
elif op == "json_parse":
text = render_value(raw.get("source"), state)
data[target] = json.loads(text) if isinstance(text, str) and text.strip() else None
elif op == "join":
items = render_value(raw.get("source"), state)
separator = str(raw.get("separator") or "\n")
data[target] = separator.join(str(i) for i in items) if isinstance(items, list) else ""
else:
raise WorkflowError(
"WORKFLOW_TRANSFORM_FAILED",
f"不支持的转换操作:{op}",
node_id=node.id,
details={"op": op},
)
except WorkflowError:
raise
except Exception as exc: # noqa: BLE001 - surface as typed node failure
raise WorkflowError(
"WORKFLOW_TRANSFORM_FAILED",
f"转换操作 {op} 执行失败",
node_id=node.id,
details={"op": op, "target": target, "reason": type(exc).__name__},
) from exc
return NodeResult(data=data)
class ConditionNodeExecutor:
"""Picks the first matching branch; activates only that outgoing port."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
state = ctx.state()
branches = node.config.get("branches") or []
if not isinstance(branches, list) or not branches:
raise WorkflowError("WORKFLOW_CONDITION_FAILED", "条件节点未配置分支", node_id=node.id)
evaluated: list[dict[str, Any]] = []
chosen: str | None = None
for raw in branches:
if not isinstance(raw, dict):
continue
name = str(raw.get("name") or "")
expression = str(raw.get("expression") or "")
matched = evaluate_condition(expression, state)
evaluated.append({"name": name, "matched": matched})
if matched and chosen is None:
chosen = name
if chosen is None:
chosen = str(node.config.get("defaultBranch") or node.config.get("default_branch") or DEFAULT_PORT)
return NodeResult(
data={"branch": chosen, "evaluated": evaluated},
metadata={ACTIVE_PORTS: [chosen]},
)
class MergeNodeExecutor:
"""Collects upstream results per strategy (all / first_success / any_success)."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
strategy = str(node.config.get("strategy") or "all")
sources = [str(s) for s in (node.config.get("sources") or [])]
if not sources:
sources = [edge.source for edge in ctx.graph.edges if edge.target == node.id]
collected: dict[str, Any] = {}
succeeded: list[str] = []
for source in sources:
result = ctx.node_results.get(source)
if result is None:
continue
collected[source] = result.get("data")
succeeded.append(source)
if strategy in ("first_success", "any_success") and not succeeded:
raise WorkflowError(
"WORKFLOW_MERGE_FAILED",
"合并节点没有任何成功的上游结果",
node_id=node.id,
details={"strategy": strategy, "sources": sources},
)
if strategy == "first_success":
first = succeeded[0]
return NodeResult(data={"selected": first, "value": collected[first]})
return NodeResult(data={"branches": collected, "succeeded": succeeded})
class LoopNodeExecutor:
"""Iteration gate. Activates ``continue`` (back-edge) or ``done``.
Iteration accounting lives in ``ctx.loop_state`` so the cap is enforced even
across a resume, and the engine invalidates body node results before the
next pass.
"""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
iteration = int(ctx.loop_state.get(node.id, 0))
configured = int(node.config.get("maxIterations") or node.config.get("max_iterations") or 1)
cap = min(configured, ctx.graph.settings.max_loop_iterations)
expression = str(node.config.get("continueWhen") or node.config.get("continue_when") or "")
should_continue = bool(expression) and evaluate_condition(expression, ctx.state())
next_iteration = iteration + 1
if should_continue and next_iteration >= cap:
should_continue = False
port = LOOP_CONTINUE_PORT if should_continue else LOOP_DONE_PORT
if should_continue:
ctx.loop_state[node.id] = next_iteration
return NodeResult(
data={
"iteration": iteration,
"nextIteration": next_iteration if should_continue else iteration,
"maxIterations": cap,
"continue": should_continue,
},
metadata={ACTIVE_PORTS: [port]},
)
__all__ = [
"ConditionNodeExecutor",
"LoopNodeExecutor",
"MergeNodeExecutor",
"OutputNodeExecutor",
"StartNodeExecutor",
"TransformNodeExecutor",
]
@@ -0,0 +1,77 @@
"""``human_input`` node: suspends the run and validates the resumed payload."""
from __future__ import annotations
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
from deerflow.workflows.schemas import NodeResult, WorkflowNode
DEFAULT_ACTIONS = ("submit",)
def _form_schema(node: WorkflowNode) -> dict[str, Any]:
schema = node.config.get("formSchema") or node.config.get("form_schema") or {}
return schema if isinstance(schema, dict) else {}
def _actions(node: WorkflowNode) -> list[str]:
raw = node.config.get("actions")
actions = [str(a) for a in raw] if isinstance(raw, list) and raw else list(DEFAULT_ACTIONS)
return actions
class HumanInputNodeExecutor:
"""First pass raises :class:`WorkflowPaused`; the resume pass consumes the
payload the API stored on the context and validates it against the form."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
payload = ctx.resume_payload
if not payload or str(payload.get("nodeId") or "") != node.id:
raise WorkflowPaused(
node.id,
{
"nodeId": node.id,
"name": node.name,
"prompt": render_template(str(node.config.get("prompt") or ""), ctx.state()),
"formSchema": _form_schema(node),
"actions": _actions(node),
},
)
action = str(payload.get("action") or "submit")
allowed = _actions(node)
if action not in allowed:
raise WorkflowError(
"WORKFLOW_HUMAN_INPUT_INVALID",
f"不支持的操作:{action}",
node_id=node.id,
details={"allowed": allowed},
)
values = payload.get("values")
values = values if isinstance(values, dict) else {}
schema = _form_schema(node)
required = schema.get("required")
if isinstance(required, list) and action != "cancel":
missing = [str(key) for key in required if values.get(str(key)) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_HUMAN_INPUT_INVALID",
f"缺少必填项:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
if action == "cancel":
raise WorkflowError(
"WORKFLOW_CANCELLED",
"用户在人工节点选择终止运行",
node_id=node.id,
)
# One-shot: a later human node in the same run must pause again.
ctx.resume_payload = None
return NodeResult(data={"action": action, "values": values})
__all__ = ["HumanInputNodeExecutor"]
@@ -0,0 +1,276 @@
"""External-effect nodes: ``http``, ``sql_read``, ``code``."""
from __future__ import annotations
import json
import logging
from typing import Any
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template, render_value
from deerflow.workflows.runtime.code_runner import run_python_snippet
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.runtime.sql_runner import execute_read_query
from deerflow.workflows.schemas import NodeResult, WorkflowNode
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
from deerflow.workflows.security.sql_policy import (
assert_parameters_bound,
assert_read_only,
assert_tables_allowed,
enforce_limit,
)
logger = logging.getLogger(__name__)
# Response headers safe to echo back into the event log.
_SAFE_RESPONSE_HEADERS = ("content-type", "content-length", "date", "etag", "location")
_REDACTED_REQUEST_HEADERS = ("authorization", "cookie", "x-api-key", "proxy-authorization")
class HttpNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.http
state = ctx.state()
url = render_template(str(node.config.get("url") or ""), state).strip()
if not url:
raise WorkflowError("WORKFLOW_HTTP_FAILED", "HTTP 节点缺少 URL", node_id=node.id)
method = str(node.config.get("method") or "GET").upper()
headers = {str(k): str(v) for k, v in (render_value(node.config.get("headers") or {}, state) or {}).items()}
query = render_value(node.config.get("query") or {}, state) or {}
body = render_value(node.config.get("body"), state)
credential_ref = node.config.get("credentialRef") or node.config.get("credential_ref")
if credential_ref:
headers.update(await self._credential_headers(str(credential_ref), node, ctx))
try:
target = await enforce_http_target(url, policy)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
timeout = min(
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
policy.timeout_seconds,
)
max_bytes = min(
int(node.config.get("maxResponseBytes") or node.config.get("max_response_bytes") or policy.max_response_bytes),
policy.max_response_bytes,
)
allow_redirects = bool(node.config.get("allowRedirects") or node.config.get("allow_redirects"))
max_hops = policy.max_redirects if allow_redirects else 0
await ctx.emit_event(
"node.progress",
data={
"phase": "http.request",
"method": method,
"host": target["host"],
"headers": [k for k in headers if k.lower() not in _REDACTED_REQUEST_HEADERS],
},
node_id=node.id,
)
import httpx
current_url = url
hops = 0
raw = b""
truncated = False
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
while True:
ctx.cancel.raise_if_cancelled(node.id)
try:
request = client.build_request(
method,
current_url,
headers=headers,
params=query or None,
json=body if isinstance(body, (dict, list)) else None,
content=body if isinstance(body, (str, bytes)) else None,
)
response = await client.send(request, stream=True)
except httpx.HTTPError as exc:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
"HTTP 请求失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
if response.status_code in (301, 302, 303, 307, 308) and hops < max_hops:
location = response.headers.get("location") or ""
await response.aclose()
if not location:
break
current_url = await enforce_redirect(location, current_url, policy)
hops += 1
continue
chunks: list[bytes] = []
size = 0
try:
async for chunk in response.aiter_bytes():
size += len(chunk)
if size > max_bytes:
truncated = True
break
chunks.append(chunk)
finally:
await response.aclose()
raw = b"".join(chunks)
break
text_body = raw.decode("utf-8", errors="replace")
parsed: Any = None
content_type = response.headers.get("content-type", "")
if "json" in content_type.lower():
try:
parsed = json.loads(text_body)
except json.JSONDecodeError:
parsed = None
data = {
"status": response.status_code,
"ok": 200 <= response.status_code < 300,
"headers": {k: v for k, v in response.headers.items() if k.lower() in _SAFE_RESPONSE_HEADERS},
"json": parsed,
"text": None if parsed is not None else text_body,
"truncated": truncated,
"redirects": hops,
}
if not data["ok"]:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
f"HTTP 请求返回 {response.status_code}",
retryable=response.status_code >= 500 or response.status_code == 429,
node_id=node.id,
details={"status": response.status_code, "preview": text_body[:500]},
)
return NodeResult(data=data, metadata={"host": target["host"]})
async def _credential_headers(self, ref: str, node: WorkflowNode, ctx: RunContext) -> dict[str, str]:
if ctx.deps.resolve_credential is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"运行环境未配置凭据解析能力",
node_id=node.id,
)
credential = await ctx.deps.resolve_credential(ref)
if not credential:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"引用的凭据不存在或无权访问",
node_id=node.id,
details={"credentialRef": ref},
)
headers = credential.get("headers")
return {str(k): str(v) for k, v in headers.items()} if isinstance(headers, dict) else {}
class SqlReadNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.sql
if not policy.enabled:
raise WorkflowError("WORKFLOW_SQL_POLICY_DENIED", "SQL 节点已被系统禁用", node_id=node.id)
data_source_id = str(node.config.get("dataSourceId") or node.config.get("data_source_id") or "")
if not data_source_id:
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 节点未选择数据源", node_id=node.id)
if ctx.deps.resolve_data_source is None:
raise WorkflowError("WORKFLOW_RESOURCE_MISSING", "运行环境未配置数据源解析能力", node_id=node.id)
source = await ctx.deps.resolve_data_source(data_source_id)
if not source or not source.get("dsn"):
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"数据源不存在或无权访问",
node_id=node.id,
details={"dataSourceId": data_source_id},
)
statement = assert_read_only(str(node.config.get("statement") or ""))
parameters = render_value(node.config.get("parameters") or {}, ctx.state()) or {}
if not isinstance(parameters, dict):
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 参数必须是对象", node_id=node.id)
assert_parameters_bound(statement, parameters)
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
max_rows = min(
int(node.config.get("maxRows") or node.config.get("max_rows") or policy.max_rows),
policy.max_rows,
int(source.get("max_rows") or policy.max_rows),
)
statement = enforce_limit(statement, max_rows)
await ctx.emit_event(
"node.progress",
data={"phase": "sql.query", "dataSourceId": data_source_id, "maxRows": max_rows},
node_id=node.id,
)
try:
result = await execute_read_query(
dsn=str(source["dsn"]),
statement=statement,
parameters={str(k): v for k, v in parameters.items()},
max_rows=max_rows,
timeout_seconds=policy.statement_timeout_seconds,
max_cell_chars=policy.max_cell_chars,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
return NodeResult(data=result)
class CodeNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.code
if not policy.enabled:
raise WorkflowError(
"WORKFLOW_CODE_SANDBOX_FAILED",
"代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)",
node_id=node.id,
)
source = str(node.config.get("source") or "")
if not source.strip():
raise WorkflowError("WORKFLOW_CODE_SANDBOX_FAILED", "代码节点没有源代码", node_id=node.id)
if len(source) > policy.max_source_chars:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
f"代码长度超过上限 {policy.max_source_chars}",
node_id=node.id,
)
inputs = render_value(node.config.get("inputs") or {}, ctx.state()) or {}
timeout = min(
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
policy.timeout_seconds,
)
allow_network = bool((node.config.get("allowNetwork") or node.config.get("allow_network")) and policy.allow_network)
try:
outcome = await run_python_snippet(
source=source,
inputs=inputs if isinstance(inputs, dict) else {"value": inputs},
timeout_seconds=timeout,
max_output_chars=policy.max_output_chars,
memory_limit_mb=policy.memory_limit_mb,
allow_network=allow_network,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
return NodeResult(
data=outcome["result"],
metadata={"stdout": outcome["stdout"][-4000:], "stderr": outcome["stderr"][-2000:]},
)
__all__ = ["CodeNodeExecutor", "HttpNodeExecutor", "SqlReadNodeExecutor"]
@@ -0,0 +1,17 @@
"""Leaf constants shared by the scheduler and node executors.
Kept dependency-free so the engine and the node package can both import it
without forming a cycle.
"""
from __future__ import annotations
# Metadata key a node sets to restrict which outgoing ports the engine activates.
ACTIVE_PORTS = "activePorts"
# Reserved port names.
LOOP_CONTINUE_PORT = "continue"
LOOP_DONE_PORT = "done"
CONDITION_DEFAULT_PORT = "else"
__all__ = ["ACTIVE_PORTS", "CONDITION_DEFAULT_PORT", "LOOP_CONTINUE_PORT", "LOOP_DONE_PORT"]
@@ -0,0 +1,23 @@
"""Workflow execution runtime (scheduler, context, event sink)."""
from __future__ import annotations
from deerflow.workflows.runtime.context import (
CancelToken,
RunContext,
WorkflowPaused,
WorkflowRuntimeDeps,
)
from deerflow.workflows.runtime.engine import EngineResult, WorkflowEngine
from deerflow.workflows.runtime.sink import PersistingWorkflowEventSink, envelope_from_row
__all__ = [
"CancelToken",
"EngineResult",
"PersistingWorkflowEventSink",
"RunContext",
"WorkflowEngine",
"WorkflowPaused",
"WorkflowRuntimeDeps",
"envelope_from_row",
]
@@ -0,0 +1,173 @@
"""Constrained Python execution for the ``code`` node.
The snippet runs in a **separate interpreter process** with:
* a scrubbed environment (no API keys, tokens, or DSNs inherited),
* a fresh temporary working directory that is deleted afterwards,
* an address-space cap and no core dumps (POSIX ``setrlimit``),
* sockets disabled unless the node explicitly allows network,
* a hard wall-clock timeout with process-group kill,
* bounded stdout/stderr capture.
This is process isolation, not a container: a determined snippet can still read
world-readable files on the host. That is why ``workflows.code.enabled``
defaults to ``false`` — turn it on only where the container sandbox or an OS
level jail already confines the backend process.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Any
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
_RUNNER_TEMPLATE = """
import json, os, sys
_ALLOW_NETWORK = {allow_network}
if not _ALLOW_NETWORK:
import socket
def _blocked(*_args, **_kwargs):
raise PermissionError("network access is disabled for this code node")
socket.socket = _blocked
socket.create_connection = _blocked
socket.getaddrinfo = _blocked
try:
import resource
_limit = {memory_bytes}
resource.setrlimit(resource.RLIMIT_AS, (_limit, _limit))
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
except Exception:
pass
inputs = json.loads(os.environ.get("WORKFLOW_NODE_INPUT") or "{{}}")
output = {{}}
_globals = {{"__name__": "__workflow_code__", "inputs": inputs, "output": output}}
with open({source_path!r}, "r", encoding="utf-8") as handle:
_source = handle.read()
exec(compile(_source, "<workflow-code>", "exec"), _globals)
_result = _globals.get("output")
if not isinstance(_result, dict):
_result = {{"value": _result}}
sys.stdout.write("\\n__WORKFLOW_RESULT__" + json.dumps(_result, ensure_ascii=False, default=str))
"""
def _safe_env() -> dict[str, str]:
env = {
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
"LANG": os.environ.get("LANG", "C.UTF-8"),
"PYTHONIOENCODING": "utf-8",
"PYTHONDONTWRITEBYTECODE": "1",
"HOME": "",
}
if sys.platform == "win32":
for key in ("SYSTEMROOT", "COMSPEC", "TEMP", "TMP"):
if os.environ.get(key):
env[key] = os.environ[key]
return env
async def run_python_snippet(
*,
source: str,
inputs: dict[str, Any],
timeout_seconds: int,
max_output_chars: int,
memory_limit_mb: int,
allow_network: bool,
) -> dict[str, Any]:
workdir = Path(tempfile.mkdtemp(prefix="wf-code-"))
try:
source_path = workdir / "snippet.py"
source_path.write_text(source, encoding="utf-8")
runner_path = workdir / "runner.py"
runner_path.write_text(
_RUNNER_TEMPLATE.format(
allow_network=bool(allow_network),
memory_bytes=max(64, int(memory_limit_mb)) * 1024 * 1024,
source_path=str(source_path),
),
encoding="utf-8",
)
env = _safe_env()
env["WORKFLOW_NODE_INPUT"] = json.dumps(inputs, ensure_ascii=False, default=str)
process = await asyncio.create_subprocess_exec(
sys.executable,
"-I",
"-S",
str(runner_path),
cwd=str(workdir),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=sys.platform != "win32",
)
try:
stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
except TimeoutError as exc:
_terminate(process)
raise WorkflowError(
"WORKFLOW_TIMEOUT",
f"代码节点执行超时({timeout_seconds}s)",
details={"timeoutSeconds": timeout_seconds},
) from exc
stdout = stdout_bytes.decode("utf-8", errors="replace")
stderr = stderr_bytes.decode("utf-8", errors="replace")[:max_output_chars]
if process.returncode != 0:
raise WorkflowError(
"WORKFLOW_CODE_SANDBOX_FAILED",
"代码节点执行失败",
details={"exitCode": process.returncode, "stderr": stderr[-2000:]},
)
marker = "__WORKFLOW_RESULT__"
index = stdout.rfind(marker)
result: dict[str, Any] = {}
logs = stdout
if index >= 0:
logs = stdout[:index]
try:
parsed = json.loads(stdout[index + len(marker) :])
result = parsed if isinstance(parsed, dict) else {"value": parsed}
except json.JSONDecodeError:
result = {}
return {
"result": result,
"stdout": logs[:max_output_chars],
"stderr": stderr,
}
finally:
shutil.rmtree(workdir, ignore_errors=True)
def _terminate(process: asyncio.subprocess.Process) -> None:
try:
if sys.platform != "win32":
os.killpg(os.getpgid(process.pid), 9)
else:
process.kill()
except (ProcessLookupError, PermissionError, OSError):
logger.debug("code node process already gone", exc_info=True)
__all__ = ["run_python_snippet"]
@@ -0,0 +1,162 @@
"""Run-scoped execution context shared by the engine and node executors."""
from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.events import WorkflowEventType
from deerflow.workflows.schemas import ArtifactRef, WorkflowGraph, WorkflowNode
EmitFn = Callable[..., Awaitable[Any]]
def _public_artifact_payload(ref: ArtifactRef) -> dict[str, Any]:
"""Wire payload for ``artifact.created``.
The sandbox/host ``path`` stays internal: clients address artifacts by id
through the run-scoped content endpoints, never by filesystem path."""
from deerflow.workflows.events import artifact_kind
return {
"artifactId": ref.artifact_id,
"name": ref.name,
"kind": artifact_kind(ref.mime_type),
"mimeType": ref.mime_type,
"sizeBytes": ref.size_bytes,
"preview": ref.preview,
}
class WorkflowPaused(Exception): # noqa: N818 - control-flow signal, not an error
"""Raised by ``human_input`` to suspend the run until a resume arrives."""
def __init__(self, node_id: str, pending_input: dict[str, Any]) -> None:
super().__init__(f"workflow paused at {node_id}")
self.node_id = node_id
self.pending_input = pending_input
class CancelToken:
"""Cooperative cancellation shared between the executor and node code."""
def __init__(self) -> None:
self._event = asyncio.Event()
self._reason = "cancelled"
def cancel(self, reason: str = "cancelled") -> None:
self._reason = reason
self._event.set()
@property
def cancelled(self) -> bool:
return self._event.is_set()
async def wait(self) -> None:
await self._event.wait()
def raise_if_cancelled(self, node_id: str | None = None) -> None:
if self._event.is_set():
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
@dataclass
class WorkflowRuntimeDeps:
"""Injected capabilities. The harness never imports the app layer, so every
integration point (agents, HTTP egress, SQL, sandbox, subworkflows) arrives
as an async callable supplied by the caller. ``None`` means "not available",
which node executors turn into a typed ``WORKFLOW_RESOURCE_MISSING``."""
run_agent: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_http: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_sql: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_code: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_subworkflow: Callable[..., Awaitable[dict[str, Any]]] | None = None
save_artifact: Callable[..., Awaitable[dict[str, Any]]] | None = None
resolve_credential: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
resolve_data_source: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
@dataclass
class RunContext:
"""Everything a node executor is allowed to see."""
run_id: str
workflow_id: str
version_id: str
owner_id: str
graph: WorkflowGraph
inputs: dict[str, Any]
deps: WorkflowRuntimeDeps
cancel: CancelToken
emit: EmitFn
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
loop_state: dict[str, int] = field(default_factory=dict)
resume_payload: dict[str, Any] | None = None
node_run_ids: dict[str, str] = field(default_factory=dict)
current_node_id: str | None = None
env: dict[str, Any] = field(default_factory=dict)
def state(self) -> dict[str, Any]:
"""Whitelisted tree visible to expressions. Credentials never appear."""
return {
"inputs": self.inputs,
"nodes": self.node_results,
"run": {"id": self.run_id, "workflowId": self.workflow_id, "versionId": self.version_id},
"loop": dict(self.loop_state),
"env": self.env,
}
async def emit_event(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
) -> None:
node = node_id or self.current_node_id
await self.emit(
event_type,
data=data or {},
node_id=node,
node_run_id=self.node_run_ids.get(node) if node else None,
)
async def register_artifact(
self,
node: WorkflowNode,
*,
name: str,
mime_type: str = "application/octet-stream",
path: str = "",
size_bytes: int | None = None,
preview: str | None = None,
) -> ArtifactRef | None:
"""Persist an artifact reference and emit ``artifact.created``."""
if self.deps.save_artifact is None:
return None
row = await self.deps.save_artifact(
run_id=self.run_id,
node_id=node.id,
name=name,
mime_type=mime_type,
path=path,
size_bytes=size_bytes,
preview=preview,
)
ref = ArtifactRef(
artifactId=str(row.get("id") or ""),
name=str(row.get("name") or name),
mimeType=str(row.get("mime_type") or mime_type),
path=str(row.get("path") or path),
sizeBytes=row.get("size_bytes"),
preview=row.get("preview"),
)
await self.emit_event("artifact.created", data=_public_artifact_payload(ref), node_id=node.id)
return ref
__all__ = ["CancelToken", "RunContext", "WorkflowPaused", "WorkflowRuntimeDeps"]
@@ -0,0 +1,531 @@
"""Persisted DAG scheduler for workflow runs.
Scheduling model
----------------
Every edge carries a state: ``pending`` → ``active`` (its source chose this
port) or ``pruned`` (the source chose a different port). A node is *ready* when
none of its non-back incoming edges is ``pending`` and at least one is
``active``; a node whose incoming edges all resolved to ``pruned`` is *skipped*
and prunes its own outgoing edges in turn. That single rule gives us fan-out
parallelism, condition branching, and merge joins without a topological
pre-pass, and it terminates because each superstep resolves at least one edge.
Loops are the one cycle we allow: the back edge (``loop`` node, port
``continue``) is excluded from readiness gating, and activating it clears the
body's results so the next iteration re-runs them. Every re-entry is capped by
``settings.max_loop_iterations``.
Durability: node results are written to the run store as they complete, so a
resumed or re-leased run replays only what is missing. ``human_input`` raises
:class:`WorkflowPaused`, which the caller turns into ``awaiting_input``.
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any, Protocol
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.ports import ACTIVE_PORTS, LOOP_CONTINUE_PORT
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
from deerflow.workflows.schemas import NodeResult, WorkflowEdge, WorkflowGraph, WorkflowNode
class ExecutorRegistry(Protocol):
"""Structural view of ``deerflow.workflows.nodes.NodeRegistry``.
Declared here rather than imported so the scheduler has no dependency on
the node package (which depends on the runtime context).
"""
def get(self, node_type: str) -> Any: ...
logger = logging.getLogger(__name__)
PENDING = "pending"
ACTIVE = "active"
PRUNED = "pruned"
@dataclass
class EngineResult:
output: dict[str, Any] = field(default_factory=dict)
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
loop_state: dict[str, int] = field(default_factory=dict)
steps: int = 0
def _is_back_edge(edge: WorkflowEdge, nodes: dict[str, WorkflowNode]) -> bool:
source = nodes.get(edge.source)
return bool(source and source.type == "loop" and (edge.source_port or "") == LOOP_CONTINUE_PORT)
def _node_started_event_data(node: WorkflowNode, node_attempt: int) -> dict[str, Any]:
"""Safe presentation metadata for workflow clients.
This stays intentionally small: UI clients need stable agent identity and
an optional explicit parallel group, but never prompt templates, tools,
credentials, or the full node config. A missing group is valid; clients
may still use their backwards-compatible name-based presentation.
"""
data: dict[str, Any] = {
"nodeType": node.type,
"nodeKind": node.type,
"name": node.name,
"attempt": node_attempt,
}
group_id = node.config.get("parallelGroupId") or node.config.get("parallel_group_id")
if isinstance(group_id, str) and group_id.strip():
data["parallelGroupId"] = group_id.strip()
round_index = node.config.get("roundIndex")
if round_index is None:
round_index = node.config.get("round_index")
if isinstance(round_index, int) and round_index >= 0:
data["roundIndex"] = round_index
if node.type == "agent":
agent_id = node.config.get("agentId") or node.config.get("agent_id")
if isinstance(agent_id, str) and agent_id.strip():
agent: dict[str, str] = {
"agentId": agent_id.strip(),
"name": str(
node.config.get("agentName")
or node.config.get("agent_name")
or node.name
or agent_id
),
}
avatar_url = node.config.get("agentAvatarUrl") or node.config.get("agent_avatar_url")
if isinstance(avatar_url, str) and avatar_url.strip():
agent["avatarUrl"] = avatar_url.strip()
data["agent"] = agent
return data
class WorkflowEngine:
def __init__(self, registry: ExecutorRegistry, *, run_store: WorkflowRunStore | None = None) -> None:
self._registry = registry
self._store = run_store
async def run(
self,
ctx: RunContext,
*,
preloaded: dict[str, dict[str, Any]] | None = None,
loop_state: dict[str, int] | None = None,
attempt: int = 1,
) -> EngineResult:
graph: WorkflowGraph = ctx.graph
nodes = graph.node_map()
outgoing: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
incoming: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
for edge in graph.edges:
outgoing[edge.source].append(edge)
incoming[edge.target].append(edge)
edge_state: dict[str, str] = {edge.id: PENDING for edge in graph.edges}
done: set[str] = set()
skipped: set[str] = set()
ctx.loop_state.update(loop_state or {})
if preloaded:
ctx.node_results.update(preloaded)
done |= set(preloaded)
# Re-derive edge states from the ports each replayed node selected.
for node_id in list(done):
self._apply_ports(
node_id,
ctx.node_results.get(node_id, {}).get("metadata") or {},
outgoing,
edge_state,
)
semaphore = asyncio.Semaphore(max(1, graph.settings.max_parallelism))
steps = 0
max_steps = max(1, graph.settings.max_steps)
while True:
ctx.cancel.raise_if_cancelled()
self._propagate_skips(nodes, incoming, outgoing, edge_state, done, skipped)
ready = [node_id for node_id in nodes if node_id not in done and node_id not in skipped and self._readiness(node_id, incoming, edge_state, nodes) == "ready"]
if not ready:
break
steps += len(ready)
if steps > max_steps:
raise WorkflowError(
"WORKFLOW_MAX_STEPS",
f"运行步数超过上限 {max_steps}",
details={"maxSteps": max_steps},
)
results = await asyncio.gather(
*(self._run_one(nodes[node_id], ctx, semaphore, attempt) for node_id in ready),
return_exceptions=True,
)
paused: WorkflowPaused | None = None
failure: BaseException | None = None
for node_id, outcome in zip(ready, results, strict=True):
if isinstance(outcome, WorkflowPaused):
paused = paused or outcome
continue
if isinstance(outcome, BaseException):
failure = failure or outcome
continue
ctx.node_results[node_id] = outcome
done.add(node_id)
metadata = outcome.get("metadata") or {}
self._apply_ports(node_id, metadata, outgoing, edge_state)
if nodes[node_id].type == "loop":
await self._maybe_reset_loop_body(nodes[node_id], ctx, nodes, outgoing, edge_state, done, skipped)
if failure is not None:
raise failure
if paused is not None:
raise paused
return EngineResult(
output=self._collect_output(nodes, ctx),
node_results=dict(ctx.node_results),
loop_state=dict(ctx.loop_state),
steps=steps,
)
# ── scheduling helpers ──────────────────────────────────────────────
def _readiness(
self,
node_id: str,
incoming: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
nodes: dict[str, WorkflowNode],
) -> str:
edges = incoming.get(node_id) or []
if not edges:
return "ready"
back = [e for e in edges if _is_back_edge(e, nodes)]
if any(edge_state[e.id] == ACTIVE for e in back):
return "ready"
normal = [e for e in edges if not _is_back_edge(e, nodes)]
if not normal:
return "blocked"
if any(edge_state[e.id] == PENDING for e in normal):
return "blocked"
if any(edge_state[e.id] == ACTIVE for e in normal):
return "ready"
return "skip"
def _apply_ports(
self,
node_id: str,
metadata: dict[str, Any],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
) -> None:
active_ports = metadata.get(ACTIVE_PORTS)
allowed = {str(p) for p in active_ports} if isinstance(active_ports, list) else None
for edge in outgoing.get(node_id) or []:
port = edge.source_port or ""
if allowed is None or not port or port in allowed:
edge_state[edge.id] = ACTIVE
else:
edge_state[edge.id] = PRUNED
def _propagate_skips(
self,
nodes: dict[str, WorkflowNode],
incoming: dict[str, list[WorkflowEdge]],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
done: set[str],
skipped: set[str],
) -> None:
changed = True
while changed:
changed = False
for node_id in nodes:
if node_id in done or node_id in skipped:
continue
if self._readiness(node_id, incoming, edge_state, nodes) != "skip":
continue
skipped.add(node_id)
for edge in outgoing.get(node_id) or []:
edge_state[edge.id] = PRUNED
changed = True
async def _maybe_reset_loop_body(
self,
loop_node: WorkflowNode,
ctx: RunContext,
nodes: dict[str, WorkflowNode],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
done: set[str],
skipped: set[str],
) -> None:
back_edges = [e for e in outgoing.get(loop_node.id) or [] if _is_back_edge(e, nodes) and edge_state[e.id] == ACTIVE]
if not back_edges:
return
body_entry = str(loop_node.config.get("bodyEntry") or loop_node.config.get("body_entry") or "")
if not body_entry or body_entry not in nodes:
raise WorkflowError(
"WORKFLOW_LOOP_FAILED",
"循环节点的 bodyEntry 不存在",
node_id=loop_node.id,
details={"bodyEntry": body_entry},
)
body = self._loop_body(body_entry, loop_node.id, outgoing, nodes)
for node_id in body:
done.discard(node_id)
skipped.discard(node_id)
ctx.node_results.pop(node_id, None)
for edge in outgoing.get(node_id) or []:
if not _is_back_edge(edge, nodes):
edge_state[edge.id] = PENDING
# The back edge stays ACTIVE so the body entry is reachable again; the
# loop node prunes it itself once it selects the ``done`` port.
if self._store is not None and body:
try:
await self._store.invalidate_node_runs(ctx.run_id, sorted(body))
except Exception: # noqa: BLE001 - bookkeeping must not kill the run
logger.warning("failed to invalidate node runs for loop reset", exc_info=True)
await ctx.emit_event(
"node.progress",
data={
"phase": "loop.iteration",
"iteration": ctx.loop_state.get(loop_node.id, 0),
"bodyEntry": body_entry,
},
node_id=loop_node.id,
)
def _loop_body(
self,
body_entry: str,
loop_id: str,
outgoing: dict[str, list[WorkflowEdge]],
nodes: dict[str, WorkflowNode],
) -> set[str]:
"""Nodes reachable from ``body_entry`` up to (and including) the loop node."""
body: set[str] = set()
stack = [body_entry]
while stack:
current = stack.pop()
if current in body:
continue
body.add(current)
if current == loop_id:
continue
for edge in outgoing.get(current) or []:
if not _is_back_edge(edge, nodes):
stack.append(edge.target)
return body
# ── node execution ──────────────────────────────────────────────────
async def _run_one(
self,
node: WorkflowNode,
ctx: RunContext,
semaphore: asyncio.Semaphore,
attempt: int,
) -> dict[str, Any]:
async with semaphore:
ctx.cancel.raise_if_cancelled(node.id)
retry_cfg = node.config.get("retry") if isinstance(node.config.get("retry"), dict) else {}
max_attempts = max(1, min(5, int(retry_cfg.get("maxAttempts") or 1)))
backoff = max(0.0, float(retry_cfg.get("backoffSeconds") or 0.5))
on_error = str(node.config.get("onError") or "fail")
timeout = self._node_timeout(node, ctx)
last_error: WorkflowError | None = None
for node_attempt in range(1, max_attempts + 1):
node_run = await self._begin_node_run(ctx, node, attempt, node_attempt)
started = time.monotonic()
await ctx.emit_event(
"node.started",
data=_node_started_event_data(node, node_attempt),
node_id=node.id,
)
try:
executor = self._registry.get(node.type)
result: NodeResult = await asyncio.wait_for(executor.execute(node, ctx), timeout=timeout)
except WorkflowPaused:
await self._finish_node_run(ctx, node_run, status="queued", output=None)
raise
except TimeoutError as exc:
last_error = WorkflowError(
"WORKFLOW_TIMEOUT",
f"节点执行超时({timeout}s)",
retryable=True,
node_id=node.id,
details={"timeoutSeconds": timeout},
)
await self._fail_node_run(ctx, node_run, last_error, started)
if node_attempt < max_attempts:
await asyncio.sleep(backoff * node_attempt)
continue
if on_error == "continue":
return self._error_result(last_error)
raise last_error from exc
except WorkflowError as exc:
last_error = exc
exc.node_id = exc.node_id or node.id
await self._fail_node_run(ctx, node_run, exc, started)
if exc.retryable and node_attempt < max_attempts:
await asyncio.sleep(backoff * node_attempt)
continue
if on_error == "continue":
return self._error_result(exc)
raise
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - unexpected node bug
logger.exception("workflow node %s crashed", node.id)
last_error = WorkflowError(
"WORKFLOW_INTERNAL",
"节点执行发生内部错误",
node_id=node.id,
details={"reason": type(exc).__name__},
)
await self._fail_node_run(ctx, node_run, last_error, started)
if on_error == "continue":
return self._error_result(last_error)
raise last_error from exc
payload = result.model_dump(by_alias=False)
duration = int((time.monotonic() - started) * 1000)
await self._finish_node_run(ctx, node_run, status="completed", output=payload, duration_ms=duration)
await ctx.emit_event(
"node.completed",
data={
"nodeType": node.type,
"durationMs": duration,
"data": payload.get("data"),
"warnings": payload.get("warnings") or [],
},
node_id=node.id,
)
return payload
raise last_error or WorkflowError("WORKFLOW_INTERNAL", "节点执行失败", node_id=node.id)
def _node_timeout(self, node: WorkflowNode, ctx: RunContext) -> int:
configured = node.config.get("timeoutSeconds") or node.config.get("timeout_seconds")
graph_limit = ctx.graph.settings.node_timeout_seconds
try:
value = int(configured) if configured else graph_limit
except (TypeError, ValueError):
value = graph_limit
return max(1, min(value, graph_limit))
def _error_result(self, error: WorkflowError) -> dict[str, Any]:
return {
"data": {},
"messages": [],
"artifacts": [],
"metadata": {"failed": True, "error": error.to_body().model_dump(by_alias=True)},
"warnings": [error.message],
}
async def _begin_node_run(self, ctx: RunContext, node: WorkflowNode, run_attempt: int, node_attempt: int) -> dict[str, Any] | None:
if self._store is None:
return None
try:
row = await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node.id,
"node_type": node.type,
"status": "running",
"attempt": node_attempt,
"iteration": int(ctx.loop_state.get(node.id, 0)),
"started_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run start for %s", node.id, exc_info=True)
return None
ctx.node_run_ids[node.id] = str(row.get("id") or "")
return row
async def _finish_node_run(
self,
ctx: RunContext,
node_run: dict[str, Any] | None,
*,
status: str,
output: dict[str, Any] | None,
duration_ms: int | None = None,
) -> None:
if self._store is None or node_run is None:
return
try:
await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node_run["node_id"],
"node_type": node_run["node_type"],
"attempt": node_run["attempt"],
"iteration": node_run["iteration"],
"status": status,
"output": output,
"duration_ms": duration_ms,
"finished_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run finish", exc_info=True)
async def _fail_node_run(
self,
ctx: RunContext,
node_run: dict[str, Any] | None,
error: WorkflowError,
started: float,
) -> None:
duration = int((time.monotonic() - started) * 1000)
body = error.to_body().model_dump(by_alias=True)
if self._store is not None and node_run is not None:
try:
await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node_run["node_id"],
"node_type": node_run["node_type"],
"attempt": node_run["attempt"],
"iteration": node_run["iteration"],
"status": "failed",
"error": body,
"duration_ms": duration,
"finished_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run failure", exc_info=True)
await ctx.emit_event(
"node.failed",
data={"error": body, "durationMs": duration},
node_id=error.node_id,
)
def _collect_output(self, nodes: dict[str, WorkflowNode], ctx: RunContext) -> dict[str, Any]:
output: dict[str, Any] = {}
for node_id, node in nodes.items():
if node.type != "output":
continue
result = ctx.node_results.get(node_id)
if isinstance(result, dict) and isinstance(result.get("data"), dict):
output.update(result["data"])
return output
__all__ = ["EngineResult", "WorkflowEngine"]
@@ -0,0 +1,117 @@
"""Persist-then-publish event sink for workflow runs."""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.workflows.events import WorkflowEventEnvelope, WorkflowEventType
logger = logging.getLogger(__name__)
PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
class PersistingWorkflowEventSink:
"""Assigns ``seq`` in the database first, then fans out to live listeners.
A persistence failure never aborts the run, but the event is **dropped
rather than published**: SSE clients tail by cursor and rebuild from the
durable log, so a live-only frame with a made-up seq would render once and
then vanish on reconnect (a "ghost event"). Callers can see the drop via
``degraded_count`` / the ``eventPersistenceDegraded`` marker on the returned
envelope; the stream endpoint synthesises a closing frame from the run row
when a terminal event could not be logged.
"""
def __init__(
self,
store: WorkflowEventStore,
*,
run_id: str,
workflow_id: str,
version_id: str,
publish: PublishFn | None = None,
) -> None:
self._store = store
self._run_id = run_id
self._workflow_id = workflow_id
self._version_id = version_id
self._publish = publish
self.degraded_count = 0
async def emit(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> WorkflowEventEnvelope:
payload = data or {}
seq = 0
degraded = False
try:
row = await self._store.append(
run_id=self._run_id,
workflow_id=self._workflow_id,
version_id=self._version_id,
event_type=event_type,
payload=payload,
node_id=node_id,
node_run_id=node_run_id,
)
seq = int(row.get("seq") or 0)
except Exception as exc: # noqa: BLE001 - the run must survive a log outage
logger.warning(
"workflow event not persisted (dropped, not published): run=%s event=%s node=%s error=%s:%s",
self._run_id,
event_type,
node_id,
type(exc).__name__,
exc,
)
self.degraded_count += 1
degraded = True
envelope = WorkflowEventEnvelope(
runId=self._run_id,
workflowId=self._workflow_id,
versionId=self._version_id,
seq=seq,
event=event_type,
nodeId=node_id,
nodeRunId=node_run_id,
timestamp=datetime.now(UTC),
data={**payload, "eventPersistenceDegraded": True} if degraded else payload,
)
if degraded:
return envelope
if self._publish is not None:
try:
await self._publish(envelope.to_sse_dict())
except Exception: # noqa: BLE001 - live fan-out is best effort
logger.debug("workflow live publish failed", exc_info=True)
return envelope
def envelope_from_row(row: dict[str, Any]) -> dict[str, Any]:
"""Turn a stored event row into the SSE wire object (replay path)."""
return {
"schemaVersion": "1.0",
"runId": row.get("run_id"),
"workflowId": row.get("workflow_id"),
"versionId": row.get("version_id"),
"seq": int(row.get("seq") or 0),
"event": row.get("event_type"),
"nodeId": row.get("node_id"),
"nodeRunId": row.get("node_run_id"),
"timestamp": row.get("created_at"),
"data": row.get("payload") or {},
}
__all__ = ["PersistingWorkflowEventSink", "envelope_from_row"]
@@ -0,0 +1,107 @@
"""Read-only query execution for the ``sql_read`` node.
Engines are cached per DSN with a tiny pool because a workflow typically hits
the same reporting database many times in one run. Values are bound as
parameters and every cell is truncated before it can reach an event payload.
"""
from __future__ import annotations
import asyncio
import logging
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
_engines: dict[str, AsyncEngine] = {}
_lock = asyncio.Lock()
async def _get_engine(dsn: str) -> AsyncEngine:
async with _lock:
engine = _engines.get(dsn)
if engine is None:
engine = create_async_engine(
dsn,
pool_pre_ping=True,
pool_size=2,
max_overflow=2,
pool_recycle=1800,
)
_engines[dsn] = engine
return engine
async def dispose_engines() -> None:
async with _lock:
for engine in _engines.values():
await engine.dispose()
_engines.clear()
def _coerce(value: Any, max_chars: int) -> Any:
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, Decimal):
return float(value)
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, (bytes, bytearray, memoryview)):
return f"<binary {len(bytes(value))} bytes>"
text_value = str(value)
return text_value if len(text_value) <= max_chars else text_value[:max_chars] + "…"
async def execute_read_query(
*,
dsn: str,
statement: str,
parameters: dict[str, Any],
max_rows: int,
timeout_seconds: int,
max_cell_chars: int = 4000,
) -> dict[str, Any]:
engine = await _get_engine(dsn)
async def _run() -> dict[str, Any]:
async with engine.connect() as conn:
result = await conn.execute(text(statement), parameters)
columns = list(result.keys())
rows = result.fetchmany(max_rows)
truncated = bool(result.fetchone())
return {
"columns": columns,
"rows": [{col: _coerce(val, max_cell_chars) for col, val in zip(columns, row, strict=False)} for row in rows],
"rowCount": len(rows),
"truncated": truncated,
}
try:
return await asyncio.wait_for(_run(), timeout=timeout_seconds)
except TimeoutError as exc:
raise WorkflowError(
"WORKFLOW_TIMEOUT",
f"数据库查询超时({timeout_seconds}s)",
retryable=True,
details={"timeoutSeconds": timeout_seconds},
) from exc
except WorkflowError:
raise
except Exception as exc: # noqa: BLE001 - never leak DSN/credentials upward
logger.warning("workflow sql_read failed", exc_info=True)
raise WorkflowError(
"WORKFLOW_SQL_FAILED",
"数据库查询执行失败",
retryable=True,
details={"reason": type(exc).__name__},
) from exc
__all__ = ["dispose_engines", "execute_read_query"]
@@ -0,0 +1,54 @@
"""Load frozen workflow sample graphs shipped with the harness."""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from deerflow.workflows.schemas import WorkflowGraph
_SAMPLES_DIR = Path(__file__).resolve().parent
_NODES_DIR = _SAMPLES_DIR / "nodes"
def samples_dir() -> Path:
return _SAMPLES_DIR
def nodes_dir() -> Path:
return _NODES_DIR
@lru_cache(maxsize=32)
def load_sample_graph(name: str = "report_generation") -> WorkflowGraph:
"""Load and validate a sample graph by stem name (without ``.json``)."""
path = _SAMPLES_DIR / f"{name}.json"
if not path.is_file():
raise FileNotFoundError(f"workflow sample not found: {path}")
raw = json.loads(path.read_text(encoding="utf-8"))
return WorkflowGraph.model_validate(raw)
@lru_cache(maxsize=64)
def load_node_sample(node_type: str) -> dict:
"""Load a minimal single-node fixture (raw dict, not a full graph)."""
path = _NODES_DIR / f"{node_type}.json"
if not path.is_file():
raise FileNotFoundError(f"node sample not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def list_node_samples() -> list[str]:
if not _NODES_DIR.is_dir():
return []
return sorted(p.stem for p in _NODES_DIR.glob("*.json"))
__all__ = [
"list_node_samples",
"load_node_sample",
"load_sample_graph",
"nodes_dir",
"samples_dir",
]
@@ -0,0 +1,15 @@
{
"schemaVersion": "1.0",
"runId": "run_01",
"workflowId": "wf_report_generation",
"versionId": "wv_03",
"seq": 42,
"event": "node.output.delta",
"nodeId": "agent_write",
"nodeRunId": "nr_09",
"timestamp": "2026-08-26T10:00:00+08:00",
"data": {
"channel": "answer",
"delta": "第一段内容"
}
}
@@ -0,0 +1,13 @@
{
"id": "agent_1",
"type": "agent",
"name": "智能体",
"config": {
"agentId": "report-writer",
"promptTemplate": "根据材料撰写:{{ nodes.merge_1.data }}",
"responseMode": "text",
"threadMode": "isolated_per_node",
"timeoutSeconds": 600,
"skillNames": []
}
}
@@ -0,0 +1,11 @@
{
"id": "code_1",
"type": "code",
"name": "代码(沙箱)",
"config": {
"language": "python",
"source": "def main(inputs):\n rows = inputs.get(\"rows\") or []\n return {\"count\": len(rows), \"titles\": [r.get(\"title\") for r in rows[:10]]}\n",
"timeoutSeconds": 60,
"allowNetwork": false
}
}
@@ -0,0 +1,18 @@
{
"id": "cond_1",
"type": "condition",
"name": "条件",
"config": {
"branches": [
{
"name": "passed",
"expression": "{{ nodes.agent_review.data.passed }} == true"
},
{
"name": "rejected",
"expression": "{{ nodes.agent_review.data.passed }} == false"
}
],
"defaultBranch": "rejected"
}
}
@@ -0,0 +1,16 @@
{
"id": "http_1",
"type": "http",
"name": "HTTP",
"config": {
"method": "GET",
"url": "https://api.example.internal/items/{{ inputs.taskId }}",
"headers": { "Accept": "application/json" },
"query": {},
"body": null,
"credentialRef": "cred_task_api",
"timeoutSeconds": 30,
"maxResponseBytes": 1048576,
"allowRedirects": false
}
}
@@ -0,0 +1,20 @@
{
"id": "human_1",
"type": "human_input",
"name": "人工介入",
"config": {
"formSchema": {
"type": "object",
"required": ["decision"],
"properties": {
"decision": {
"type": "string",
"enum": ["approve", "revise"]
},
"comment": { "type": "string" }
}
},
"prompt": "请确认是否放行当前报告草稿。",
"actions": ["approve", "revise"]
}
}
@@ -0,0 +1,10 @@
{
"id": "loop_1",
"type": "loop",
"name": "受限循环",
"config": {
"maxIterations": 2,
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
"bodyEntry": "agent_write"
}
}
@@ -0,0 +1,9 @@
{
"id": "merge_1",
"type": "merge",
"name": "合并",
"config": {
"strategy": "all",
"sources": ["sql_1", "http_1", "agent_collect"]
}
}
@@ -0,0 +1,11 @@
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": {
"reportMarkdown": "{{ nodes.agent_write.data.answer }}"
},
"llmRepair": false
}
}
@@ -0,0 +1,12 @@
{
"id": "skill_1",
"type": "skill",
"name": "技能(agent_skill)",
"config": {
"mode": "agent_skill",
"agentId": "report-collector",
"skillNames": ["web-research"],
"promptTemplate": "使用指定技能完成:{{ inputs.topic }}",
"timeoutSeconds": 300
}
}
@@ -0,0 +1,13 @@
{
"id": "sql_1",
"type": "sql_read",
"name": "SQL 只读",
"config": {
"dataSourceId": "ds_report_readonly",
"statement": "SELECT id, title FROM report_source WHERE task_id = :task_id",
"parameters": {
"task_id": "{{ inputs.taskId }}"
},
"maxRows": 1000
}
}
@@ -0,0 +1,6 @@
{
"id": "start_1",
"type": "start",
"name": "开始",
"config": {}
}
@@ -0,0 +1,13 @@
{
"id": "sub_1",
"type": "subworkflow",
"name": "子工作流",
"config": {
"workflowId": "wf_collect_pack",
"versionId": "wv_01",
"inputMapping": {
"topic": "{{ inputs.topic }}",
"taskId": "{{ inputs.taskId }}"
}
}
}
@@ -0,0 +1,20 @@
{
"id": "transform_1",
"type": "transform",
"name": "模板/转换",
"config": {
"operations": [
{
"op": "map",
"from": "{{ nodes.sql_1.data.rows }}",
"to": "rows"
},
{
"op": "template",
"template": "# {{ inputs.topic }}\n\n共 {{ nodes.sql_1.metadata.rowCount }} 行。",
"to": "markdown"
}
],
"template": null
}
}
@@ -0,0 +1,260 @@
{
"schemaVersion": "1.0",
"id": "wf_report_generation",
"name": "专题报告生成",
"description": "阶段0验收基线:并行取数 → 合并 → 整理 → 撰写 → 审核 → 条件退回/输出",
"inputSchema": {
"type": "object",
"required": ["topic", "taskId"],
"properties": {
"topic": { "type": "string", "description": "报告主题" },
"taskId": { "type": "string", "description": "业务任务 ID" },
"dateRange": {
"type": "object",
"properties": {
"start": { "type": "string" },
"end": { "type": "string" }
}
}
}
},
"outputSchema": {
"type": "object",
"required": ["reportMarkdown", "passed"],
"properties": {
"reportMarkdown": { "type": "string" },
"passed": { "type": "boolean" },
"artifacts": { "type": "array" },
"issues": { "type": "array" }
}
},
"nodes": [
{
"id": "start_1",
"type": "start",
"name": "开始",
"config": {}
},
{
"id": "sql_1",
"type": "sql_read",
"name": "查询内部结构化数据",
"config": {
"dataSourceId": "ds_report_readonly",
"statement": "SELECT id, title, content, updated_at FROM report_source WHERE task_id = :task_id ORDER BY updated_at DESC",
"parameters": {
"task_id": "{{ inputs.taskId }}"
},
"maxRows": 1000
}
},
{
"id": "http_1",
"type": "http",
"name": "调用只读业务接口",
"config": {
"method": "GET",
"url": "https://api.example.internal/tasks/{{ inputs.taskId }}/summary",
"headers": {
"Accept": "application/json"
},
"credentialRef": "cred_task_api",
"timeoutSeconds": 30,
"maxResponseBytes": 524288,
"allowRedirects": false
}
},
{
"id": "agent_collect",
"type": "agent",
"name": "外部材料采集",
"config": {
"agentId": "report-collector",
"promptTemplate": "围绕主题「{{ inputs.topic }}」搜集外部公开材料,任务 ID:{{ inputs.taskId }}。输出 JSON:{sources:[], notes:string}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["sources", "notes"],
"properties": {
"sources": { "type": "array" },
"notes": { "type": "string" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600,
"skillNames": ["web-research"]
}
},
{
"id": "merge_1",
"type": "merge",
"name": "合并材料并保留来源",
"config": {
"strategy": "all",
"sources": ["sql_1", "http_1", "agent_collect"]
}
},
{
"id": "agent_clean",
"type": "agent",
"name": "数据整理与事实去重",
"config": {
"agentId": "report-cleaner",
"promptTemplate": "整理并去重以下材料,输出严格 JSON:{{ nodes.merge_1.data }}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["facts", "sources"],
"properties": {
"facts": { "type": "array" },
"sources": { "type": "array" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600
}
},
{
"id": "agent_write",
"type": "agent",
"name": "撰写 Markdown 报告",
"config": {
"agentId": "report-writer",
"promptTemplate": "根据整理结果撰写专题报告(Markdown)。主题:{{ inputs.topic }}\n材料:{{ nodes.agent_clean.data }}\n若存在审核意见:{{ nodes.agent_review.data.revisedAdvice }}",
"responseMode": "text",
"threadMode": "isolated_per_node",
"timeoutSeconds": 900
}
},
{
"id": "agent_review",
"type": "agent",
"name": "审核报告",
"config": {
"agentId": "report-reviewer",
"promptTemplate": "审核以下报告,输出 JSON {passed:boolean, issues:array, revisedAdvice:string}:\n{{ nodes.agent_write.data.answer }}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["passed", "issues", "revisedAdvice"],
"properties": {
"passed": { "type": "boolean" },
"issues": { "type": "array" },
"revisedAdvice": { "type": "string" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600
}
},
{
"id": "cond_review",
"type": "condition",
"name": "审核结果分支",
"config": {
"branches": [
{
"name": "passed",
"expression": "{{ nodes.agent_review.data.passed }} == true"
},
{
"name": "rejected",
"expression": "{{ nodes.agent_review.data.passed }} == false"
}
],
"defaultBranch": "rejected"
}
},
{
"id": "loop_revise",
"type": "loop",
"name": "退回修改(最多 2 次)",
"config": {
"maxIterations": 2,
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
"bodyEntry": "agent_write"
}
},
{
"id": "transform_pack",
"type": "transform",
"name": "组装最终字段",
"config": {
"operations": [
{
"op": "map",
"from": "{{ nodes.agent_write.data.answer }}",
"to": "reportMarkdown"
},
{
"op": "map",
"from": "{{ nodes.agent_review.data.passed }}",
"to": "passed"
},
{
"op": "map",
"from": "{{ nodes.agent_review.data.issues }}",
"to": "issues"
}
]
}
},
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": {
"reportMarkdown": "{{ nodes.transform_pack.data.reportMarkdown }}",
"passed": "{{ nodes.transform_pack.data.passed }}",
"issues": "{{ nodes.transform_pack.data.issues }}"
},
"llmRepair": false
}
}
],
"edges": [
{ "id": "e_start_sql", "source": "start_1", "target": "sql_1" },
{ "id": "e_start_http", "source": "start_1", "target": "http_1" },
{ "id": "e_start_collect", "source": "start_1", "target": "agent_collect" },
{ "id": "e_sql_merge", "source": "sql_1", "target": "merge_1" },
{ "id": "e_http_merge", "source": "http_1", "target": "merge_1" },
{ "id": "e_collect_merge", "source": "agent_collect", "target": "merge_1" },
{ "id": "e_merge_clean", "source": "merge_1", "target": "agent_clean" },
{ "id": "e_clean_write", "source": "agent_clean", "target": "agent_write" },
{ "id": "e_write_review", "source": "agent_write", "target": "agent_review" },
{ "id": "e_review_cond", "source": "agent_review", "target": "cond_review" },
{
"id": "e_cond_pass",
"source": "cond_review",
"target": "transform_pack",
"sourcePort": "passed"
},
{
"id": "e_cond_reject",
"source": "cond_review",
"target": "loop_revise",
"sourcePort": "rejected"
},
{
"id": "e_loop_write",
"source": "loop_revise",
"target": "agent_write",
"sourcePort": "continue"
},
{
"id": "e_loop_fail",
"source": "loop_revise",
"target": "transform_pack",
"sourcePort": "exhausted"
},
{ "id": "e_pack_out", "source": "transform_pack", "target": "output_1" }
],
"settings": {
"runTimeoutSeconds": 1800,
"nodeTimeoutSeconds": 300,
"maxSteps": 200,
"maxLoopIterations": 5,
"maxParallelism": 4
}
}
@@ -0,0 +1,41 @@
{
"schemaVersion": "1.0",
"id": "wf_start_output",
"name": "空图:开始→输出",
"description": "阶段2运行骨架用的最小可执行图",
"inputSchema": {
"type": "object",
"properties": {
"message": { "type": "string" }
}
},
"outputSchema": {
"type": "object",
"required": ["message"],
"properties": {
"message": { "type": "string" }
}
},
"nodes": [
{ "id": "start_1", "type": "start", "name": "开始", "config": {} },
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": { "message": "{{ inputs.message }}" },
"llmRepair": false
}
}
],
"edges": [
{ "id": "e1", "source": "start_1", "target": "output_1" }
],
"settings": {
"runTimeoutSeconds": 60,
"nodeTimeoutSeconds": 30,
"maxSteps": 10,
"maxLoopIterations": 1,
"maxParallelism": 1
}
}
@@ -0,0 +1,73 @@
"""Full JSON Schema validation for workflow input/output values.
The engine used to check only ``required`` keys; everything else (wrong type,
unknown enum value, nested ``properties``/``items`` violations) slipped through
until downstream nodes exploded with confusing errors. This module runs the
graph's ``inputSchema`` / ``outputSchema`` through ``Draft202012Validator`` and
returns structured, frontend-friendly errors (path + message) instead of
leaking Python tracebacks.
Kept as pure functions so both the node executors (``WorkflowError``) and the
HTTP router (400 responses) can share the exact same rules.
"""
from __future__ import annotations
from typing import Any
from jsonschema import Draft202012Validator
# Cap the reported errors so a wildly invalid payload cannot produce a
# multi-megabyte error body.
_MAX_ERRORS = 20
def schema_issues(value: Any, schema: dict[str, Any] | None) -> list[dict[str, Any]]:
"""Validate ``value`` against ``schema``.
Returns ``[]`` when the schema is empty/absent (nothing declared = nothing
enforced). Each issue is ``{"path": [...], "message": str}`` with ``path``
pointing at the offending field, e.g. ``["options", 0, "name"]``.
"""
if not isinstance(schema, dict) or not schema:
return []
try:
validator = Draft202012Validator(schema)
# Construction alone does not check the schema — malformed keywords
# (e.g. a non-string ``pattern``) only explode during validation.
errors = sorted(validator.iter_errors(value), key=lambda err: list(err.absolute_path))
except Exception: # noqa: BLE001 - a malformed user schema must not 500
return [{"path": [], "message": "schema 定义本身无效,无法用于校验"}]
out: list[dict[str, Any]] = []
seen: set[tuple[Any, ...]] = set()
for err in errors:
path = [str(p) if not isinstance(p, int) else p for p in err.absolute_path]
key = tuple(path)
if key in seen:
continue
seen.add(key)
out.append({"path": path, "message": _humanize(err)})
if len(out) >= _MAX_ERRORS:
out.append({"path": [], "message": f"……其余问题已省略(共 {len(errors)} 处)"})
break
return out
def _humanize(err: Any) -> str:
"""Map the common jsonschema phrasings to short Chinese messages."""
message = str(err.message)
validator = getattr(err, "validator", "")
path = ".".join(str(p) for p in err.absolute_path) or "输入"
if validator == "required":
missing = message.split("'")[1] if "'" in message else "?"
return f"{path} 缺少必填字段:{missing}"
if validator == "enum":
return f"{path} 的值不在允许范围内:{message}"
if validator == "type":
return f"{path} 类型不正确:{message}"
if validator == "additionalProperties":
return f"{path} 含有未声明的额外字段:{message}"
return f"{path} 不符合 schema:{message}"
__all__ = ["schema_issues"]
@@ -0,0 +1,374 @@
"""Frozen workflow graph schema v1.0 (internal DTO).
Coze canvas DTO is converted into this structure at the Gateway boundary.
The runtime must never depend on Coze-private fields.
Wire JSON uses camelCase aliases; Python code uses snake_case.
"""
from __future__ import annotations
from hashlib import sha256
from typing import Any, Literal
from pydantic import BaseModel, Field, field_validator, model_validator
SCHEMA_VERSION = "1.0"
NodeType = Literal[
"start",
"output",
"agent",
"skill",
"http",
"sql_read",
"code",
"transform",
"condition",
"merge",
"human_input",
"subworkflow",
"loop",
]
ALL_NODE_TYPES: tuple[NodeType, ...] = (
"start",
"output",
"agent",
"skill",
"http",
"sql_read",
"code",
"transform",
"condition",
"merge",
"human_input",
"subworkflow",
"loop",
)
RunStatus = Literal[
"queued",
"running",
"awaiting_input",
"completed",
"failed",
"cancel_requested",
"cancelled",
]
ALL_RUN_STATUSES: tuple[RunStatus, ...] = (
"queued",
"running",
"awaiting_input",
"completed",
"failed",
"cancel_requested",
"cancelled",
)
TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset({"completed", "failed", "cancelled"})
NodeRunStatus = Literal[
"queued",
"running",
"completed",
"failed",
"skipped",
"cancelled",
]
SkillMode = Literal["agent_skill", "callable_skill"]
MergeStrategy = Literal["all", "first_success", "any_success"]
HttpMethod = Literal["GET", "POST", "PUT", "PATCH", "DELETE"]
AgentResponseMode = Literal["text", "json"]
AgentThreadMode = Literal["isolated_per_node", "shared_run_thread"]
ExecutionMode = Literal["normal"]
class WorkflowGraphSettings(BaseModel):
"""Per-graph resource limits (still capped by system limits at publish)."""
run_timeout_seconds: int = Field(default=1800, ge=1, alias="runTimeoutSeconds")
node_timeout_seconds: int = Field(default=300, ge=1, alias="nodeTimeoutSeconds")
max_steps: int = Field(default=200, ge=1, alias="maxSteps")
max_loop_iterations: int = Field(default=5, ge=1, alias="maxLoopIterations")
max_parallelism: int = Field(default=4, ge=1, alias="maxParallelism")
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowNode(BaseModel):
"""A single node on the internal standard graph."""
id: str = Field(min_length=1)
type: NodeType
name: str = ""
config: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
@field_validator("id")
@classmethod
def _id_not_blank(cls, value: str) -> str:
stripped = value.strip()
if not stripped:
raise ValueError("node id must be non-empty")
return stripped
class WorkflowEdge(BaseModel):
"""Directed edge. ``source_port`` names a condition / loop outlet when set."""
id: str = Field(min_length=1)
source: str
target: str
source_port: str | None = Field(default=None, alias="sourcePort")
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowGraph(BaseModel):
"""Internal standard workflow graph (publish / run snapshot)."""
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
id: str = Field(min_length=1)
name: str = ""
description: str = ""
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
nodes: list[WorkflowNode] = Field(default_factory=list)
edges: list[WorkflowEdge] = Field(default_factory=list)
settings: WorkflowGraphSettings = Field(default_factory=WorkflowGraphSettings)
model_config = {"extra": "forbid", "populate_by_name": True}
@model_validator(mode="after")
def _unique_ids(self) -> WorkflowGraph:
node_ids = [n.id for n in self.nodes]
if len(node_ids) != len(set(node_ids)):
raise ValueError("duplicate node id")
edge_ids = [e.id for e in self.edges]
if len(edge_ids) != len(set(edge_ids)):
raise ValueError("duplicate edge id")
known = set(node_ids)
for edge in self.edges:
if edge.source not in known:
raise ValueError(f"edge {edge.id} source not found: {edge.source}")
if edge.target not in known:
raise ValueError(f"edge {edge.id} target not found: {edge.target}")
return self
def node_map(self) -> dict[str, WorkflowNode]:
return {node.id: node for node in self.nodes}
def canonical_json(self) -> str:
"""Deterministic JSON used for ``graph_hash`` (sorted keys, no whitespace)."""
return self.model_dump_json(by_alias=True, exclude_none=True)
def graph_hash(self) -> str:
return sha256(self.canonical_json().encode("utf-8")).hexdigest()
class ArtifactRef(BaseModel):
"""Reference to a large/binary result; full bytes live outside events."""
artifact_id: str = Field(alias="artifactId")
name: str
mime_type: str = Field(default="application/octet-stream", alias="mimeType")
path: str = ""
size_bytes: int | None = Field(default=None, alias="sizeBytes")
preview: str | None = None
model_config = {"extra": "forbid", "populate_by_name": True}
class NodeResult(BaseModel):
"""Uniform result returned by every node executor."""
data: dict[str, Any] = Field(default_factory=dict)
messages: list[dict[str, Any]] = Field(default_factory=list)
artifacts: list[ArtifactRef] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
warnings: list[str] = Field(default_factory=list)
model_config = {"extra": "forbid", "populate_by_name": True}
class StartRunRequest(BaseModel):
"""Body for ``POST /api/workflows/{workflow_id}/runs``."""
version_id: str = Field(alias="versionId")
inputs: dict[str, Any] = Field(default_factory=dict)
idempotency_key: str | None = Field(default=None, alias="idempotencyKey")
execution_mode: ExecutionMode = Field(default="normal", alias="executionMode")
model_config = {"extra": "forbid", "populate_by_name": True}
class ResumeRunRequest(BaseModel):
"""Body for ``POST /api/workflows/runs/{run_id}/resume``."""
resume_token: str = Field(alias="resumeToken")
action: str
payload: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
# ── Typed node config helpers (optional; validators may use later) ───────────
class AgentNodeConfig(BaseModel):
agent_id: str = Field(alias="agentId")
prompt_template: str = Field(default="", alias="promptTemplate")
response_mode: AgentResponseMode = Field(default="text", alias="responseMode")
response_schema: dict[str, Any] = Field(default_factory=dict, alias="responseSchema")
thread_mode: AgentThreadMode = Field(default="isolated_per_node", alias="threadMode")
timeout_seconds: int = Field(default=600, ge=1, alias="timeoutSeconds")
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
model_config = {"extra": "forbid", "populate_by_name": True}
class SkillNodeConfig(BaseModel):
mode: SkillMode = "agent_skill"
agent_id: str | None = Field(default=None, alias="agentId")
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
prompt_template: str = Field(default="", alias="promptTemplate")
# callable_skill fields (phase 4+; accepted in schema, not executed in phase 0)
entry: str | None = None
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
timeout_seconds: int = Field(default=300, ge=1, alias="timeoutSeconds")
model_config = {"extra": "forbid", "populate_by_name": True}
class HttpNodeConfig(BaseModel):
method: HttpMethod = "GET"
url: str
headers: dict[str, str] = Field(default_factory=dict)
query: dict[str, Any] = Field(default_factory=dict)
body: Any = None
credential_ref: str | None = Field(default=None, alias="credentialRef")
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
max_response_bytes: int = Field(default=1_048_576, ge=1, alias="maxResponseBytes")
allow_redirects: bool = Field(default=False, alias="allowRedirects")
model_config = {"extra": "forbid", "populate_by_name": True}
class SqlReadNodeConfig(BaseModel):
data_source_id: str = Field(alias="dataSourceId")
statement: str
parameters: dict[str, Any] = Field(default_factory=dict)
max_rows: int = Field(default=1000, ge=1, alias="maxRows")
model_config = {"extra": "forbid", "populate_by_name": True}
class CodeNodeConfig(BaseModel):
language: Literal["python"] = "python"
source: str
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
allow_network: bool = Field(default=False, alias="allowNetwork")
model_config = {"extra": "forbid", "populate_by_name": True}
class TransformNodeConfig(BaseModel):
operations: list[dict[str, Any]] = Field(default_factory=list)
template: str | None = None
model_config = {"extra": "forbid", "populate_by_name": True}
class ConditionBranch(BaseModel):
name: str
expression: str
model_config = {"extra": "forbid", "populate_by_name": True}
class ConditionNodeConfig(BaseModel):
branches: list[ConditionBranch] = Field(default_factory=list)
default_branch: str | None = Field(default=None, alias="defaultBranch")
model_config = {"extra": "forbid", "populate_by_name": True}
class MergeNodeConfig(BaseModel):
strategy: MergeStrategy = "all"
sources: list[str] = Field(default_factory=list)
model_config = {"extra": "forbid", "populate_by_name": True}
class HumanInputNodeConfig(BaseModel):
form_schema: dict[str, Any] = Field(default_factory=dict, alias="formSchema")
prompt: str = ""
actions: list[str] = Field(default_factory=lambda: ["submit"])
model_config = {"extra": "forbid", "populate_by_name": True}
class SubworkflowNodeConfig(BaseModel):
workflow_id: str = Field(alias="workflowId")
version_id: str = Field(alias="versionId")
input_mapping: dict[str, str] = Field(default_factory=dict, alias="inputMapping")
model_config = {"extra": "forbid", "populate_by_name": True}
class LoopNodeConfig(BaseModel):
max_iterations: int = Field(default=5, ge=1, alias="maxIterations")
continue_when: str = Field(default="", alias="continueWhen")
body_entry: str = Field(alias="bodyEntry")
model_config = {"extra": "forbid", "populate_by_name": True}
class OutputNodeConfig(BaseModel):
mapping: dict[str, str] = Field(default_factory=dict)
llm_repair: bool = Field(default=False, alias="llmRepair")
model_config = {"extra": "forbid", "populate_by_name": True}
__all__ = [
"SCHEMA_VERSION",
"ALL_NODE_TYPES",
"ALL_RUN_STATUSES",
"TERMINAL_RUN_STATUSES",
"AgentNodeConfig",
"AgentResponseMode",
"AgentThreadMode",
"ArtifactRef",
"CodeNodeConfig",
"ConditionBranch",
"ConditionNodeConfig",
"ExecutionMode",
"HttpMethod",
"HttpNodeConfig",
"HumanInputNodeConfig",
"LoopNodeConfig",
"MergeNodeConfig",
"MergeStrategy",
"NodeResult",
"NodeRunStatus",
"NodeType",
"OutputNodeConfig",
"ResumeRunRequest",
"RunStatus",
"SkillMode",
"SkillNodeConfig",
"SqlReadNodeConfig",
"StartRunRequest",
"SubworkflowNodeConfig",
"TransformNodeConfig",
"WorkflowEdge",
"WorkflowGraph",
"WorkflowGraphSettings",
"WorkflowNode",
]
@@ -0,0 +1,18 @@
"""Security policies for external-effect workflow nodes."""
from __future__ import annotations
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
from deerflow.workflows.security.sql_policy import (
assert_parameters_bound,
assert_read_only,
enforce_limit,
)
__all__ = [
"assert_parameters_bound",
"assert_read_only",
"enforce_http_target",
"enforce_limit",
"enforce_redirect",
]
@@ -0,0 +1,114 @@
"""Egress policy for the ``http`` node (SSRF defence).
Checks, in order: scheme → host allow/deny lists → port → every resolved IP
address. Resolving the host and validating *all* A/AAAA answers is what stops
`http://internal.example.com` from pointing at `169.254.169.254` or `127.0.0.1`.
Known limitation: we validate the resolved addresses and then let httpx dial the
hostname again, so a DNS answer that changes between the two lookups is not
covered (classic DNS-rebinding TOCTOU). Closing that fully requires dialing the
pinned IP with SNI/Host overrides; for now the deployment guidance is to keep
``allow_private_networks`` false and prefer an explicit ``allowed_hosts`` list.
"""
from __future__ import annotations
import asyncio
import ipaddress
import socket
from urllib.parse import urlparse, urlsplit
from deerflow.config.workflow_config import WorkflowHttpPolicyConfig
from deerflow.workflows.errors import WorkflowError
_ALLOWED_SCHEMES = ("http", "https")
_DEFAULT_PORTS = {"http": 80, "https": 443}
# Cloud metadata endpoints get their own explicit deny so a permissive
# intranet configuration cannot accidentally expose instance credentials.
_METADATA_HOSTS = frozenset({"169.254.169.254", "metadata.google.internal", "metadata"})
def _denied(message: str, **details: object) -> WorkflowError:
return WorkflowError("WORKFLOW_HTTP_POLICY_DENIED", message, details=dict(details))
def _host_matches(host: str, patterns: list[str]) -> bool:
target = host.lower().rstrip(".")
for raw in patterns:
pattern = str(raw).strip().lower().rstrip(".")
if not pattern:
continue
if pattern.startswith("."):
if target == pattern[1:] or target.endswith(pattern):
return True
elif pattern.startswith("*."):
if target == pattern[2:] or target.endswith(pattern[1:]):
return True
elif target == pattern:
return True
return False
def _is_private(address: str) -> bool:
try:
ip = ipaddress.ip_address(address)
except ValueError:
return False
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified
async def _resolve(host: str, port: int) -> list[str]:
loop = asyncio.get_running_loop()
try:
infos = await loop.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
"无法解析目标域名",
retryable=True,
details={"host": host, "reason": type(exc).__name__},
) from exc
return sorted({str(info[4][0]) for info in infos})
async def enforce_http_target(url: str, policy: WorkflowHttpPolicyConfig) -> dict[str, object]:
"""Validate a target URL. Returns the resolved metadata or raises."""
if not policy.enabled:
raise _denied("HTTP 节点已被系统禁用")
parsed = urlparse(url)
scheme = (parsed.scheme or "").lower()
if scheme not in _ALLOWED_SCHEMES:
raise _denied("仅允许 http/https 协议", scheme=scheme or "")
host = (parsed.hostname or "").lower()
if not host:
raise _denied("URL 缺少主机名")
port = parsed.port or _DEFAULT_PORTS[scheme]
if host in _METADATA_HOSTS:
raise _denied("目标为云元数据地址,已拒绝", host=host)
if _host_matches(host, policy.blocked_hosts):
raise _denied("目标主机在禁止名单中", host=host)
if policy.allowed_hosts and not _host_matches(host, policy.allowed_hosts):
raise _denied("目标主机不在允许名单中", host=host)
if policy.allowed_ports and port not in policy.allowed_ports:
raise _denied("目标端口不被允许", port=port)
addresses = await _resolve(host, port)
if not policy.allow_private_networks:
private = [addr for addr in addresses if _is_private(addr)]
if private:
raise _denied("目标解析到内网/环回地址,已拒绝", host=host)
if any(addr in _METADATA_HOSTS for addr in addresses):
raise _denied("目标解析到云元数据地址,已拒绝", host=host)
return {"host": host, "port": port, "scheme": scheme, "addresses": addresses}
async def enforce_redirect(location: str, base_url: str, policy: WorkflowHttpPolicyConfig) -> str:
"""Validate a redirect hop, resolving relative locations against the base."""
if location.startswith("/"):
base = urlsplit(base_url)
location = f"{base.scheme}://{base.netloc}{location}"
await enforce_http_target(location, policy)
return location
__all__ = ["enforce_http_target", "enforce_redirect"]
@@ -0,0 +1,97 @@
"""Envelope encryption for workflow secrets (data-source DSNs, credentials).
The key comes from ``WORKFLOW_SECRET_KEY`` (or ``DEERFLOW_SECRET_KEY``) and is
stretched with SHA-256 into a Fernet key, so operators can supply any
sufficiently long passphrase.
When no key is configured we refuse to encrypt rather than silently storing
plaintext: the caller decides whether to reject the write or fall back. That
choice is deliberate — a silent plaintext fallback is how credentials end up in
database backups without anyone noticing.
"""
from __future__ import annotations
import base64
import hashlib
import logging
import os
logger = logging.getLogger(__name__)
_ENV_KEYS = ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY")
_PREFIX = "enc:v1:"
class SecretKeyMissingError(RuntimeError):
def __init__(self) -> None:
super().__init__("未配置 WORKFLOW_SECRET_KEY,无法加密保存敏感连接信息")
def _fernet():
key_material = ""
for name in _ENV_KEYS:
value = os.environ.get(name)
if value and value.strip():
key_material = value.strip()
break
if not key_material:
raise SecretKeyMissingError()
try:
from cryptography.fernet import Fernet
except ImportError as exc: # pragma: no cover - dependency is declared
raise SecretKeyMissingError() from exc
digest = hashlib.sha256(key_material.encode("utf-8")).digest()
return Fernet(base64.urlsafe_b64encode(digest))
def secrets_available() -> bool:
try:
_fernet()
except Exception: # noqa: BLE001 - any failure means "cannot encrypt"
return False
return True
def encrypt_secret(plaintext: str) -> str:
token = _fernet().encrypt(plaintext.encode("utf-8")).decode("ascii")
return f"{_PREFIX}{token}"
def decrypt_secret(stored: str) -> str:
if not stored:
return ""
if not stored.startswith(_PREFIX):
# Legacy / externally seeded plaintext row.
return stored
return _fernet().decrypt(stored[len(_PREFIX) :].encode("ascii")).decode("utf-8")
def is_encrypted(stored: str) -> bool:
return bool(stored) and stored.startswith(_PREFIX)
def mask_dsn(dsn: str) -> str:
"""Human-readable summary that never reveals user/password/host details."""
if not dsn:
return ""
scheme, _, rest = dsn.partition("://")
if not rest:
return f"{scheme[:24]}://***"
host_part = rest.split("@")[-1]
host = host_part.split("/")[0]
database = host_part.split("/", 1)[1].split("?")[0] if "/" in host_part else ""
masked_host = host.split(":")[0]
if len(masked_host) > 4:
masked_host = f"{masked_host[:2]}***{masked_host[-2:]}"
return f"{scheme}://***@{masked_host}/{database}"
__all__ = [
"SecretKeyMissingError",
"decrypt_secret",
"encrypt_secret",
"is_encrypted",
"mask_dsn",
"secrets_available",
]
@@ -0,0 +1,136 @@
"""Read-only statement policy for the ``sql_read`` node.
The node is *only* allowed to run a single read query. We reject anything that
is not a `SELECT`/`WITH`, anything containing a statement separator, and any
write/DDL/administrative keyword — after stripping comments, because
``SELECT 1 -- ; DROP TABLE t`` and ``/*!*/`` style tricks are exactly how these
filters get bypassed.
Parameters are always bound (never interpolated), so this policy is a second
line of defence rather than the only one. Production deployments should also
point the data source at a read-only database account.
"""
from __future__ import annotations
import re
from deerflow.workflows.errors import WorkflowError
_FORBIDDEN_KEYWORDS = (
"insert",
"update",
"delete",
"drop",
"alter",
"create",
"truncate",
"replace into",
"grant",
"revoke",
"merge",
"call",
"execute",
"exec",
"copy",
"load_file",
"outfile",
"dumpfile",
"into outfile",
"pg_read_file",
"pg_sleep",
"sleep",
"benchmark",
"shutdown",
"vacuum",
"attach",
"detach",
"pragma",
"set ",
"lock",
"do ",
)
_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
_LINE_COMMENT = re.compile(r"(--|#)[^\n]*")
_PARAM_RE = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)")
def _denied(message: str, **details: object) -> WorkflowError:
return WorkflowError("WORKFLOW_SQL_POLICY_DENIED", message, details=dict(details))
def strip_comments(statement: str) -> str:
return _LINE_COMMENT.sub(" ", _BLOCK_COMMENT.sub(" ", statement))
def assert_read_only(statement: str) -> str:
"""Validate and return the normalised single-statement query."""
raw = (statement or "").strip()
if not raw:
raise _denied("SQL 语句为空")
cleaned = strip_comments(raw).strip()
if not cleaned:
raise _denied("SQL 语句仅包含注释")
body = cleaned.rstrip().rstrip(";").strip()
if ";" in body:
raise _denied("只允许执行单条语句")
normalized = re.sub(r"\s+", " ", body.lower()).strip()
if not (normalized.startswith("select") or normalized.startswith("with")):
raise _denied("只允许 SELECT / WITH 查询")
for keyword in _FORBIDDEN_KEYWORDS:
token = keyword.strip()
if re.search(rf"\b{re.escape(token)}\b", normalized):
raise _denied("语句包含被禁止的关键字", keyword=token)
return body
def enforce_limit(statement: str, max_rows: int) -> str:
"""Append a LIMIT when the query does not already bound its result size."""
lowered = statement.lower()
if re.search(r"\blimit\b", lowered) or re.search(r"\bfetch\s+first\b", lowered):
return statement
return f"{statement} LIMIT {int(max_rows)}"
_FROM_JOIN_RE = re.compile(r"\b(?:from|join)\s+([A-Za-z_][\w.]*)", re.IGNORECASE)
def _table_name(raw: str) -> str:
return raw.split(".")[-1].strip("`\"[]").lower()
def assert_tables_allowed(statement: str, allowed_tables: list[str]) -> None:
"""When the data source has a table allowlist, every FROM/JOIN target must
appear on it. An empty allowlist means no extra restriction."""
if not allowed_tables:
return
allowed = {item.lower() for item in allowed_tables} | {_table_name(item) for item in allowed_tables}
for name in _FROM_JOIN_RE.findall(strip_comments(statement)):
short = _table_name(name)
if short not in allowed and name.lower() not in allowed:
raise _denied("查询引用了未授权的表", table=name, allowedTables=list(allowed_tables))
def declared_parameters(statement: str) -> set[str]:
return set(_PARAM_RE.findall(strip_comments(statement)))
def assert_parameters_bound(statement: str, parameters: dict[str, object]) -> None:
missing = sorted(declared_parameters(statement) - set(parameters))
if missing:
raise WorkflowError(
"WORKFLOW_SQL_FAILED",
f"缺少查询参数:{'、'.join(missing)}",
details={"missing": missing},
)
__all__ = [
"assert_parameters_bound",
"assert_read_only",
"assert_tables_allowed",
"declared_parameters",
"enforce_limit",
"strip_comments",
]
@@ -0,0 +1,335 @@
"""Publish-time structural validator for workflow graphs (phase 1).
Full resource-permission and SQL EXPLAIN checks land in later phases.
This module never imports ``app.*``.
"""
from __future__ import annotations
from collections import defaultdict, deque
from collections.abc import Callable
from typing import Any
from pydantic import ValidationError
from deerflow.workflows.errors import WorkflowError, WorkflowErrorBody
from deerflow.workflows.expressions import collect_referenced_nodes
from deerflow.workflows.schemas import ALL_NODE_TYPES, WorkflowGraph
ResourceChecker = Callable[[str, dict[str, Any]], "str | dict[str, Any] | None"]
"""Return an error message when a node resource is missing/forbidden.
May instead return ``{"message": ..., "field": "config.agentId"}`` so API
responses can point the canvas form at the exact offending field. ``None``
means the node's resources check out."""
class ValidationIssue(WorkflowErrorBody):
"""Alias kept for readability in API responses."""
def _collect_reachable(start_ids: set[str], adjacency: dict[str, list[str]]) -> set[str]:
seen: set[str] = set()
queue: deque[str] = deque(start_ids)
while queue:
node_id = queue.popleft()
if node_id in seen:
continue
seen.add(node_id)
for nxt in adjacency.get(node_id, []):
if nxt not in seen:
queue.append(nxt)
return seen
def _has_uncontrolled_cycle(graph: WorkflowGraph) -> bool:
"""Detect cycles that are not entered via an explicit ``loop`` node.
Phase-1 heuristic: if the condensation of the graph (ignoring edges whose
source is a ``loop`` node) still has a back-edge, reject.
"""
loop_ids = {n.id for n in graph.nodes if n.type == "loop"}
adjacency: dict[str, list[str]] = defaultdict(list)
for edge in graph.edges:
if edge.source in loop_ids:
continue
adjacency[edge.source].append(edge.target)
visiting: set[str] = set()
visited: set[str] = set()
def dfs(node_id: str) -> bool:
visiting.add(node_id)
for nxt in adjacency.get(node_id, []):
if nxt in visiting:
return True
if nxt not in visited and dfs(nxt):
return True
visiting.discard(node_id)
visited.add(node_id)
return False
for node in graph.nodes:
if node.id not in visited and dfs(node.id):
return True
return False
def validate_workflow_graph(
raw: dict[str, Any] | WorkflowGraph,
*,
system_max_steps: int = 500,
system_max_loop_iterations: int = 20,
system_max_parallelism: int = 16,
resource_checker: ResourceChecker | None = None,
) -> list[ValidationIssue]:
"""Validate a graph. Empty list means publishable."""
issues: list[ValidationIssue] = []
try:
graph = raw if isinstance(raw, WorkflowGraph) else WorkflowGraph.model_validate(raw)
except ValidationError as exc:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="图结构不符合 schema v1.0",
details={"errors": exc.errors()},
)
)
return issues
starts = [n for n in graph.nodes if n.type == "start"]
outputs = [n for n in graph.nodes if n.type == "output"]
if len(starts) != 1:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message=f"工作流必须恰好有 1 个开始节点,当前为 {len(starts)}",
details={"count": len(starts)},
)
)
if len(outputs) < 1:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="工作流至少需要 1 个输出节点",
)
)
unknown = [n for n in graph.nodes if n.type not in ALL_NODE_TYPES]
for node in unknown:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message=f"未知节点类型: {node.type}",
node_id=node.id,
)
)
if _has_uncontrolled_cycle(graph):
issues.append(
ValidationIssue(
code="WORKFLOW_GRAPH_CYCLE",
message="检测到未受控环;循环必须通过显式 loop 节点",
)
)
if starts:
adjacency: dict[str, list[str]] = defaultdict(list)
for edge in graph.edges:
adjacency[edge.source].append(edge.target)
reachable = _collect_reachable({starts[0].id}, adjacency)
for node in graph.nodes:
if node.id not in reachable:
issues.append(
ValidationIssue(
code="WORKFLOW_NODE_NOT_FOUND",
message=f"节点不可从开始节点到达: {node.id}",
node_id=node.id,
)
)
for out in outputs:
if out.id not in reachable:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message=f"输出节点不可达: {out.id}",
node_id=out.id,
)
)
settings = graph.settings
if settings.max_steps > system_max_steps:
issues.append(
ValidationIssue(
code="WORKFLOW_LIMIT_EXCEEDED",
message=f"maxSteps 超过系统上限 {system_max_steps}",
details={"maxSteps": settings.max_steps, "limit": system_max_steps},
)
)
if settings.max_loop_iterations > system_max_loop_iterations:
issues.append(
ValidationIssue(
code="WORKFLOW_LIMIT_EXCEEDED",
message=f"maxLoopIterations 超过系统上限 {system_max_loop_iterations}",
details={
"maxLoopIterations": settings.max_loop_iterations,
"limit": system_max_loop_iterations,
},
)
)
if settings.max_parallelism > system_max_parallelism:
issues.append(
ValidationIssue(
code="WORKFLOW_LIMIT_EXCEEDED",
message=f"maxParallelism 超过系统上限 {system_max_parallelism}",
details={"maxParallelism": settings.max_parallelism, "limit": system_max_parallelism},
)
)
for node in graph.nodes:
if node.type == "condition":
branches = node.config.get("branches") or []
ports = {str(b.get("name")) for b in branches if isinstance(b, dict) and b.get("name")}
default_branch = node.config.get("defaultBranch") or node.config.get("default_branch")
if default_branch:
ports.add(str(default_branch))
edge_ports = {e.source_port for e in graph.edges if e.source == node.id and e.source_port}
missing = edge_ports - ports
for port in missing:
issues.append(
ValidationIssue(
code="WORKFLOW_EDGE_INVALID",
message=f"条件节点出口未声明: {port}",
node_id=node.id,
details={"sourcePort": port},
)
)
if node.type == "loop":
max_iter = node.config.get("maxIterations") or node.config.get("max_iterations")
if max_iter is None:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="loop 节点必须配置 maxIterations",
node_id=node.id,
)
)
body_entry = node.config.get("bodyEntry") or node.config.get("body_entry")
if not body_entry:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="loop 节点必须配置 bodyEntry",
node_id=node.id,
)
)
elif body_entry not in graph.node_map():
issues.append(
ValidationIssue(
code="WORKFLOW_NODE_NOT_FOUND",
message=f"loop bodyEntry 不存在: {body_entry}",
node_id=node.id,
)
)
if resource_checker is not None:
verdict = resource_checker(node.type, node.config or {})
if verdict:
message = verdict if isinstance(verdict, str) else str(verdict.get("message") or "节点资源配置无效")
field = verdict.get("field") if isinstance(verdict, dict) else None
issues.append(
ValidationIssue(
code="WORKFLOW_RESOURCE_MISSING",
message=message,
node_id=node.id,
details={"field": field} if field else None,
)
)
issues.extend(_check_upstream_references(graph))
return issues
def _predecessors(graph: WorkflowGraph) -> dict[str, set[str]]:
preds: dict[str, set[str]] = defaultdict(set)
for edge in graph.edges:
preds[edge.target].add(edge.source)
return preds
def _ancestors(node_id: str, preds: dict[str, set[str]]) -> set[str]:
seen: set[str] = set()
stack = list(preds.get(node_id) or ())
while stack:
current = stack.pop()
if current in seen:
continue
seen.add(current)
stack.extend(preds.get(current) or ())
return seen
def _check_upstream_references(graph: WorkflowGraph) -> list[ValidationIssue]:
"""§8.1: node input references must come from upstream nodes."""
issues: list[ValidationIssue] = []
known = graph.node_map()
preds = _predecessors(graph)
for node in graph.nodes:
refs = collect_referenced_nodes(node.config)
ancestors = _ancestors(node.id, preds)
for ref in sorted(refs):
if ref not in known:
issues.append(
ValidationIssue(
code="WORKFLOW_NODE_NOT_FOUND",
message=f"表达式引用了不存在的节点: {ref}",
node_id=node.id,
details={"referencedNodeId": ref},
)
)
continue
if ref == node.id and node.type != "loop":
issues.append(
ValidationIssue(
code="WORKFLOW_EXPRESSION_INVALID",
message="节点不能引用自身输出",
node_id=node.id,
details={"referencedNodeId": ref},
)
)
continue
if ref != node.id and ref not in ancestors:
issues.append(
ValidationIssue(
code="WORKFLOW_EXPRESSION_INVALID",
message=f"节点引用必须来自上游,{ref} 不在 {node.id} 的上游",
node_id=node.id,
details={"referencedNodeId": ref},
)
)
return issues
def assert_publishable(raw: dict[str, Any] | WorkflowGraph, **kwargs: Any) -> WorkflowGraph:
"""Validate and return the typed graph, or raise ``WorkflowError``."""
issues = validate_workflow_graph(raw, **kwargs)
if issues:
first = issues[0]
raise WorkflowError(
first.code,
first.message,
node_id=first.node_id,
details={"issues": [i.model_dump(by_alias=True) for i in issues]},
)
return raw if isinstance(raw, WorkflowGraph) else WorkflowGraph.model_validate(raw)
__all__ = [
"ResourceChecker",
"ValidationIssue",
"assert_publishable",
"validate_workflow_graph",
]