代码提交

This commit is contained in:
董宏凯 committed 2026-08-30 01:25:20 +08:00
1 parent 4ac7bc21cd
commit 17cf849361
110 files changed
+17007 -518

No files matched your search

@@ -42,6 +42,7 @@ from deerflow.config.token_usage_config import TokenUsageConfig
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
from deerflow.config.vision_config import VisionConfig
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.config.y_log_config import YLogConfig
load_dotenv()
@@ -149,6 +150,7 @@ class AppConfig(BaseModel):
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
y_log: YLogConfig = Field(default_factory=YLogConfig, description="操作日志上报配置(前端经 /api/public/config/y-log 读取)")
workflows: WorkflowConfig = Field(default_factory=WorkflowConfig, description="Workflow Studio limits / iframe origins")
sentiment_agent: SentimentAgentConfig = Field(
default_factory=SentimentAgentConfig,
description="舆情分析虚拟智能体上游代理超时(地址由前端 runtime-config.js 随请求传入)",
@@ -0,0 +1,81 @@
"""Workflow Studio configuration (limits, egress policy, sandbox, iframe origins)."""
from __future__ import annotations
from pydantic import BaseModel, Field
class WorkflowEmbedConfig(BaseModel):
"""iframe ticket / origin policy."""
allowed_origins: list[str] = Field(
default_factory=list,
description="Allowed parent origins for workflow embed tickets (exact match)",
)
ticket_ttl_seconds: int = Field(default=60, ge=5, le=600)
class WorkflowHttpPolicyConfig(BaseModel):
"""Egress policy for the ``http`` node.
Default-deny on private address space. ``allowed_hosts`` is an explicit
allowlist (exact host or ``.suffix`` match); when it is empty every public
host is allowed but private/loopback/link-local targets are still refused,
which is what stops SSRF against cloud metadata and internal services.
"""
enabled: bool = Field(default=True)
allowed_hosts: list[str] = Field(default_factory=list)
blocked_hosts: list[str] = Field(default_factory=list)
allow_private_networks: bool = Field(
default=False,
description="Allow loopback/private/link-local targets (only for trusted intranet deployments)",
)
allowed_ports: list[int] = Field(default_factory=lambda: [80, 443, 8080, 8443])
max_response_bytes: int = Field(default=2_097_152, ge=1024)
max_redirects: int = Field(default=0, ge=0, le=5)
timeout_seconds: int = Field(default=60, ge=1)
class WorkflowSqlPolicyConfig(BaseModel):
"""Read-only policy for the ``sql_read`` node."""
enabled: bool = Field(default=True)
max_rows: int = Field(default=1000, ge=1)
statement_timeout_seconds: int = Field(default=30, ge=1)
max_cell_chars: int = Field(default=4000, ge=64)
class WorkflowCodeNodeConfig(BaseModel):
"""``code`` node execution. Off by default: the local runner is a hardened
subprocess, not a container, so operators must opt in knowingly."""
enabled: bool = Field(default=False)
timeout_seconds: int = Field(default=30, ge=1, le=600)
max_source_chars: int = Field(default=20_000, ge=64)
max_output_chars: int = Field(default=100_000, ge=256)
memory_limit_mb: int = Field(default=512, ge=64)
allow_network: bool = Field(default=False)
class WorkflowRetentionConfig(BaseModel):
event_retention_days: int = Field(default=30, ge=1)
run_retention_days: int = Field(default=90, ge=1)
class WorkflowConfig(BaseModel):
"""System-level workflow limits and policies."""
enabled: bool = Field(default=True, description="Master switch for workflow studio APIs")
max_steps: int = Field(default=500, ge=1)
max_loop_iterations: int = Field(default=20, ge=1)
max_parallelism: int = Field(default=16, ge=1)
run_timeout_seconds: int = Field(default=3600, ge=1)
node_timeout_seconds: int = Field(default=600, ge=1)
max_concurrent_runs_per_user: int = Field(default=5, ge=1)
lease_ttl_seconds: int = Field(default=60, ge=10)
embed: WorkflowEmbedConfig = Field(default_factory=WorkflowEmbedConfig)
http: WorkflowHttpPolicyConfig = Field(default_factory=WorkflowHttpPolicyConfig)
sql: WorkflowSqlPolicyConfig = Field(default_factory=WorkflowSqlPolicyConfig)
code: WorkflowCodeNodeConfig = Field(default_factory=WorkflowCodeNodeConfig)
retention: WorkflowRetentionConfig = Field(default_factory=WorkflowRetentionConfig)
@@ -0,0 +1,64 @@
"""Create workflow_definitions / workflow_versions tables.
Revision ID: 20260828_01
Revises: 20260825_01
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
op.create_table(
"workflow_definitions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("description", sa.Text(), nullable=False, server_default=""),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
sa.Column("draft_graph_json", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_workflow_definitions_owner_id", "workflow_definitions", ["owner_id"])
op.create_index("ix_workflow_definitions_status", "workflow_definitions", ["status"])
if not inspector.has_table("workflow_versions"):
op.create_table(
"workflow_versions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_number", sa.Integer(), nullable=False),
sa.Column("graph_json", sa.Text(), nullable=False),
sa.Column("graph_hash", sa.String(length=64), nullable=False),
sa.Column("input_schema", sa.Text(), nullable=True),
sa.Column("output_schema", sa.Text(), nullable=True),
sa.Column("published_by", sa.String(length=64), nullable=True),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
)
op.create_index("ix_workflow_versions_workflow_id", "workflow_versions", ["workflow_id"])
op.create_index("ix_workflow_versions_graph_hash", "workflow_versions", ["graph_hash"])
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_versions"):
op.drop_table("workflow_versions")
if inspector.has_table("workflow_definitions"):
op.drop_table("workflow_definitions")
@@ -0,0 +1,127 @@
"""Create workflow run / node-run / event / artifact tables.
Revision ID: 20260828_02
Revises: 20260828_01
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_02"
down_revision = "20260828_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_runs"):
op.create_table(
"workflow_runs",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("workflow_version_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
sa.Column("input_json", sa.Text(), nullable=True),
sa.Column("output_json", sa.Text(), nullable=True),
sa.Column("error_json", sa.Text(), nullable=True),
sa.Column("context_json", sa.Text(), nullable=True),
sa.Column("pending_input_json", sa.Text(), nullable=True),
sa.Column("resume_token", sa.String(length=64), nullable=True),
sa.Column("lease_owner", sa.String(length=64), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("attempt", sa.Integer(), nullable=False, server_default="0"),
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
sa.Column("version", sa.Integer(), nullable=False, server_default="0"),
sa.Column("retry_of_run_id", sa.String(length=64), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),
)
op.create_index("ix_workflow_runs_workflow_id", "workflow_runs", ["workflow_id"])
op.create_index("ix_workflow_runs_workflow_version_id", "workflow_runs", ["workflow_version_id"])
op.create_index("ix_workflow_runs_owner_id", "workflow_runs", ["owner_id"])
op.create_index("ix_workflow_runs_status", "workflow_runs", ["status"])
op.create_index("ix_workflow_runs_resume_token", "workflow_runs", ["resume_token"])
if not inspector.has_table("workflow_node_runs"):
op.create_table(
"workflow_node_runs",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=False),
sa.Column("node_type", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
sa.Column("iteration", sa.Integer(), nullable=False, server_default="0"),
sa.Column("is_current", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("input_snapshot", sa.Text(), nullable=True),
sa.Column("output_snapshot", sa.Text(), nullable=True),
sa.Column("error_json", sa.Text(), nullable=True),
sa.Column("parent_node_run_id", sa.String(length=64), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"
),
)
op.create_index("ix_workflow_node_runs_run_id", "workflow_node_runs", ["run_id"])
op.create_index("ix_workflow_node_runs_status", "workflow_node_runs", ["status"])
if not inspector.has_table("workflow_run_events"):
op.create_table(
"workflow_run_events",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_id", sa.String(length=64), nullable=False),
sa.Column("seq", sa.Integer(), nullable=False),
sa.Column("event_type", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=True),
sa.Column("node_run_id", sa.String(length=64), nullable=True),
sa.Column("payload_json", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
)
op.create_index("ix_workflow_run_events_run_id", "workflow_run_events", ["run_id"])
op.create_index("ix_workflow_run_events_run_seq", "workflow_run_events", ["run_id", "seq"])
if not inspector.has_table("workflow_run_artifacts"):
op.create_table(
"workflow_run_artifacts",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=False),
sa.Column("name", sa.String(length=512), nullable=False),
sa.Column("mime_type", sa.String(length=191), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("preview", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_workflow_run_artifacts_run_id", "workflow_run_artifacts", ["run_id"])
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
for table in (
"workflow_run_artifacts",
"workflow_run_events",
"workflow_node_runs",
"workflow_runs",
):
if inspector.has_table(table):
op.drop_table(table)
@@ -0,0 +1,49 @@
"""Create workflow_data_sources table.
Revision ID: 20260828_03
Revises: 20260828_02
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_03"
down_revision = "20260828_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_data_sources"):
return
op.create_table(
"workflow_data_sources",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=191), nullable=False),
sa.Column("description", sa.String(length=512), nullable=False, server_default=""),
sa.Column("kind", sa.String(length=32), nullable=False, server_default="sql"),
sa.Column("driver", sa.String(length=64), nullable=False, server_default=""),
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
sa.Column("secret_encrypted", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("masked_target", sa.String(length=512), nullable=False, server_default=""),
sa.Column("max_rows", sa.Integer(), nullable=False, server_default="1000"),
sa.Column("allowed_tables", sa.Text(), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("owner_id", sa.String(length=64), nullable=True),
sa.Column("created_by", sa.String(length=64), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("name", name="uq_workflow_data_sources_name"),
)
op.create_index("ix_workflow_data_sources_owner_id", "workflow_data_sources", ["owner_id"])
def downgrade() -> None:
bind = op.get_bind()
if sa.inspect(bind).has_table("workflow_data_sources"):
op.drop_table("workflow_data_sources")
@@ -0,0 +1,66 @@
"""Add workflow counter columns: version allocator + event seq counter.
Revision ID: 20260828_04
Revises: 20260828_03
Create Date: 2026-08-28
Functional-robustness remediation (audit items P1-02 / P1-03):
* ``workflow_definitions.next_version_number`` — publish allocates the version
number by CAS-bumping this counter instead of ``MAX(version_number) + 1``,
so two concurrent publishes get adjacent distinct numbers instead of a
unique-constraint 500.
* ``workflow_runs.next_event_seq`` — the event log draws each event's ``seq``
from this counter in the insert's own transaction instead of ``MAX(seq)+1``.
Both columns carry a ``server_default`` so existing databases get them via the
startup ``_ensure_orm_columns_sync`` sweep even without running Alembic; rows
keep the default (1 / 0) and the first write self-heals from the existing
maximum on unique-conflict.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_04"
down_revision = "20260828_03"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_definitions"):
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
if "next_version_number" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("next_version_number", sa.Integer(), nullable=False, server_default="1"),
)
if inspector.has_table("workflow_runs"):
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
if "next_event_seq" not in existing:
op.add_column(
"workflow_runs",
sa.Column("next_event_seq", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_definitions"):
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
if "next_version_number" in existing:
op.drop_column("workflow_definitions", "next_version_number")
if inspector.has_table("workflow_runs"):
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
if "next_event_seq" in existing:
op.drop_column("workflow_runs", "next_event_seq")
@@ -0,0 +1,50 @@
"""Add non-secret HTTP metadata to workflow data sources.
Revision ID: 20260829_01
Revises: 20260828_04
Create Date: 2026-08-29
HTTP request headers remain in the encrypted secret blob. The endpoint and
method allowlist are intentionally separate non-secret fields so the canvas
can show a reusable interface resource without exposing credentials.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260829_01"
down_revision = "20260828_04"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_data_sources"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
if "base_url" not in existing:
op.add_column(
"workflow_data_sources",
sa.Column("base_url", sa.String(length=2048), nullable=False, server_default=""),
)
if "allowed_methods" not in existing:
op.add_column(
"workflow_data_sources",
sa.Column("allowed_methods", sa.Text(), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_data_sources"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
if "allowed_methods" in existing:
op.drop_column("workflow_data_sources", "allowed_methods")
if "base_url" in existing:
op.drop_column("workflow_data_sources", "base_url")
@@ -0,0 +1,51 @@
"""Add Coze canvas draft column to workflow_definitions.
Revision ID: 20260829_02
Revises: 20260829_01
Create Date: 2026-08-29
The canvas document (positions, Coze form internals) is stored separately from
the executable WorkflowGraph so neither representation can impersonate the
other. ``published_canvas_schema_hash`` records which canvas the current
published version was compiled from, for audit only.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260829_02"
down_revision = "20260829_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "draft_canvas_schema_json" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
)
if "published_canvas_schema_hash" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("published_canvas_schema_hash", sa.String(length=64), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "published_canvas_schema_hash" in existing:
op.drop_column("workflow_definitions", "published_canvas_schema_hash")
if "draft_canvas_schema_json" in existing:
op.drop_column("workflow_definitions", "draft_canvas_schema_json")
@@ -50,6 +50,14 @@ from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
@@ -128,6 +136,13 @@ __all__ = [
"KnowledgeSourceRow",
"KnowledgeTagRow",
"LightAppRow",
"WorkflowDataSourceRow",
"WorkflowDefinitionRow",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunEventRow",
"WorkflowRunRow",
"WorkflowVersionRow",
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
@@ -0,0 +1,19 @@
"""Workflow data-source persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_data_sources.store import (
MemoryWorkflowDataSourceStore,
SqlWorkflowDataSourceStore,
WorkflowDataSourceStore,
make_workflow_data_source_store,
)
__all__ = [
"MemoryWorkflowDataSourceStore",
"SqlWorkflowDataSourceStore",
"WorkflowDataSourceRow",
"WorkflowDataSourceStore",
"make_workflow_data_source_store",
]
@@ -0,0 +1,51 @@
"""ORM model for workflow SQL data sources and HTTP credentials."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowDataSourceRow(Base):
"""A read-only database connection usable by ``sql_read`` nodes.
``secret_ciphertext`` holds the DSN encrypted with the workflow secret key;
it is never returned by any API. ``owner_id`` NULL means the data source is
shared (admin-managed)."""
__tablename__ = "workflow_data_sources"
__table_args__ = (UniqueConstraint("name", name="uq_workflow_data_sources_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(191), nullable=False)
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="sql")
driver: Mapped[str] = mapped_column(String(64), nullable=False, default="")
base_url: Mapped[str] = mapped_column(
String(2048),
nullable=False,
default="",
server_default=sa_text("''"),
)
# Nullable so the startup column synchronizer can safely patch existing databases.
allowed_methods: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
secret_ciphertext: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
secret_encrypted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
masked_target: Mapped[str] = mapped_column(String(512), nullable=False, default="")
max_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=1000, server_default=sa_text("1000"))
allowed_tables: Mapped[list] = mapped_column(PortableJSON(), default=list)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
owner_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,293 @@
"""Workflow data-source store (SQL + in-memory).
Secrets are encrypted before they reach the database and only ever decrypted
inside :meth:`resolve_dsn`, which the run executor calls. No read path returns
the ciphertext.
"""
from __future__ import annotations
import copy
import uuid
from abc import ABC, abstractmethod
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.workflows.security.secret_box import decrypt_secret, encrypt_secret, mask_dsn
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _public_dict(row: WorkflowDataSourceRow) -> dict[str, Any]:
return {
"id": row.id,
"name": row.name,
"description": row.description or "",
"kind": row.kind,
"driver": row.driver or "",
"base_url": row.base_url or "",
"allowed_methods": list(row.allowed_methods or []),
"masked_target": row.masked_target or "",
"secret_encrypted": bool(row.secret_encrypted),
"max_rows": row.max_rows,
"allowed_tables": list(row.allowed_tables or []),
"enabled": bool(row.enabled),
"owner_id": row.owner_id,
"created_by": row.created_by,
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
class WorkflowDataSourceStore(ABC):
@abstractmethod
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_source(self, source_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_source(self, source_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
"""Decrypted connection info for the executor. Never expose via HTTP."""
raise NotImplementedError
class SqlWorkflowDataSourceStore(WorkflowDataSourceStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
stmt = select(WorkflowDataSourceRow).order_by(WorkflowDataSourceRow.name.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
visible = []
for row in rows:
if row.owner_id is None and include_shared:
visible.append(row)
elif owner_id is not None and row.owner_id == owner_id:
visible.append(row)
return [_public_dict(row) for row in visible]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
return _public_dict(row) if row else None
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
dsn = str(data.get("dsn") or "")
kind = str(data.get("kind") or "sql")
base_url = str(data.get("base_url") or "")
row = WorkflowDataSourceRow(
id=str(data.get("id") or uuid.uuid4()),
name=str(data["name"])[:191],
description=str(data.get("description") or "")[:512],
kind=kind,
driver=str(
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
)[:64],
base_url=base_url[:2048],
allowed_methods=list(data.get("allowed_methods") or []),
secret_ciphertext=encrypt_secret(dsn) if dsn else "",
secret_encrypted=bool(dsn),
masked_target=(base_url if kind == "http" else mask_dsn(dsn))[:512],
max_rows=int(data.get("max_rows") or 1000),
allowed_tables=list(data.get("allowed_tables") or []),
enabled=bool(data.get("enabled", True)),
owner_id=data.get("owner_id"),
created_by=data.get("created_by"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = _public_dict(row)
await session.commit()
return out
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None:
return None
if data.get("name"):
row.name = str(data["name"])[:191]
if "description" in data:
row.description = str(data.get("description") or "")[:512]
if "base_url" in data:
row.base_url = str(data.get("base_url") or "")[:2048]
if row.kind == "http":
row.masked_target = row.base_url[:512]
if "allowed_methods" in data:
row.allowed_methods = list(data.get("allowed_methods") or [])
if data.get("dsn"):
dsn = str(data["dsn"])
row.secret_ciphertext = encrypt_secret(dsn)
row.secret_encrypted = True
if row.kind == "http":
row.driver = "HTTP API"
else:
row.masked_target = mask_dsn(dsn)[:512]
row.driver = dsn.split("://", 1)[0][:64]
if data.get("max_rows") is not None:
row.max_rows = int(data["max_rows"])
if "allowed_tables" in data:
row.allowed_tables = list(data.get("allowed_tables") or [])
if "enabled" in data:
row.enabled = bool(data["enabled"])
row.updated_at = datetime.now(UTC)
await session.flush()
out = _public_dict(row)
await session.commit()
return out
async def delete_source(self, source_id: str) -> bool:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None or not row.enabled or not row.secret_ciphertext:
return None
return {
"id": row.id,
"dsn": decrypt_secret(row.secret_ciphertext),
"kind": row.kind,
"base_url": row.base_url or "",
"allowed_methods": list(row.allowed_methods or []),
"max_rows": row.max_rows,
"allowed_tables": list(row.allowed_tables or []),
"owner_id": row.owner_id,
}
class MemoryWorkflowDataSourceStore(WorkflowDataSourceStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._secrets: dict[str, str] = {}
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
out = []
for row in self._rows.values():
if row["owner_id"] is None and include_shared:
out.append(row)
elif owner_id is not None and row["owner_id"] == owner_id:
out.append(row)
return [copy.deepcopy(r) for r in sorted(out, key=lambda r: r["name"])]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
row = self._rows.get(source_id)
return copy.deepcopy(row) if row else None
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
source_id = str(data.get("id") or uuid.uuid4())
dsn = str(data.get("dsn") or "")
kind = str(data.get("kind") or "sql")
base_url = str(data.get("base_url") or "")
now = _iso(datetime.now(UTC))
row = {
"id": source_id,
"name": str(data["name"]),
"description": str(data.get("description") or ""),
"kind": kind,
"driver": str(
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
),
"base_url": base_url,
"allowed_methods": list(data.get("allowed_methods") or []),
"masked_target": base_url if kind == "http" else mask_dsn(dsn),
"secret_encrypted": bool(dsn),
"max_rows": int(data.get("max_rows") or 1000),
"allowed_tables": list(data.get("allowed_tables") or []),
"enabled": bool(data.get("enabled", True)),
"owner_id": data.get("owner_id"),
"created_by": data.get("created_by"),
"created_at": now,
"updated_at": now,
}
self._rows[source_id] = row
self._secrets[source_id] = dsn
return copy.deepcopy(row)
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._rows.get(source_id)
if row is None:
return None
for key in (
"name",
"description",
"base_url",
"allowed_methods",
"max_rows",
"allowed_tables",
"enabled",
):
if key in data and data[key] is not None:
row[key] = data[key]
if "base_url" in data and row["kind"] == "http":
row["masked_target"] = str(row["base_url"])
if data.get("dsn"):
self._secrets[source_id] = str(data["dsn"])
if row["kind"] == "http":
row["driver"] = "HTTP API"
else:
row["masked_target"] = mask_dsn(str(data["dsn"]))
row["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(row)
async def delete_source(self, source_id: str) -> bool:
self._secrets.pop(source_id, None)
return self._rows.pop(source_id, None) is not None
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
row = self._rows.get(source_id)
if row is None or not row["enabled"]:
return None
return {
"id": source_id,
"dsn": self._secrets.get(source_id, ""),
"kind": row["kind"],
"base_url": row.get("base_url", ""),
"allowed_methods": list(row.get("allowed_methods") or []),
"max_rows": row["max_rows"],
"allowed_tables": row["allowed_tables"],
"owner_id": row["owner_id"],
}
def make_workflow_data_source_store(session_factory):
if session_factory is None:
return MemoryWorkflowDataSourceStore()
return SqlWorkflowDataSourceStore(session_factory)
__all__ = [
"MemoryWorkflowDataSourceStore",
"SqlWorkflowDataSourceStore",
"WorkflowDataSourceStore",
"make_workflow_data_source_store",
]
@@ -0,0 +1,22 @@
"""Workflow run event-log persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
__all__ = [
"MemoryWorkflowEventStore",
"SqlWorkflowEventStore",
"WorkflowEventStore",
"WorkflowRunEventRow",
"make_workflow_event_store",
]
def make_workflow_event_store(session_factory, run_store=None):
if session_factory is None:
return MemoryWorkflowEventStore(run_store=run_store)
return SqlWorkflowEventStore(session_factory)
@@ -0,0 +1,42 @@
"""Abstract workflow run event store (durable, replayable)."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowEventStore(ABC):
@abstractmethod
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
"""Persist one event, assigning the next per-run ``seq``."""
raise NotImplementedError
@abstractmethod
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def max_seq(self, run_id: str) -> int:
raise NotImplementedError
@abstractmethod
async def purge_before(self, *, older_than_days: int) -> int:
"""Retention purge by event age; returns deleted row count."""
raise NotImplementedError
@abstractmethod
async def purge_for_runs(self, run_ids: list[str]) -> int:
"""Delete every event belonging to the given runs (used when the runs
themselves are retained away). Returns deleted row count."""
raise NotImplementedError
@@ -0,0 +1,103 @@
"""In-memory workflow event log (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
class MemoryWorkflowEventStore(WorkflowEventStore):
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
self._events: dict[str, list[dict[str, Any]]] = {}
self._max_per_run = max_per_run
self._run_store = run_store
self._lock = asyncio.Lock()
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
async with self._lock:
rows = self._events.setdefault(run_id, [])
event = {
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
"run_id": run_id,
"workflow_id": workflow_id,
"version_id": version_id,
"event_type": event_type,
"node_id": node_id,
"node_run_id": node_run_id,
"payload": payload or {},
"created_at": datetime.now(UTC).isoformat(),
}
rows.append(event)
if len(rows) > self._max_per_run:
del rows[: len(rows) - self._max_per_run]
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
# where the counter bump and the insert share one transaction).
try:
await self._run_store.note_event_seq(run_id, event["seq"])
except Exception: # noqa: BLE001 - counter sync is best effort
pass
return copy.deepcopy(event)
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
return [copy.deepcopy(r) for r in rows[:limit]]
async def max_seq(self, run_id: str) -> int:
rows = self._events.get(run_id, [])
return int(rows[-1]["seq"]) if rows else 0
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted = 0
for run_id, rows in list(self._events.items()):
kept: list[dict[str, Any]] = []
for row in rows:
created = _parse_dt(row.get("created_at"))
if created is not None and created < cutoff:
deleted += 1
else:
kept.append(row)
if kept:
self._events[run_id] = kept
else:
del self._events[run_id]
return deleted
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._lock:
deleted = 0
for run_id in run_ids:
rows = self._events.pop(run_id, None)
if rows:
deleted += len(rows)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,34 @@
"""ORM model for the durable workflow run event log."""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class WorkflowRunEventRow(Base):
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
__tablename__ = "workflow_run_events"
__table_args__ = (
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,157 @@
"""SQLAlchemy-backed workflow event log."""
from __future__ import annotations
import json
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
_MAX_PAYLOAD_CHARS = 64_000
_APPEND_ATTEMPTS = 5
def _dump_payload(payload: dict[str, Any] | None) -> str:
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
if len(text) > _MAX_PAYLOAD_CHARS:
return json.dumps(
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
ensure_ascii=False,
separators=(",", ":"),
)
return text
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
try:
payload = json.loads(row.payload_json or "{}")
except json.JSONDecodeError:
payload = {}
return {
"seq": row.seq,
"run_id": row.run_id,
"workflow_id": row.workflow_id,
"version_id": row.version_id,
"event_type": row.event_type,
"node_id": row.node_id,
"node_run_id": row.node_run_id,
"payload": payload if isinstance(payload, dict) else {},
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
}
class SqlWorkflowEventStore(WorkflowEventStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
max_retries: int = _APPEND_ATTEMPTS,
) -> dict[str, Any]:
"""Persist one event, taking its ``seq`` from the run's own counter.
``MAX(seq) + 1`` let two appenders draw the same number and burn a
unique-constraint retry; here the counter increment and the insert share
one transaction, and the row write-lock held by the UPDATE serialises
concurrent appenders until commit — so seqs are strictly increasing,
unique and gap-free per run, in commit order.
"""
text = _dump_payload(payload)
for _ in range(max_retries):
async with self._sf() as session:
seq = await self._take_seq(session, run_id)
row = WorkflowRunEventRow(
run_id=run_id,
workflow_id=workflow_id,
version_id=version_id,
seq=seq,
event_type=event_type,
node_id=node_id,
node_run_id=node_run_id,
payload_json=text,
)
session.add(row)
try:
await session.flush()
except IntegrityError:
# Only reachable for legacy rows whose counter was backfilled
# to 0 while events already existed. Re-sync the counter to
# MAX(seq) in its own committed transaction, then retry.
await session.rollback()
await self._resync_counter(run_id)
continue
out = event_to_dict(row)
await session.commit()
return out
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
async def _take_seq(self, session: Any, run_id: str) -> int:
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
missing (event store used standalone in tests)."""
result = await session.execute(
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
)
if result.rowcount:
# Our own uncommitted increment; the row lock guards it until commit.
current = (
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
).scalar()
return int(current or 1)
current = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar()
return int(current or 0) + 1
async def _resync_counter(self, run_id: str) -> None:
"""Point a legacy/stale run counter back at MAX(seq)."""
async with self._sf() as session:
max_seq = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar() or 0
await session.execute(
update(WorkflowRunRow)
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
.values(next_event_seq=int(max_seq))
)
await session.commit()
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [event_to_dict(r) for r in rows]
async def max_seq(self, run_id: str) -> int:
async with self._sf() as session:
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
return int(value or 0)
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
await session.commit()
return int(result.rowcount or 0)
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,38 @@
"""Workflow run / node-run / artifact persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunConflictError,
WorkflowRunNotFoundError,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
__all__ = [
"CANCELLABLE_RUN_STATUSES",
"MemoryWorkflowRunStore",
"SqlWorkflowRunStore",
"TERMINAL_RUN_STATUSES",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunConflictError",
"WorkflowRunNotFoundError",
"WorkflowRunRow",
"WorkflowRunStore",
"make_workflow_run_store",
]
def make_workflow_run_store(session_factory):
if session_factory is None:
return MemoryWorkflowRunStore()
return SqlWorkflowRunStore(session_factory)
@@ -0,0 +1,161 @@
"""Abstract workflow run / node-run / artifact store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
class WorkflowRunNotFoundError(Exception):
def __init__(self, run_id: str) -> None:
super().__init__(f"workflow run not found: {run_id}")
self.run_id = run_id
class WorkflowRunConflictError(Exception):
"""Raised when a state transition is not legal for the current status."""
def __init__(self, run_id: str, status: str) -> None:
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
self.run_id = run_id
self.status = status
class WorkflowRunStore(ABC):
# ── runs ────────────────────────────────────────────────────────────
@abstractmethod
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
"""Create a run. Returns ``(run, created)``; an existing
``idempotency_key`` yields the stored run with ``created=False``."""
raise NotImplementedError
@abstractmethod
async def get_run(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
raise NotImplementedError
@abstractmethod
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
raise NotImplementedError
@abstractmethod
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
"""CAS update. When ``lease_owner`` is given the row must still be leased
by that owner. Terminal statuses clear the lease."""
raise NotImplementedError
@abstractmethod
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Pause atomically: status + token + pending descriptor (+ optional run
context) in one conditional update, so a concurrent resume can never be
overwritten by a follow-up context write."""
raise NotImplementedError
@abstractmethod
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
``queued``. Returns None when the token is wrong/already used."""
raise NotImplementedError
@abstractmethod
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic human-input resume in a single conditional update.
Validates the token, stores ``payload`` as ``context.resume_payload``
(replacing the whole run context with ``context`` when given), clears the
token/pending descriptor and flips the run to ``queued`` — all under
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
the resumed run on success, ``None`` when the token is wrong, already
used, or the run is no longer awaiting input (caller maps that to 409).
"""
raise NotImplementedError
# ── node runs ───────────────────────────────────────────────────────
@abstractmethod
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
raise NotImplementedError
@abstractmethod
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
"""Mark node runs stale (used when a loop re-enters its body)."""
raise NotImplementedError
# ── artifacts ───────────────────────────────────────────────────────
@abstractmethod
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
"""Fetch one artifact row scoped to its run (authorization key)."""
raise NotImplementedError
# ── retention ───────────────────────────────────────────────────────
@abstractmethod
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
"""Delete finished runs older than the cutoff, with their node runs and
artifact rows. Only terminal runs are eligible. Returns deleted run ids
(bounded per call so a sweeper can loop)."""
raise NotImplementedError
@@ -0,0 +1,388 @@
"""In-memory workflow run store (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
class MemoryWorkflowRunStore(WorkflowRunStore):
def __init__(self) -> None:
self._runs: dict[str, dict[str, Any]] = {}
self._node_runs: dict[str, list[dict[str, Any]]] = {}
self._artifacts: dict[str, list[dict[str, Any]]] = {}
self._lock = asyncio.Lock()
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
async with self._lock:
key = data.get("idempotency_key")
if key:
for run in self._runs.values():
if run.get("idempotency_key") == str(key):
return copy.deepcopy(run), False
now = datetime.now(UTC)
run_id = str(data.get("id") or uuid.uuid4())
run = {
"id": run_id,
"workflow_id": str(data["workflow_id"]),
"workflow_version_id": str(data["workflow_version_id"]),
"status": str(data.get("status") or "queued"),
"owner_id": str(data["owner_id"]),
"idempotency_key": str(key) if key else None,
"input": data.get("input") or {},
"output": None,
"error": None,
"context": data.get("context") or {},
"pending_input": None,
"resume_token": None,
"lease_owner": None,
"lease_until": None,
"heartbeat_at": None,
"attempt": 0,
"max_attempts": int(data.get("max_attempts") or 3),
"version": 0,
"retry_of_run_id": data.get("retry_of_run_id"),
"next_event_seq": 0,
"started_at": None,
"finished_at": None,
"created_at": _iso(now),
"updated_at": _iso(now),
"_created": now,
}
self._runs[run_id] = run
return copy.deepcopy(run), True
async def get_run(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
return copy.deepcopy(run) if run else None
async def note_event_seq(self, run_id: str, seq: int) -> None:
"""Mirror the SQL counter semantics for no-DB mode: the memory event
store reports each handed-out seq so run views can expose lastSeq."""
async with self._lock:
run = self._runs.get(run_id)
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
run["next_event_seq"] = int(seq)
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
if workflow_id:
rows = [r for r in rows if r["workflow_id"] == workflow_id]
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
out: list[str] = []
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
if run["status"] == "queued":
out.append(run["id"])
elif run["status"] == "running" and run["lease_until"]:
if datetime.fromisoformat(run["lease_until"]) < now:
out.append(run["id"])
if len(out) >= limit:
break
return out
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
now = datetime.now(UTC)
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
if not claimable:
return None
run.update(
status="running",
lease_owner=lease_owner,
lease_until=_iso(lease_until),
heartbeat_at=_iso(now),
attempt=run["attempt"] + 1,
version=run["version"] + 1,
started_at=run["started_at"] or _iso(now),
updated_at=_iso(now),
)
return copy.deepcopy(run)
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
run = self._runs.get(run_id)
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
return False
run["lease_until"] = _iso(lease_until)
run["heartbeat_at"] = _iso(datetime.now(UTC))
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
mapping = {
"output_json": "output",
"error_json": "error",
"context_json": "context",
"pending_input_json": "pending_input",
}
for key, value in fields.items():
run[mapping.get(key, key)] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
run["lease_owner"] = None
run["lease_until"] = None
run["resume_token"] = None
run["pending_input"] = None
run["finished_at"] = run.get("finished_at") or _iso(now)
for key in ("started_at", "finished_at"):
if isinstance(run.get(key), datetime):
run[key] = _iso(run[key])
run["version"] += 1
run["updated_at"] = _iso(now)
return copy.deepcopy(run)
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None:
return None
if run["status"] in ("cancel_requested", "cancelled"):
return copy.deepcopy(run)
if run["status"] not in CANCELLABLE_RUN_STATUSES:
return None
run["status"] = "cancel_requested"
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None or run["status"] != "cancel_requested":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input=None,
finished_at=_iso(now),
updated_at=_iso(now),
)
run["version"] += 1
return copy.deepcopy(run)
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "running":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="awaiting_input",
pending_input=copy.deepcopy(pending_input),
resume_token=resume_token,
lease_owner=None,
lease_until=None,
updated_at=_iso(now),
)
if context is not None:
run["context"] = copy.deepcopy(context)
run["version"] += 1
return copy.deepcopy(run)
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
run.update(status="queued", resume_token=None, pending_input=None)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
base_context = context if context is not None else run.get("context") or {}
run.update(
context={**base_context, "resume_payload": copy.deepcopy(payload)},
status="queued",
resume_token=None,
pending_input=None,
)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
rows = self._node_runs.setdefault(run_id, [])
existing = next(
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
None,
)
if existing is None:
existing = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": run_id,
"node_id": node_id,
"node_type": str(data.get("node_type") or "unknown"),
"status": "queued",
"attempt": attempt,
"iteration": iteration,
"is_current": True,
"input": None,
"output": None,
"error": None,
"parent_node_run_id": None,
"duration_ms": None,
"started_at": None,
"finished_at": None,
"created_at": _iso(datetime.now(UTC)),
}
rows.append(existing)
for key in (
"status",
"node_type",
"input",
"output",
"error",
"parent_node_run_id",
"duration_ms",
"started_at",
"finished_at",
):
if key in data and data[key] is not None:
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
if "is_current" in data:
existing["is_current"] = bool(data["is_current"])
return copy.deepcopy(existing)
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
rows = self._node_runs.get(run_id, [])
if current_only:
rows = [r for r in rows if r["is_current"]]
return [copy.deepcopy(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
count = 0
for row in self._node_runs.get(run_id, []):
if row["node_id"] in node_ids and row["is_current"]:
row["is_current"] = False
count += 1
return count
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": str(data["run_id"]),
"node_id": str(data.get("node_id") or ""),
"name": str(data.get("name") or ""),
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
"path": str(data.get("path") or ""),
"size_bytes": data.get("size_bytes"),
"preview": data.get("preview"),
"created_at": _iso(datetime.now(UTC)),
}
self._artifacts.setdefault(row["run_id"], []).append(row)
return copy.deepcopy(row)
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
for row in self._artifacts.get(run_id, []):
if row["id"] == artifact_id:
return copy.deepcopy(row)
return None
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted: list[str] = []
for run_id, run in list(self._runs.items()):
if run["status"] not in TERMINAL_RUN_STATUSES:
continue
finished = _parse_dt(run.get("finished_at"))
if finished is None or finished >= cutoff:
continue
deleted.append(run_id)
for run_id in deleted:
self._runs.pop(run_id, None)
self._node_runs.pop(run_id, None)
self._artifacts.pop(run_id, None)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,96 @@
"""ORM models for workflow runs and per-node runs."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowRunRow(Base):
"""One workflow execution bound to an immutable published version."""
__tablename__ = "workflow_runs"
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
# Non-sensitive run context (never credentials).
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
# Pending human-input descriptor when status == awaiting_input.
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
# Last seq handed out by the event log for this run (0 = none yet). The
# event store increments it in the same transaction that inserts the event
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
# first append self-heals from MAX(seq) on conflict.
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowNodeRunRow(Base):
"""Per-node execution record (also the resume checkpoint for a paused run)."""
__tablename__ = "workflow_node_runs"
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
# queued | running | completed | failed | skipped | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class WorkflowRunArtifactRow(Base):
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
__tablename__ = "workflow_run_artifacts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
name: Mapped[str] = mapped_column(String(512), nullable=False)
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,566 @@
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
_RUN_UPDATABLE = {
"status",
"output_json",
"error_json",
"context_json",
"pending_input_json",
"resume_token",
"started_at",
"finished_at",
}
# Retention deletes in bounded batches so a long-neglected table cannot produce
# one enormous transaction.
_PURGE_BATCH = 500
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _load_text_json(raw: str | None) -> Any:
if not raw:
return None
try:
return json.loads(raw)
except (TypeError, json.JSONDecodeError):
return None
def _dump_text_json(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
return {
"id": row.id,
"workflow_id": row.workflow_id,
"workflow_version_id": row.workflow_version_id,
"status": row.status,
"owner_id": row.owner_id,
"idempotency_key": row.idempotency_key,
"input": row.input_json or {},
"output": row.output_json,
"error": row.error_json,
"context": row.context_json or {},
"pending_input": row.pending_input_json,
"resume_token": row.resume_token,
"lease_owner": row.lease_owner,
"lease_until": _iso(row.lease_until),
"heartbeat_at": _iso(row.heartbeat_at),
"attempt": row.attempt,
"max_attempts": row.max_attempts,
"version": row.version,
"retry_of_run_id": row.retry_of_run_id,
# Last seq handed out by the event log (0 = no events yet).
"next_event_seq": int(row.next_event_seq or 0),
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"node_type": row.node_type,
"status": row.status,
"attempt": row.attempt,
"iteration": row.iteration,
"is_current": bool(row.is_current),
"input": _load_text_json(row.input_snapshot),
"output": _load_text_json(row.output_snapshot),
"error": row.error_json,
"parent_node_run_id": row.parent_node_run_id,
"duration_ms": row.duration_ms,
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
}
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"name": row.name,
"mime_type": row.mime_type,
"path": row.path,
"size_bytes": row.size_bytes,
"preview": row.preview,
"created_at": _iso(row.created_at),
}
class SqlWorkflowRunStore(WorkflowRunStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
key = data.get("idempotency_key")
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
row = WorkflowRunRow(
id=str(data.get("id") or uuid.uuid4()),
workflow_id=str(data["workflow_id"]),
workflow_version_id=str(data["workflow_version_id"]),
status=str(data.get("status") or "queued"),
owner_id=str(data["owner_id"]),
idempotency_key=str(key) if key else None,
input_json=data.get("input") or {},
context_json=data.get("context") or {},
max_attempts=int(data.get("max_attempts") or 3),
retry_of_run_id=data.get("retry_of_run_id"),
)
async with self._sf() as session:
session.add(row)
try:
await session.flush()
except IntegrityError:
await session.rollback()
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
raise
out = run_to_dict(row)
await session.commit()
return out, True
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
return run_to_dict(row) if row else None
async def get_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
stmt = select(WorkflowRunRow)
if workflow_id:
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
if owner_id:
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowRunRow.status == status)
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [run_to_dict(r) for r in rows]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
stmt = (
select(WorkflowRunRow.id)
.where(
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
)
)
.order_by(WorkflowRunRow.created_at.asc())
.limit(limit)
)
async with self._sf() as session:
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
),
)
.values(
status="running",
lease_owner=lease_owner,
lease_until=lease_until,
heartbeat_at=now,
attempt=WorkflowRunRow.attempt + 1,
version=WorkflowRunRow.version + 1,
started_at=func.coalesce(WorkflowRunRow.started_at, now),
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.lease_owner == lease_owner,
WorkflowRunRow.status == "running",
)
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
return False
await session.commit()
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
for key, value in fields.items():
if key in _RUN_UPDATABLE:
values[key] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
values["lease_owner"] = None
values["lease_until"] = None
values["resume_token"] = None
values["pending_input_json"] = None
values.setdefault("finished_at", now)
where = [WorkflowRunRow.id == run_id]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
)
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(*where)
.values(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input_json=None,
finished_at=now,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
values: dict[str, Any] = {
"status": "awaiting_input",
"pending_input_json": pending_input,
"resume_token": resume_token,
"lease_owner": None,
"lease_until": None,
"version": WorkflowRunRow.version + 1,
"updated_at": now,
}
if context is not None:
values["context_json"] = context
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
if context is None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
if row is None:
return None
context = dict(row.context_json or {})
new_context = {**context, "resume_payload": payload}
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
context_json=new_context,
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
async with self._sf() as session:
row = (
await session.execute(
select(WorkflowNodeRunRow).where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id == node_id,
WorkflowNodeRunRow.attempt == attempt,
WorkflowNodeRunRow.iteration == iteration,
)
)
).scalar_one_or_none()
if row is None:
row = WorkflowNodeRunRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=run_id,
node_id=node_id,
node_type=str(data.get("node_type") or "unknown"),
attempt=attempt,
iteration=iteration,
)
session.add(row)
row.status = str(data.get("status") or row.status)
row.node_type = str(data.get("node_type") or row.node_type)
row.is_current = bool(data.get("is_current", True))
if "input" in data:
row.input_snapshot = _dump_text_json(data.get("input"))
if "output" in data:
row.output_snapshot = _dump_text_json(data.get("output"))
if "error" in data:
row.error_json = data.get("error")
if data.get("parent_node_run_id"):
row.parent_node_run_id = str(data["parent_node_run_id"])
if data.get("duration_ms") is not None:
row.duration_ms = int(data["duration_ms"])
if data.get("started_at") is not None:
row.started_at = data["started_at"]
if data.get("finished_at") is not None:
row.finished_at = data["finished_at"]
await session.flush()
out = node_run_to_dict(row)
await session.commit()
return out
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
if current_only:
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [node_run_to_dict(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
if not node_ids:
return 0
async with self._sf() as session:
result = await session.execute(
update(WorkflowNodeRunRow)
.where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id.in_(node_ids),
WorkflowNodeRunRow.is_current.is_(True),
)
.values(is_current=False)
)
await session.commit()
return int(result.rowcount or 0)
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = WorkflowRunArtifactRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=str(data["run_id"]),
node_id=str(data.get("node_id") or ""),
name=str(data.get("name") or "")[:512],
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
path=str(data.get("path") or ""),
size_bytes=data.get("size_bytes"),
preview=data.get("preview"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = artifact_to_dict(row)
await session.commit()
return out
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [artifact_to_dict(r) for r in rows]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
stmt = select(WorkflowRunArtifactRow).where(
WorkflowRunArtifactRow.id == artifact_id,
WorkflowRunArtifactRow.run_id == run_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return artifact_to_dict(row) if row is not None else None
# ── retention ───────────────────────────────────────────────────────
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
run_ids = (
(
await session.execute(
select(WorkflowRunRow.id)
.where(
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
WorkflowRunRow.finished_at.is_not(None),
WorkflowRunRow.finished_at < cutoff,
)
.limit(_PURGE_BATCH)
)
)
.scalars()
.all()
)
if not run_ids:
return []
ids = list(run_ids)
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
await session.commit()
return ids
@@ -0,0 +1,31 @@
"""Workflow definition / version persistence."""
from __future__ import annotations
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
WorkflowVersionNotFoundError,
)
from deerflow.persistence.workflows.memory import MemoryWorkflowStore
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
from deerflow.persistence.workflows.sql import WorkflowRepository
__all__ = [
"MemoryWorkflowStore",
"WorkflowDefinitionRow",
"WorkflowDraftConflictError",
"WorkflowNotFoundError",
"WorkflowRepository",
"WorkflowStore",
"WorkflowVersionNotFoundError",
"WorkflowVersionRow",
"make_workflow_store",
]
def make_workflow_store(session_factory):
if session_factory is None:
return MemoryWorkflowStore()
return WorkflowRepository(session_factory)
@@ -0,0 +1,112 @@
"""Abstract workflow definition / version store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowDraftConflictError(Exception):
"""Raised when ``expected_revision`` does not match the stored draft revision."""
def __init__(self, workflow_id: str, current_revision: int) -> None:
super().__init__(f"draft conflict for {workflow_id}: current={current_revision}")
self.workflow_id = workflow_id
self.current_revision = current_revision
class WorkflowNotFoundError(Exception):
def __init__(self, workflow_id: str) -> None:
super().__init__(f"workflow not found: {workflow_id}")
self.workflow_id = workflow_id
class WorkflowVersionNotFoundError(Exception):
def __init__(self, version_id: str) -> None:
super().__init__(f"workflow version not found: {version_id}")
self.version_id = version_id
class WorkflowStore(ABC):
@abstractmethod
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
"""CAS update of ``draft_graph_json``. Raises ``WorkflowDraftConflictError``."""
raise NotImplementedError
@abstractmethod
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
"""Atomically CAS-update BOTH the execution graph and the canvas document.
A single conditional UPDATE bumps ``draft_revision`` and writes
``draft_graph_json`` + ``draft_canvas_schema_json`` together, so the two
representations can never diverge across revisions. Raises
``WorkflowDraftConflictError`` on a stale ``expected_revision``.
"""
raise NotImplementedError
@abstractmethod
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_version(self, version_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
"""Lightweight list for subworkflow resource catalog."""
raise NotImplementedError
@@ -0,0 +1,216 @@
"""In-memory workflow store for tests / no-DB mode."""
from __future__ import annotations
import copy
import uuid
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryWorkflowStore(WorkflowStore):
def __init__(self) -> None:
self._definitions: dict[str, dict[str, Any]] = {}
self._versions: dict[str, dict[str, Any]] = {}
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
rows = list(self._definitions.values())
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
elif not include_archived:
rows = [r for r in rows if r["status"] != "archived"]
rows.sort(key=lambda r: r.get("updated_at") or "", reverse=True)
return [
{k: v for k, v in r.items() if k not in ("draft_graph", "draft_canvas_schema", "next_version_number")}
for r in rows
]
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
row = self._definitions.get(workflow_id)
if row is None:
return None
out = copy.deepcopy(row)
if not include_draft:
out.pop("draft_graph", None)
return out
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
graph = copy.deepcopy(data.get("draft_graph") or data.get("graph") or {
"schemaVersion": "1.0",
"id": workflow_id,
"name": data.get("name") or "未命名工作流",
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
"edges": [],
"inputSchema": {"type": "object", "properties": {}},
"outputSchema": {"type": "object", "properties": {}},
"settings": {},
})
graph.setdefault("id", workflow_id)
now = _now()
row = {
"id": workflow_id,
"name": data.get("name") or graph.get("name") or "未命名工作流",
"description": data.get("description") or "",
"owner_id": data["owner_id"],
"status": data.get("status") or "draft",
"draft_revision": 0,
"next_version_number": 1,
"draft_graph": graph,
"draft_canvas_schema": str(data.get("draft_canvas_schema") or "{}"),
"created_at": now,
"updated_at": now,
}
self._definitions[workflow_id] = row
return copy.deepcopy(row)
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._definitions.get(workflow_id)
if row is None:
return None
if "name" in data and data["name"] is not None:
row["name"] = data["name"]
if "description" in data and data["description"] is not None:
row["description"] = data["description"]
if "status" in data and data["status"] is not None:
row["status"] = data["status"]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
return await self.update_definition(workflow_id, {"status": "archived"})
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by
row = self._definitions.get(workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row["draft_revision"] != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
payload = copy.deepcopy(graph)
payload.setdefault("id", workflow_id)
row["draft_graph"] = payload
row["draft_revision"] = expected_revision + 1
if payload.get("name"):
row["name"] = str(payload["name"])[:255]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by
row = self._definitions.get(workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row["draft_revision"] != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
payload = copy.deepcopy(graph)
payload.setdefault("id", workflow_id)
row["draft_graph"] = payload
row["draft_canvas_schema"] = canvas_schema or "{}"
row["draft_revision"] = expected_revision + 1
if payload.get("name"):
row["name"] = str(payload["name"])[:255]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
definition = self._definitions.get(workflow_id)
if definition is None:
raise WorkflowNotFoundError(workflow_id)
existing = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
# Allocate from the definition's own counter (mirrors the SQL store) so
# the number never goes backwards even if events are purged/replayed.
counter = int(definition.get("next_version_number") or 1)
version_number = max(counter, max((v["version_number"] for v in existing), default=0) + 1)
definition["next_version_number"] = version_number + 1
version_id = f"wv_{uuid.uuid4().hex[:12]}"
version = {
"id": version_id,
"workflow_id": workflow_id,
"version_number": version_number,
"graph": copy.deepcopy(graph),
"graph_hash": graph_hash,
"input_schema": graph.get("inputSchema") or graph.get("input_schema") or {},
"output_schema": graph.get("outputSchema") or graph.get("output_schema") or {},
"published_by": published_by,
"published_at": _now(),
"change_note": change_note or "",
}
self._versions[version_id] = version
definition["status"] = "active"
if canvas_schema_hash:
definition["published_canvas_schema_hash"] = str(canvas_schema_hash)[:64]
definition["updated_at"] = _now()
return copy.deepcopy(version)
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
rows = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
rows.sort(key=lambda r: r["version_number"], reverse=True)
return [{k: v for k, v in r.items() if k != "graph"} for r in rows]
async def get_version(self, version_id: str) -> dict[str, Any] | None:
row = self._versions.get(version_id)
return copy.deepcopy(row) if row is not None else None
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
by_wf: dict[str, dict[str, Any]] = {}
for version in self._versions.values():
wf_id = version["workflow_id"]
if exclude_workflow_id and wf_id == exclude_workflow_id:
continue
definition = self._definitions.get(wf_id)
if definition is None or definition.get("status") == "archived":
continue
current = by_wf.get(wf_id)
if current is None or version["version_number"] > current["version_number"]:
by_wf[wf_id] = {
"workflow_id": wf_id,
"name": definition["name"],
"version_id": version["id"],
"version_number": version["version_number"],
"graph_hash": version["graph_hash"],
"published_at": version["published_at"],
}
return sorted(by_wf.values(), key=lambda r: r.get("published_at") or "", reverse=True)
@@ -0,0 +1,69 @@
"""ORM models for workflow definitions and immutable published versions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowDefinitionRow(Base):
"""Mutable workflow metadata + current draft canvas."""
__tablename__ = "workflow_definitions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str] = mapped_column(Text, default="", nullable=False)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# draft | active | archived
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
draft_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
# Monotonic version allocator. "1" = no version published yet; publish takes
# the value then CAS-bumps it, replacing the old MAX(version_number)+1 race.
next_version_number: Mapped[int] = mapped_column(
Integer, nullable=False, default=1, server_default=sa_text("1")
)
draft_graph_json: Mapped[str] = mapped_column(PortableLongText(), default="{}", nullable=False)
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
# the executable WorkflowGraph lives in draft_graph_json above.
draft_canvas_schema_json: Mapped[str] = mapped_column(
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
)
# sha256 of the canvas document the current published version was compiled
# from (audit trail only; publish always freezes the execution graph).
published_canvas_schema_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowVersionRow(Base):
"""Immutable published graph snapshot. Runs bind to version id."""
__tablename__ = "workflow_versions"
__table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(
String(64),
ForeignKey("workflow_definitions.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
graph_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
graph_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
input_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
change_note: Mapped[str] = mapped_column(Text, default="", nullable=False)
@@ -0,0 +1,385 @@
"""SQLAlchemy-backed workflow definition / version store."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
# Concurrent publishers CAS-retry against the workflow's own counter; a handful
# of attempts is far more than any realistic contention window needs.
_PUBLISH_ATTEMPTS = 6
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _parse_graph(raw: str | dict[str, Any] | None) -> dict[str, Any]:
if raw is None:
return {}
if isinstance(raw, dict):
return raw
text = str(raw).strip()
if not text:
return {}
try:
data = json.loads(text)
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def _dump_graph(graph: dict[str, Any]) -> str:
return json.dumps(graph, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _definition_dict(row: WorkflowDefinitionRow, *, include_draft: bool) -> dict[str, Any]:
data = {
"id": row.id,
"name": row.name,
"description": row.description or "",
"owner_id": row.owner_id,
"status": row.status,
"draft_revision": row.draft_revision,
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
if include_draft:
data["draft_graph"] = _parse_graph(row.draft_graph_json)
# Canvas document as stored (opaque JSON text; "{}" when never saved).
data["draft_canvas_schema"] = row.draft_canvas_schema_json or "{}"
return data
def _version_dict(row: WorkflowVersionRow, *, include_graph: bool = True) -> dict[str, Any]:
data = {
"id": row.id,
"workflow_id": row.workflow_id,
"version_number": row.version_number,
"graph_hash": row.graph_hash,
"input_schema": row.input_schema or {},
"output_schema": row.output_schema or {},
"published_by": row.published_by,
"published_at": _iso(row.published_at),
"change_note": row.change_note or "",
}
if include_graph:
data["graph"] = _parse_graph(row.graph_json)
return data
class WorkflowRepository(WorkflowStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
stmt = select(WorkflowDefinitionRow)
if owner_id:
stmt = stmt.where(WorkflowDefinitionRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowDefinitionRow.status == status)
elif not include_archived:
stmt = stmt.where(WorkflowDefinitionRow.status != "archived")
stmt = stmt.order_by(WorkflowDefinitionRow.updated_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_definition_dict(row, include_draft=False) for row in result.scalars()]
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
return _definition_dict(row, include_draft=include_draft) if row is not None else None
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
graph = data.get("draft_graph") or data.get("graph") or {
"schemaVersion": "1.0",
"id": workflow_id,
"name": data.get("name") or "未命名工作流",
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
"edges": [],
"inputSchema": {"type": "object", "properties": {}},
"outputSchema": {"type": "object", "properties": {}},
"settings": {},
}
if isinstance(graph, dict) and not graph.get("id"):
graph = {**graph, "id": workflow_id}
if isinstance(graph, dict) and data.get("name") and not graph.get("name"):
graph = {**graph, "name": data["name"]}
row = WorkflowDefinitionRow(
id=workflow_id,
name=data.get("name") or graph.get("name") or "未命名工作流",
description=data.get("description") or "",
owner_id=data["owner_id"],
status=data.get("status") or "draft",
draft_revision=0,
draft_graph_json=_dump_graph(graph),
draft_canvas_schema_json=str(data.get("draft_canvas_schema") or "{}"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = _definition_dict(row, include_draft=True)
await session.commit()
return out
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
return None
if "name" in data and data["name"] is not None:
row.name = data["name"]
if "description" in data and data["description"] is not None:
row.description = data["description"]
if "status" in data and data["status"] is not None:
row.status = data["status"]
row.updated_at = datetime.now(UTC)
await session.flush()
out = _definition_dict(row, include_draft=True)
await session.commit()
return out
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
return await self.update_definition(workflow_id, {"status": "archived"})
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by # audit reserved for later
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row.draft_revision != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
payload = dict(graph)
payload.setdefault("id", workflow_id)
if row.name and not payload.get("name"):
payload["name"] = row.name
# Conditional UPDATE on the revision itself: two sessions that both
# read the same revision cannot both win — the loser's WHERE clause
# matches zero rows and surfaces a deterministic conflict instead of
# silently overwriting the other editor's draft.
values: dict[str, Any] = {
"draft_graph_json": _dump_graph(payload),
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
"updated_at": datetime.now(UTC),
}
if payload.get("name"):
values["name"] = str(payload["name"])[:255]
result = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.draft_revision == expected_revision,
)
.values(**values)
)
if result.rowcount == 0:
await session.rollback()
current = await session.get(WorkflowDefinitionRow, workflow_id)
if current is None:
raise WorkflowNotFoundError(workflow_id)
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
await session.commit()
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
assert fresh is not None
return _definition_dict(fresh, include_draft=True)
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by # audit reserved for later
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row.draft_revision != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
payload = dict(graph)
payload.setdefault("id", workflow_id)
if row.name and not payload.get("name"):
payload["name"] = row.name
# One conditional UPDATE writes graph + canvas + revision together:
# the two representations are always from the same save, and a
# concurrent editor loses deterministically instead of interleaving
# a new canvas onto someone else's graph.
values: dict[str, Any] = {
"draft_graph_json": _dump_graph(payload),
"draft_canvas_schema_json": canvas_schema or "{}",
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
"updated_at": datetime.now(UTC),
}
if payload.get("name"):
values["name"] = str(payload["name"])[:255]
result = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.draft_revision == expected_revision,
)
.values(**values)
)
if result.rowcount == 0:
await session.rollback()
current = await session.get(WorkflowDefinitionRow, workflow_id)
if current is None:
raise WorkflowNotFoundError(workflow_id)
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
await session.commit()
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
assert fresh is not None
return _definition_dict(fresh, include_draft=True)
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
"""Allocate the version number from the workflow's own counter.
``MAX(version_number) + 1`` races between concurrent publishes; here the
number is claimed by CAS-bumping ``next_version_number`` inside the same
transaction that inserts the version row, and the unique
``(workflow_id, version_number)`` constraint is the final safety net.
Concurrent publishers get adjacent, distinct numbers.
"""
last_error: Exception | None = None
for _ in range(_PUBLISH_ATTEMPTS):
try:
async with self._sf() as session:
definition = await session.get(WorkflowDefinitionRow, workflow_id)
if definition is None:
raise WorkflowNotFoundError(workflow_id)
counter_seen = int(definition.next_version_number or 1)
# Legacy rows got the column auto-added with server_default 1
# while versions already exist — never allocate below the max.
latest = (
await session.execute(
select(func.max(WorkflowVersionRow.version_number)).where(
WorkflowVersionRow.workflow_id == workflow_id
)
)
).scalar() or 0
number = max(counter_seen, int(latest) + 1)
claimed = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.next_version_number == counter_seen,
)
.values(next_version_number=number + 1)
)
if claimed.rowcount == 0:
# Another publish landed first — release nothing (the
# counter bump never happened) and retry with fresh state.
await session.rollback()
last_error = None
continue
version = WorkflowVersionRow(
id=f"wv_{uuid.uuid4().hex[:12]}",
workflow_id=workflow_id,
version_number=number,
graph_json=_dump_graph(graph),
graph_hash=graph_hash,
input_schema=graph.get("inputSchema") or graph.get("input_schema") or {},
output_schema=graph.get("outputSchema") or graph.get("output_schema") or {},
published_by=published_by,
change_note=change_note or "",
)
definition.status = "active"
definition.updated_at = datetime.now(UTC)
if canvas_schema_hash:
definition.published_canvas_schema_hash = canvas_schema_hash[:64]
session.add(version)
await session.flush()
out = _version_dict(version, include_graph=True)
await session.commit()
return out
except IntegrityError as exc: # pragma: no cover - CAS makes this near-impossible
last_error = exc
continue
if last_error is not None:
raise last_error
raise RuntimeError(f"failed to allocate a workflow version number for {workflow_id}")
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
stmt = (
select(WorkflowVersionRow)
.where(WorkflowVersionRow.workflow_id == workflow_id)
.order_by(WorkflowVersionRow.version_number.desc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_version_dict(row, include_graph=False) for row in result.scalars()]
async def get_version(self, version_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowVersionRow, version_id)
return _version_dict(row, include_graph=True) if row is not None else None
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
stmt = (
select(WorkflowVersionRow, WorkflowDefinitionRow)
.join(WorkflowDefinitionRow, WorkflowDefinitionRow.id == WorkflowVersionRow.workflow_id)
.where(WorkflowDefinitionRow.status != "archived")
.order_by(WorkflowVersionRow.published_at.desc())
)
if exclude_workflow_id:
stmt = stmt.where(WorkflowVersionRow.workflow_id != exclude_workflow_id)
async with self._sf() as session:
result = await session.execute(stmt)
out: list[dict[str, Any]] = []
seen: set[str] = set()
for version, definition in result.all():
# Latest version per workflow only.
if version.workflow_id in seen:
continue
seen.add(version.workflow_id)
out.append(
{
"workflow_id": definition.id,
"name": definition.name,
"version_id": version.id,
"version_number": version.version_number,
"graph_hash": version.graph_hash,
"published_at": _iso(version.published_at),
}
)
return out
@@ -0,0 +1,64 @@
"""Workflow Studio protocol package (phase 0 — frozen contracts).
Execution, persistence, and Gateway routers land in later phases.
This package must never import ``app.*``.
"""
from deerflow.workflows.errors import (
ALL_WORKFLOW_ERROR_CODES,
WorkflowError,
WorkflowErrorBody,
WorkflowErrorCode,
)
from deerflow.workflows.events import (
ALL_WORKFLOW_EVENT_TYPES,
TERMINAL_EVENT_TYPES,
WorkflowEventEnvelope,
WorkflowEventSink,
WorkflowEventType,
)
from deerflow.workflows.schemas import (
ALL_NODE_TYPES,
ALL_RUN_STATUSES,
SCHEMA_VERSION,
TERMINAL_RUN_STATUSES,
ArtifactRef,
NodeResult,
NodeType,
ResumeRunRequest,
RunStatus,
StartRunRequest,
WorkflowEdge,
WorkflowGraph,
WorkflowGraphSettings,
WorkflowNode,
)
from deerflow.workflows.validator import assert_publishable, validate_workflow_graph
__all__ = [
"ALL_NODE_TYPES",
"ALL_RUN_STATUSES",
"ALL_WORKFLOW_ERROR_CODES",
"ALL_WORKFLOW_EVENT_TYPES",
"SCHEMA_VERSION",
"TERMINAL_EVENT_TYPES",
"TERMINAL_RUN_STATUSES",
"ArtifactRef",
"NodeResult",
"NodeType",
"ResumeRunRequest",
"RunStatus",
"StartRunRequest",
"WorkflowEdge",
"WorkflowError",
"WorkflowErrorBody",
"WorkflowErrorCode",
"WorkflowEventEnvelope",
"WorkflowEventSink",
"WorkflowEventType",
"WorkflowGraph",
"WorkflowGraphSettings",
"WorkflowNode",
"assert_publishable",
"validate_workflow_graph",
]
@@ -0,0 +1,150 @@
"""Stable workflow error codes and exception types (protocol v1).
User-visible ``message`` must never include stack traces, SQL passwords,
full response bodies, or internal host paths. Full exceptions go to
controlled server logs only.
"""
from __future__ import annotations
from typing import Any, Literal
from pydantic import BaseModel, Field
WorkflowErrorCode = Literal[
# graph / publish
"WORKFLOW_SCHEMA_INVALID",
"WORKFLOW_GRAPH_CYCLE",
"WORKFLOW_NODE_NOT_FOUND",
"WORKFLOW_EDGE_INVALID",
"WORKFLOW_EXPRESSION_INVALID",
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
"WORKFLOW_RESOURCE_MISSING",
"WORKFLOW_RESOURCE_FORBIDDEN",
"WORKFLOW_LIMIT_EXCEEDED",
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"WORKFLOW_DRAFT_CONFLICT",
"WORKFLOW_VERSION_NOT_FOUND",
# run lifecycle
"WORKFLOW_RUN_NOT_FOUND",
"WORKFLOW_RUN_CONFLICT",
"WORKFLOW_RUN_NOT_RESUMABLE",
"WORKFLOW_RESUME_TOKEN_INVALID",
"WORKFLOW_CANCELLED",
"WORKFLOW_TIMEOUT",
"WORKFLOW_MAX_STEPS",
"WORKFLOW_MAX_LOOP_ITERATIONS",
"WORKFLOW_BUDGET_EXCEEDED",
# node categories
"WORKFLOW_INPUT_INVALID",
"WORKFLOW_AGENT_FAILED",
"WORKFLOW_SKILL_FAILED",
"WORKFLOW_HTTP_POLICY_DENIED",
"WORKFLOW_HTTP_FAILED",
"WORKFLOW_SQL_POLICY_DENIED",
"WORKFLOW_SQL_FAILED",
"WORKFLOW_CODE_SANDBOX_FAILED",
"WORKFLOW_TRANSFORM_FAILED",
"WORKFLOW_CONDITION_FAILED",
"WORKFLOW_MERGE_FAILED",
"WORKFLOW_HUMAN_INPUT_INVALID",
"WORKFLOW_SUBWORKFLOW_FAILED",
"WORKFLOW_LOOP_FAILED",
# auth / embed
"WORKFLOW_FORBIDDEN",
"WORKFLOW_EMBED_TICKET_INVALID",
"WORKFLOW_EMBED_ORIGIN_DENIED",
# generic
"WORKFLOW_INTERNAL",
]
ALL_WORKFLOW_ERROR_CODES: tuple[WorkflowErrorCode, ...] = (
"WORKFLOW_SCHEMA_INVALID",
"WORKFLOW_GRAPH_CYCLE",
"WORKFLOW_NODE_NOT_FOUND",
"WORKFLOW_EDGE_INVALID",
"WORKFLOW_EXPRESSION_INVALID",
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
"WORKFLOW_RESOURCE_MISSING",
"WORKFLOW_RESOURCE_FORBIDDEN",
"WORKFLOW_LIMIT_EXCEEDED",
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"WORKFLOW_DRAFT_CONFLICT",
"WORKFLOW_VERSION_NOT_FOUND",
"WORKFLOW_RUN_NOT_FOUND",
"WORKFLOW_RUN_CONFLICT",
"WORKFLOW_RUN_NOT_RESUMABLE",
"WORKFLOW_RESUME_TOKEN_INVALID",
"WORKFLOW_CANCELLED",
"WORKFLOW_TIMEOUT",
"WORKFLOW_MAX_STEPS",
"WORKFLOW_MAX_LOOP_ITERATIONS",
"WORKFLOW_BUDGET_EXCEEDED",
"WORKFLOW_INPUT_INVALID",
"WORKFLOW_AGENT_FAILED",
"WORKFLOW_SKILL_FAILED",
"WORKFLOW_HTTP_POLICY_DENIED",
"WORKFLOW_HTTP_FAILED",
"WORKFLOW_SQL_POLICY_DENIED",
"WORKFLOW_SQL_FAILED",
"WORKFLOW_CODE_SANDBOX_FAILED",
"WORKFLOW_TRANSFORM_FAILED",
"WORKFLOW_CONDITION_FAILED",
"WORKFLOW_MERGE_FAILED",
"WORKFLOW_HUMAN_INPUT_INVALID",
"WORKFLOW_SUBWORKFLOW_FAILED",
"WORKFLOW_LOOP_FAILED",
"WORKFLOW_FORBIDDEN",
"WORKFLOW_EMBED_TICKET_INVALID",
"WORKFLOW_EMBED_ORIGIN_DENIED",
"WORKFLOW_INTERNAL",
)
class WorkflowErrorBody(BaseModel):
"""Wire-format error body shared by API responses and ``run.failed`` events."""
code: WorkflowErrorCode
message: str
retryable: bool = False
node_id: str | None = Field(default=None, alias="nodeId")
details: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowError(Exception):
"""Raised by validators / node executors; convertible to :class:`WorkflowErrorBody`."""
def __init__(
self,
code: WorkflowErrorCode,
message: str,
*,
retryable: bool = False,
node_id: str | None = None,
details: dict[str, Any] | None = None,
) -> None:
super().__init__(message)
self.code = code
self.message = message
self.retryable = retryable
self.node_id = node_id
self.details = details or {}
def to_body(self) -> WorkflowErrorBody:
return WorkflowErrorBody(
code=self.code,
message=self.message,
retryable=self.retryable,
node_id=self.node_id,
details=self.details,
)
__all__ = [
"ALL_WORKFLOW_ERROR_CODES",
"WorkflowError",
"WorkflowErrorBody",
"WorkflowErrorCode",
]
@@ -0,0 +1,158 @@
"""Workflow SSE event protocol v1.0.
Invariant: **persist first, then publish**. ``seq`` is monotonic per run.
Heartbeat comments (``: heartbeat``) are live-only and never stored.
Canonical ``data`` payloads (frozen contract — REST replay and SSE must agree):
node.tool.started {"name": "web_search", "toolCallId": "call_001",
"title": "联网检索", "args": {"path": "..."}}
node.tool.finished {"name": "web_search", "toolCallId": "call_001",
"status": "succeeded" | "failed", "summary": "获得 12 条结果"}
run.awaiting_input {"resumeToken": "one-time-token", "prompt": "请确认是否发布",
"formSchema": {...}, "actions": ["approve", "reject"]}
artifact.created {"artifactId": "artifact_001", "name": "weekly-report.docx",
"kind": "file" | "markdown" | "json" | "table",
"mimeType": "...", "sizeBytes": 1234, "preview": "..."}
node.started {"nodeType": ..., "name": ..., "attempt": 1,
"agent": {...}, "parallelGroupId": "...", "roundIndex": 2}
Rules: tool identity is ``name`` (never ``toolName``); outcomes are ``status``
(never ``ok``); human-input descriptors are flattened (never nested under
``pendingInput``); artifacts never carry filesystem paths; agent nodes expose
display metadata only (no system prompts, skill whitelists or credentials).
"""
from __future__ import annotations
from datetime import UTC, datetime
from typing import Any, Literal, Protocol, runtime_checkable
from pydantic import BaseModel, Field
SCHEMA_VERSION = "1.0"
WorkflowEventType = Literal[
"run.created",
"run.queued",
"run.started",
"node.queued",
"node.started",
"node.progress",
"node.output.delta",
"node.tool.started",
"node.tool.finished",
"artifact.created",
"node.completed",
"node.failed",
"run.awaiting_input",
"run.resumed",
"run.cancel_requested",
"run.cancelled",
"run.completed",
"run.failed",
]
ALL_WORKFLOW_EVENT_TYPES: tuple[WorkflowEventType, ...] = (
"run.created",
"run.queued",
"run.started",
"node.queued",
"node.started",
"node.progress",
"node.output.delta",
"node.tool.started",
"node.tool.finished",
"artifact.created",
"node.completed",
"node.failed",
"run.awaiting_input",
"run.resumed",
"run.cancel_requested",
"run.cancelled",
"run.completed",
"run.failed",
)
TERMINAL_EVENT_TYPES: frozenset[WorkflowEventType] = frozenset({"run.completed", "run.failed", "run.cancelled"})
_KIND_BY_MIME: tuple[tuple[tuple[str, ...], str], ...] = (
(("text/markdown", "text/x-markdown"), "markdown"),
(("application/json", "text/json", "application/javascript"), "json"),
(("text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "table"),
)
def artifact_kind(mime_type: str) -> str:
"""Coarse wire-level artifact category: file | markdown | json | table."""
lowered = (mime_type or "").lower()
for prefixes, kind in _KIND_BY_MIME:
if any(lowered.startswith(prefix) for prefix in prefixes):
return kind
return "file"
class WorkflowEventEnvelope(BaseModel):
"""Common envelope for every persisted / SSE workflow event."""
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
run_id: str = Field(alias="runId")
workflow_id: str = Field(alias="workflowId")
version_id: str = Field(alias="versionId")
seq: int = 0
event_type: WorkflowEventType = Field(alias="event")
node_id: str | None = Field(default=None, alias="nodeId")
node_run_id: str | None = Field(default=None, alias="nodeRunId")
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
data: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
def to_sse_dict(self) -> dict[str, Any]:
"""JSON object for the SSE ``data:`` line (camelCase wire fields)."""
return {
"schemaVersion": self.schema_version,
"runId": self.run_id,
"workflowId": self.workflow_id,
"versionId": self.version_id,
"seq": self.seq,
"event": self.event_type,
"nodeId": self.node_id,
"nodeRunId": self.node_run_id,
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
"data": self.data,
}
def to_sse_frame(self) -> str:
"""Full SSE frame including ``id`` / ``event`` / ``data`` lines."""
import json
payload = json.dumps(self.to_sse_dict(), ensure_ascii=False, separators=(",", ":"))
return f"id: {self.seq}\nevent: {self.event_type}\ndata: {payload}\n\n"
@runtime_checkable
class WorkflowEventSink(Protocol):
"""Persist-then-publish sink used by the future executor."""
async def emit(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> WorkflowEventEnvelope:
"""Persist event (assign ``seq``), then fan out to live subscribers."""
...
__all__ = [
"ALL_WORKFLOW_EVENT_TYPES",
"SCHEMA_VERSION",
"TERMINAL_EVENT_TYPES",
"WorkflowEventEnvelope",
"WorkflowEventSink",
"WorkflowEventType",
"artifact_kind",
]
@@ -0,0 +1,264 @@
"""Safe expression / template evaluation for workflow node configs.
Two surfaces, both intentionally tiny:
* ``render_template("… {{ nodes.a.data.title }} …", state)`` — string
interpolation over a whitelisted state tree.
* ``evaluate_condition("nodes.a.data.score > 0.5", state)`` — boolean
predicate over the same tree.
Neither uses :func:`eval`. The condition grammar is a single comparison or a
chain joined by ``and`` / ``or``; operands are either a state path or a JSON
literal. Anything else raises ``WORKFLOW_EXPRESSION_INVALID`` so a malformed
graph fails loudly at publish time instead of silently at run time.
"""
from __future__ import annotations
import json
import re
from typing import Any
from deerflow.workflows.errors import WorkflowError
_TEMPLATE_RE = re.compile(r"\{\{\s*([^{}]+?)\s*\}\}")
_ROOT_KEYS = ("inputs", "nodes", "run", "loop", "env")
_PATH_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\])*$")
_COMPARATORS = ("==", "!=", ">=", "<=", ">", "<", " in ", " contains ")
_MAX_RENDER_CHARS = 200_000
def _invalid(expr: str, reason: str) -> WorkflowError:
return WorkflowError(
"WORKFLOW_EXPRESSION_INVALID",
f"表达式不合法:{reason}",
details={"expression": expr[:200]},
)
def resolve_path(path: str, state: dict[str, Any]) -> Any:
"""Read ``a.b[0].c`` from the state tree. Missing segments yield ``None``."""
expr = path.strip()
if not _PATH_RE.match(expr):
raise _invalid(expr, "路径格式不合法")
root = expr.split(".", 1)[0].split("[", 1)[0]
if root not in _ROOT_KEYS:
raise _invalid(expr, f"根节点必须是 {'/'.join(_ROOT_KEYS)} 之一")
current: Any = state
for segment in re.findall(r"[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\]", expr):
if current is None:
return None
if segment.startswith("["):
index = int(segment[1:-1])
if not isinstance(current, (list, tuple)) or index >= len(current):
return None
current = current[index]
elif isinstance(current, dict):
current = current.get(segment)
else:
return None
return current
def _stringify(value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, bool):
return "true" if value else "false"
if isinstance(value, (int, float)):
return str(value)
return json.dumps(value, ensure_ascii=False, default=str)
def render_template(template: str, state: dict[str, Any]) -> str:
"""Replace every ``{{ path }}`` with its stringified state value."""
if not template:
return ""
def _sub(match: re.Match[str]) -> str:
return _stringify(resolve_path(match.group(1), state))
rendered = _TEMPLATE_RE.sub(_sub, template)
if len(rendered) > _MAX_RENDER_CHARS:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
"模板渲染结果超出长度上限",
details={"limit": _MAX_RENDER_CHARS},
)
return rendered
def render_value(value: Any, state: dict[str, Any]) -> Any:
"""Recursively render templates inside dict / list / str structures.
A string that is exactly one ``{{ path }}`` keeps the resolved value's
native type instead of being stringified.
"""
if isinstance(value, str):
match = _TEMPLATE_RE.fullmatch(value.strip())
if match:
return resolve_path(match.group(1), state)
return render_template(value, state)
if isinstance(value, dict):
return {k: render_value(v, state) for k, v in value.items()}
if isinstance(value, list):
return [render_value(v, state) for v in value]
return value
def _operand(token: str, state: dict[str, Any]) -> Any:
text = token.strip()
if not text:
raise _invalid(token, "操作数为空")
match = _TEMPLATE_RE.fullmatch(text)
if match:
return resolve_path(match.group(1), state)
lowered = text.lower()
if lowered in ("true", "false"):
return lowered == "true"
if lowered in ("null", "none"):
return None
if (text[0], text[-1]) in (('"', '"'), ("'", "'")) and len(text) >= 2:
return text[1:-1]
try:
return json.loads(text)
except json.JSONDecodeError:
pass
if _PATH_RE.match(text) and text.split(".", 1)[0].split("[", 1)[0] in _ROOT_KEYS:
return resolve_path(text, state)
return text
def _truthy(value: Any) -> bool:
if isinstance(value, str):
return value.strip().lower() not in ("", "false", "0", "null", "none")
return bool(value)
def _compare(left: Any, op: str, right: Any) -> bool:
if op == "==":
return left == right
if op == "!=":
return left != right
if op == "in":
try:
return left in right # type: ignore[operator]
except TypeError:
return False
if op == "contains":
try:
return right in left # type: ignore[operator]
except TypeError:
return False
try:
if op == ">":
return left > right
if op == "<":
return left < right
if op == ">=":
return left >= right
if op == "<=":
return left <= right
except TypeError:
return False
raise _invalid(op, f"不支持的比较运算符 {op}")
def _split_top_level(expr: str, keyword: str) -> list[str] | None:
"""Split on ``keyword`` outside quotes / braces; None when absent."""
parts: list[str] = []
depth = 0
quote: str | None = None
buffer: list[str] = []
index = 0
pattern = f" {keyword} "
while index < len(expr):
char = expr[index]
if quote:
buffer.append(char)
if char == quote:
quote = None
index += 1
continue
if char in ('"', "'"):
quote = char
buffer.append(char)
index += 1
continue
if char in "[{(":
depth += 1
elif char in "]})":
depth -= 1
if depth == 0 and expr[index : index + len(pattern)] == pattern:
parts.append("".join(buffer))
buffer = []
index += len(pattern)
continue
buffer.append(char)
index += 1
if not parts:
return None
parts.append("".join(buffer))
return parts
def evaluate_condition(expression: str, state: dict[str, Any]) -> bool:
"""Evaluate a boolean predicate without ``eval``."""
expr = (expression or "").strip()
if not expr:
return False
or_parts = _split_top_level(expr, "or")
if or_parts:
return any(evaluate_condition(part, state) for part in or_parts)
and_parts = _split_top_level(expr, "and")
if and_parts:
return all(evaluate_condition(part, state) for part in and_parts)
if expr.startswith("not "):
return not evaluate_condition(expr[4:], state)
if expr.startswith("(") and expr.endswith(")"):
return evaluate_condition(expr[1:-1], state)
for comparator in _COMPARATORS:
op = comparator.strip()
index = expr.find(comparator)
if index <= 0:
continue
left = _operand(expr[:index], state)
right = _operand(expr[index + len(comparator) :], state)
return _compare(left, op, right)
return _truthy(_operand(expr, state))
_NODES_IDENT_RE = re.compile(r"(?:\{\{\s*)?nodes\.([A-Za-z_][A-Za-z0-9_\-]*)")
def collect_referenced_nodes(value: Any) -> set[str]:
"""Node ids referenced through ``nodes.<id>`` anywhere in a config tree.
Covers both ``{{ nodes.x.data }}`` templates and bare condition paths
such as ``nodes.x.data.score > 0.5``.
"""
found: set[str] = set()
if isinstance(value, str):
found.update(_NODES_IDENT_RE.findall(value))
elif isinstance(value, dict):
for item in value.values():
found |= collect_referenced_nodes(item)
elif isinstance(value, list):
for item in value:
found |= collect_referenced_nodes(item)
return found
__all__ = [
"collect_referenced_nodes",
"evaluate_condition",
"render_template",
"render_value",
"resolve_path",
]
@@ -0,0 +1,65 @@
"""Node executors for the workflow runtime."""
from __future__ import annotations
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.nodes.agents import (
AgentNodeExecutor,
SkillNodeExecutor,
SubworkflowNodeExecutor,
)
from deerflow.workflows.nodes.base import ACTIVE_PORTS, NodeExecutor, NodeRegistry
from deerflow.workflows.nodes.core import (
ConditionNodeExecutor,
LoopNodeExecutor,
MergeNodeExecutor,
OutputNodeExecutor,
StartNodeExecutor,
TransformNodeExecutor,
)
from deerflow.workflows.nodes.human import HumanInputNodeExecutor
from deerflow.workflows.nodes.integrations import (
CodeNodeExecutor,
HttpNodeExecutor,
SqlReadNodeExecutor,
)
__all__ = [
"ACTIVE_PORTS",
"AgentNodeExecutor",
"CodeNodeExecutor",
"ConditionNodeExecutor",
"HttpNodeExecutor",
"HumanInputNodeExecutor",
"LoopNodeExecutor",
"MergeNodeExecutor",
"NodeExecutor",
"NodeRegistry",
"OutputNodeExecutor",
"SkillNodeExecutor",
"SqlReadNodeExecutor",
"StartNodeExecutor",
"SubworkflowNodeExecutor",
"TransformNodeExecutor",
"build_default_registry",
]
def build_default_registry(config: WorkflowConfig | None = None) -> NodeRegistry:
"""Registry with every executable node type wired up."""
cfg = config or WorkflowConfig()
registry = NodeRegistry()
registry.register("start", StartNodeExecutor())
registry.register("output", OutputNodeExecutor())
registry.register("transform", TransformNodeExecutor())
registry.register("condition", ConditionNodeExecutor())
registry.register("merge", MergeNodeExecutor())
registry.register("loop", LoopNodeExecutor())
registry.register("human_input", HumanInputNodeExecutor())
registry.register("http", HttpNodeExecutor(cfg))
registry.register("sql_read", SqlReadNodeExecutor(cfg))
registry.register("code", CodeNodeExecutor(cfg))
registry.register("agent", AgentNodeExecutor())
registry.register("skill", SkillNodeExecutor())
registry.register("subworkflow", SubworkflowNodeExecutor())
return registry
@@ -0,0 +1,361 @@
"""LLM-backed nodes: ``agent``, ``skill``, ``subworkflow``.
These are the only nodes that call the DeerFlow LangGraph lead agent. The agent
invocation itself is injected (``deps.run_agent``) so the harness keeps its
import firewall; this module owns prompt rendering, streaming fan-out, JSON
response coercion, and error typing.
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template, render_value
from deerflow.workflows.nodes.base import require_dep
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schemas import ArtifactRef, NodeResult, WorkflowNode
logger = logging.getLogger(__name__)
_JSON_BLOCK = re.compile(r"```(?:json)?\s*(.+?)\s*```", re.DOTALL)
_DELTA_FLUSH_SECONDS = 0.05
_DELTA_FLUSH_CHARS = 1024
class _DeltaCoalescer:
"""Merge model tokens into 50ms / 1KB event payloads (§9.5)."""
def __init__(self, ctx: RunContext, node_id: str) -> None:
self._ctx = ctx
self._node_id = node_id
self._parts: list[str] = []
self._chars = 0
self._task: asyncio.Task | None = None
self._lock = asyncio.Lock()
async def push(self, chunk: str) -> None:
if not chunk:
return
async with self._lock:
self._parts.append(chunk)
self._chars += len(chunk)
if self._chars >= _DELTA_FLUSH_CHARS:
await self._flush_unlocked()
return
if self._task is None or self._task.done():
self._task = asyncio.create_task(self._delayed())
async def _delayed(self) -> None:
await asyncio.sleep(_DELTA_FLUSH_SECONDS)
async with self._lock:
await self._flush_unlocked()
async def _flush_unlocked(self) -> None:
if not self._parts:
return
text = "".join(self._parts)
self._parts.clear()
self._chars = 0
await self._ctx.emit_event(
"node.output.delta",
data={"channel": "answer", "delta": text, "text": text},
node_id=self._node_id,
)
async def flush(self) -> None:
task = self._task
if task is not None and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
async with self._lock:
await self._flush_unlocked()
def _extract_json(text: str) -> Any:
candidate = text.strip()
match = _JSON_BLOCK.search(candidate)
if match:
candidate = match.group(1).strip()
else:
start = min((i for i in (candidate.find("{"), candidate.find("[")) if i >= 0), default=-1)
if start > 0:
candidate = candidate[start:]
try:
return json.loads(candidate)
except json.JSONDecodeError:
return None
_SAFE_ARG_STRING_CHARS = 160
_SAFE_ARG_MAX_KEYS = 12
def _safe_tool_args(args: Any) -> dict[str, Any]:
"""Minimal safe projection of tool-call args for the event log.
Scalars only (strings truncated), no nested containers, bounded key count —
enough for the canvas to label the step without leaking full payloads such
as entire documents, prompts or credentials that a tool may carry."""
if not isinstance(args, dict):
return {}
out: dict[str, Any] = {}
for key, value in list(args.items())[:_SAFE_ARG_MAX_KEYS]:
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
continue
if isinstance(value, str):
value = value[:_SAFE_ARG_STRING_CHARS]
out[str(key)[:64]] = value
return out
class _AgentInvocation:
"""Shared plumbing for ``agent`` and ``skill`` (agent_skill mode)."""
async def _invoke(
self,
node: WorkflowNode,
ctx: RunContext,
*,
agent_id: str,
prompt: str,
skill_names: list[str],
response_mode: str,
response_schema: dict[str, Any],
thread_mode: str,
failure_code: str,
) -> NodeResult:
run_agent = require_dep(ctx.deps.run_agent, node=node, what="智能体执行")
coalescer = _DeltaCoalescer(ctx, node.id)
async def on_delta(chunk: str) -> None:
await coalescer.push(chunk)
async def on_tool(event: dict[str, Any]) -> None:
phase = str(event.get("phase") or "started")
name = str(event.get("name") or "")
call_id = str(event.get("id") or "")
if phase == "started":
await ctx.emit_event(
"node.tool.started",
data={
"name": name,
"toolCallId": call_id,
"title": str(event.get("title") or name),
"args": _safe_tool_args(event.get("args")),
},
node_id=node.id,
)
return
ok = event.get("ok")
await ctx.emit_event(
"node.tool.finished",
data={
"name": name,
"toolCallId": call_id,
"status": "succeeded" if ok in (True, None) else "failed",
"summary": event.get("summary"),
},
node_id=node.id,
)
try:
try:
outcome = await run_agent(
run_id=ctx.run_id,
node_id=node.id,
owner_id=ctx.owner_id,
agent_id=agent_id,
prompt=prompt,
skill_names=skill_names,
thread_mode=thread_mode,
cancel=ctx.cancel,
on_delta=on_delta,
on_tool=on_tool,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
except Exception as exc: # noqa: BLE001 - agent failures are expected
logger.warning("workflow agent node %s failed", node.id, exc_info=True)
raise WorkflowError(
failure_code,
"智能体执行失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
finally:
await coalescer.flush()
text = str(outcome.get("text") or "")
data: dict[str, Any] = {"text": text}
if response_mode == "json":
payload = outcome.get("json")
if payload is None:
payload = _extract_json(text)
if payload is None:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"智能体未返回可解析的 JSON",
retryable=True,
node_id=node.id,
details={"preview": text[:500]},
)
required = response_schema.get("required") if isinstance(response_schema, dict) else None
if isinstance(required, list) and isinstance(payload, dict):
missing = [str(k) for k in required if payload.get(str(k)) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
f"智能体返回缺少字段:{'、'.join(missing)}",
retryable=True,
node_id=node.id,
details={"missing": missing},
)
data["json"] = payload
artifacts: list[ArtifactRef] = []
for raw in outcome.get("artifacts") or []:
if not isinstance(raw, dict) or not raw.get("name"):
continue
ref = await ctx.register_artifact(
node,
name=str(raw.get("name")),
mime_type=str(raw.get("mime_type") or raw.get("mimeType") or "application/octet-stream"),
path=str(raw.get("path") or ""),
size_bytes=raw.get("size_bytes") or raw.get("sizeBytes"),
preview=raw.get("preview"),
)
if ref is not None:
artifacts.append(ref)
return NodeResult(
data=data,
messages=[m for m in (outcome.get("messages") or []) if isinstance(m, dict)],
artifacts=artifacts,
metadata={
"agentId": agent_id,
"threadId": outcome.get("thread_id"),
"usage": outcome.get("usage") or {},
},
)
class AgentNodeExecutor(_AgentInvocation):
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
if not agent_id:
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点未选择智能体", node_id=node.id)
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
prompt = render_template(template, ctx.state()).strip()
if not prompt:
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点提示词为空", node_id=node.id)
schema = node.config.get("responseSchema") or node.config.get("response_schema") or {}
return await self._invoke(
node,
ctx,
agent_id=agent_id,
prompt=prompt,
skill_names=[str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])],
response_mode=str(node.config.get("responseMode") or node.config.get("response_mode") or "text"),
response_schema=schema if isinstance(schema, dict) else {},
thread_mode=str(node.config.get("threadMode") or node.config.get("thread_mode") or "isolated_per_node"),
failure_code="WORKFLOW_AGENT_FAILED",
)
class SkillNodeExecutor(_AgentInvocation):
"""``agent_skill`` runs the agent with a restricted skill set.
``callable_skill`` (direct script invocation) is not executable yet."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
mode = str(node.config.get("mode") or "agent_skill")
if mode != "agent_skill":
raise WorkflowError(
"WORKFLOW_SKILL_FAILED",
"callable_skill 模式暂未开放执行,请改用 agent_skill",
node_id=node.id,
details={"mode": mode},
)
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
skill_names = [str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])]
if not agent_id or not skill_names:
raise WorkflowError("WORKFLOW_SKILL_FAILED", "技能节点需要同时指定智能体与技能", node_id=node.id)
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
prompt = render_template(template, ctx.state()).strip() or f"请使用技能 {skill_names[0]} 完成任务。"
return await self._invoke(
node,
ctx,
agent_id=agent_id,
prompt=prompt,
skill_names=skill_names,
response_mode="text",
response_schema={},
thread_mode="isolated_per_node",
failure_code="WORKFLOW_SKILL_FAILED",
)
class SubworkflowNodeExecutor:
"""Runs a published child workflow synchronously and folds in its output."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
run_subworkflow = require_dep(ctx.deps.run_subworkflow, node=node, what="子工作流执行")
workflow_id = str(node.config.get("workflowId") or node.config.get("workflow_id") or "")
version_id = str(node.config.get("versionId") or node.config.get("version_id") or "")
if not workflow_id or not version_id:
raise WorkflowError("WORKFLOW_SUBWORKFLOW_FAILED", "子工作流节点缺少 workflowId/versionId", node_id=node.id)
mapping = node.config.get("inputMapping") or node.config.get("input_mapping") or {}
state = ctx.state()
inputs = {str(key): render_value(template, state) for key, template in mapping.items()} if isinstance(mapping, dict) else {}
await ctx.emit_event(
"node.progress",
data={"phase": "subworkflow.start", "workflowId": workflow_id, "versionId": version_id},
node_id=node.id,
)
try:
outcome = await run_subworkflow(
parent_run_id=ctx.run_id,
node_id=node.id,
owner_id=ctx.owner_id,
workflow_id=workflow_id,
version_id=version_id,
inputs=inputs,
cancel=ctx.cancel,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
except Exception as exc: # noqa: BLE001
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
"子工作流执行失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
status = str(outcome.get("status") or "completed")
if status != "completed":
raise WorkflowError(
"WORKFLOW_SUBWORKFLOW_FAILED",
f"子工作流未成功完成({status})",
node_id=node.id,
details={"childRunId": outcome.get("run_id"), "status": status},
)
output = outcome.get("output")
return NodeResult(
data=output if isinstance(output, dict) else {"value": output},
metadata={"childRunId": outcome.get("run_id"), "workflowId": workflow_id},
)
__all__ = ["AgentNodeExecutor", "SkillNodeExecutor", "SubworkflowNodeExecutor"]
@@ -0,0 +1,52 @@
"""Node executor protocol and registry."""
from __future__ import annotations
from typing import Any, Protocol, runtime_checkable
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.ports import ACTIVE_PORTS
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schemas import NodeResult, NodeType, WorkflowNode
@runtime_checkable
class NodeExecutor(Protocol):
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult: ...
class NodeRegistry:
"""Maps node type → executor. Unknown types fail with a typed error."""
def __init__(self) -> None:
self._executors: dict[str, NodeExecutor] = {}
def register(self, node_type: NodeType | str, executor: NodeExecutor) -> None:
self._executors[str(node_type)] = executor
def get(self, node_type: str) -> NodeExecutor:
executor = self._executors.get(node_type)
if executor is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
f"节点类型暂不支持执行:{node_type}",
details={"nodeType": node_type},
)
return executor
def supported(self) -> list[str]:
return sorted(self._executors)
def require_dep(value: Any, *, node: WorkflowNode, what: str) -> Any:
if value is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
f"运行环境缺少{what}能力,无法执行该节点",
node_id=node.id,
details={"capability": what},
)
return value
__all__ = ["ACTIVE_PORTS", "NodeExecutor", "NodeRegistry", "require_dep"]
@@ -0,0 +1,238 @@
"""Deterministic control / data nodes: start, output, transform, condition, merge, loop."""
from __future__ import annotations
import json
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import evaluate_condition, render_template, render_value
from deerflow.workflows.nodes.base import ACTIVE_PORTS
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.schema_validation import schema_issues
from deerflow.workflows.schemas import NodeResult, WorkflowNode
DEFAULT_PORT = "else"
LOOP_CONTINUE_PORT = "continue"
LOOP_DONE_PORT = "done"
def _required_keys(schema: dict[str, Any]) -> list[str]:
required = schema.get("required")
if isinstance(required, list):
return [str(k) for k in required]
return []
class StartNodeExecutor:
"""Validates run inputs against the graph input schema and exposes them."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
schema = ctx.graph.input_schema or {}
missing = [key for key in _required_keys(schema) if ctx.inputs.get(key) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_INPUT_INVALID",
f"缺少必填输入:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
# Full JSON Schema pass (types / enums / nested objects) beyond the
# friendly missing-key check above.
issues = schema_issues(ctx.inputs, schema)
if issues:
raise WorkflowError(
"WORKFLOW_INPUT_INVALID",
"输入不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
node_id=node.id,
details={"errors": issues},
)
declared = node.config.get("fields")
data = dict(ctx.inputs)
if isinstance(declared, list) and declared:
names = {str(f.get("name")) for f in declared if isinstance(f, dict) and f.get("name")}
data = {k: v for k, v in data.items() if k in names} or data
return NodeResult(data=data)
class OutputNodeExecutor:
"""Projects the final run output. Empty mapping = merge all upstream data."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
mapping = node.config.get("mapping") or {}
state = ctx.state()
if isinstance(mapping, dict) and mapping:
data = {str(key): render_value(template, state) for key, template in mapping.items()}
else:
data = {}
for result in ctx.node_results.values():
payload = result.get("data")
if isinstance(payload, dict):
data.update(payload)
schema = ctx.graph.output_schema or {}
missing = [key for key in _required_keys(schema) if data.get(key) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
f"输出缺少必填字段:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
issues = schema_issues(data, schema)
if issues:
raise WorkflowError(
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
"输出不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
node_id=node.id,
details={"errors": issues},
)
return NodeResult(data=data, metadata={"isRunOutput": True})
class TransformNodeExecutor:
"""Pure data reshaping. Supported ops: set / pick / omit / json_parse / join."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
state = ctx.state()
data: dict[str, Any] = {}
template = node.config.get("template")
if isinstance(template, str) and template:
data["text"] = render_template(template, state)
for raw in node.config.get("operations") or []:
if not isinstance(raw, dict):
continue
op = str(raw.get("op") or "set")
target = str(raw.get("target") or raw.get("name") or "value")
try:
if op == "set":
data[target] = render_value(raw.get("value"), state)
elif op == "pick":
source = render_value(raw.get("source"), state)
keys = [str(k) for k in (raw.get("keys") or [])]
data[target] = {k: source.get(k) for k in keys} if isinstance(source, dict) else None
elif op == "omit":
source = render_value(raw.get("source"), state)
keys = {str(k) for k in (raw.get("keys") or [])}
data[target] = {k: v for k, v in source.items() if k not in keys} if isinstance(source, dict) else None
elif op == "json_parse":
text = render_value(raw.get("source"), state)
data[target] = json.loads(text) if isinstance(text, str) and text.strip() else None
elif op == "join":
items = render_value(raw.get("source"), state)
separator = str(raw.get("separator") or "\n")
data[target] = separator.join(str(i) for i in items) if isinstance(items, list) else ""
else:
raise WorkflowError(
"WORKFLOW_TRANSFORM_FAILED",
f"不支持的转换操作:{op}",
node_id=node.id,
details={"op": op},
)
except WorkflowError:
raise
except Exception as exc: # noqa: BLE001 - surface as typed node failure
raise WorkflowError(
"WORKFLOW_TRANSFORM_FAILED",
f"转换操作 {op} 执行失败",
node_id=node.id,
details={"op": op, "target": target, "reason": type(exc).__name__},
) from exc
return NodeResult(data=data)
class ConditionNodeExecutor:
"""Picks the first matching branch; activates only that outgoing port."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
state = ctx.state()
branches = node.config.get("branches") or []
if not isinstance(branches, list) or not branches:
raise WorkflowError("WORKFLOW_CONDITION_FAILED", "条件节点未配置分支", node_id=node.id)
evaluated: list[dict[str, Any]] = []
chosen: str | None = None
for raw in branches:
if not isinstance(raw, dict):
continue
name = str(raw.get("name") or "")
expression = str(raw.get("expression") or "")
matched = evaluate_condition(expression, state)
evaluated.append({"name": name, "matched": matched})
if matched and chosen is None:
chosen = name
if chosen is None:
chosen = str(node.config.get("defaultBranch") or node.config.get("default_branch") or DEFAULT_PORT)
return NodeResult(
data={"branch": chosen, "evaluated": evaluated},
metadata={ACTIVE_PORTS: [chosen]},
)
class MergeNodeExecutor:
"""Collects upstream results per strategy (all / first_success / any_success)."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
strategy = str(node.config.get("strategy") or "all")
sources = [str(s) for s in (node.config.get("sources") or [])]
if not sources:
sources = [edge.source for edge in ctx.graph.edges if edge.target == node.id]
collected: dict[str, Any] = {}
succeeded: list[str] = []
for source in sources:
result = ctx.node_results.get(source)
if result is None:
continue
collected[source] = result.get("data")
succeeded.append(source)
if strategy in ("first_success", "any_success") and not succeeded:
raise WorkflowError(
"WORKFLOW_MERGE_FAILED",
"合并节点没有任何成功的上游结果",
node_id=node.id,
details={"strategy": strategy, "sources": sources},
)
if strategy == "first_success":
first = succeeded[0]
return NodeResult(data={"selected": first, "value": collected[first]})
return NodeResult(data={"branches": collected, "succeeded": succeeded})
class LoopNodeExecutor:
"""Iteration gate. Activates ``continue`` (back-edge) or ``done``.
Iteration accounting lives in ``ctx.loop_state`` so the cap is enforced even
across a resume, and the engine invalidates body node results before the
next pass.
"""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
iteration = int(ctx.loop_state.get(node.id, 0))
configured = int(node.config.get("maxIterations") or node.config.get("max_iterations") or 1)
cap = min(configured, ctx.graph.settings.max_loop_iterations)
expression = str(node.config.get("continueWhen") or node.config.get("continue_when") or "")
should_continue = bool(expression) and evaluate_condition(expression, ctx.state())
next_iteration = iteration + 1
if should_continue and next_iteration >= cap:
should_continue = False
port = LOOP_CONTINUE_PORT if should_continue else LOOP_DONE_PORT
if should_continue:
ctx.loop_state[node.id] = next_iteration
return NodeResult(
data={
"iteration": iteration,
"nextIteration": next_iteration if should_continue else iteration,
"maxIterations": cap,
"continue": should_continue,
},
metadata={ACTIVE_PORTS: [port]},
)
__all__ = [
"ConditionNodeExecutor",
"LoopNodeExecutor",
"MergeNodeExecutor",
"OutputNodeExecutor",
"StartNodeExecutor",
"TransformNodeExecutor",
]
@@ -0,0 +1,77 @@
"""``human_input`` node: suspends the run and validates the resumed payload."""
from __future__ import annotations
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
from deerflow.workflows.schemas import NodeResult, WorkflowNode
DEFAULT_ACTIONS = ("submit",)
def _form_schema(node: WorkflowNode) -> dict[str, Any]:
schema = node.config.get("formSchema") or node.config.get("form_schema") or {}
return schema if isinstance(schema, dict) else {}
def _actions(node: WorkflowNode) -> list[str]:
raw = node.config.get("actions")
actions = [str(a) for a in raw] if isinstance(raw, list) and raw else list(DEFAULT_ACTIONS)
return actions
class HumanInputNodeExecutor:
"""First pass raises :class:`WorkflowPaused`; the resume pass consumes the
payload the API stored on the context and validates it against the form."""
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
payload = ctx.resume_payload
if not payload or str(payload.get("nodeId") or "") != node.id:
raise WorkflowPaused(
node.id,
{
"nodeId": node.id,
"name": node.name,
"prompt": render_template(str(node.config.get("prompt") or ""), ctx.state()),
"formSchema": _form_schema(node),
"actions": _actions(node),
},
)
action = str(payload.get("action") or "submit")
allowed = _actions(node)
if action not in allowed:
raise WorkflowError(
"WORKFLOW_HUMAN_INPUT_INVALID",
f"不支持的操作:{action}",
node_id=node.id,
details={"allowed": allowed},
)
values = payload.get("values")
values = values if isinstance(values, dict) else {}
schema = _form_schema(node)
required = schema.get("required")
if isinstance(required, list) and action != "cancel":
missing = [str(key) for key in required if values.get(str(key)) in (None, "")]
if missing:
raise WorkflowError(
"WORKFLOW_HUMAN_INPUT_INVALID",
f"缺少必填项:{'、'.join(missing)}",
node_id=node.id,
details={"missing": missing},
)
if action == "cancel":
raise WorkflowError(
"WORKFLOW_CANCELLED",
"用户在人工节点选择终止运行",
node_id=node.id,
)
# One-shot: a later human node in the same run must pause again.
ctx.resume_payload = None
return NodeResult(data={"action": action, "values": values})
__all__ = ["HumanInputNodeExecutor"]
@@ -0,0 +1,276 @@
"""External-effect nodes: ``http``, ``sql_read``, ``code``."""
from __future__ import annotations
import json
import logging
from typing import Any
from deerflow.config.workflow_config import WorkflowConfig
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.expressions import render_template, render_value
from deerflow.workflows.runtime.code_runner import run_python_snippet
from deerflow.workflows.runtime.context import RunContext
from deerflow.workflows.runtime.sql_runner import execute_read_query
from deerflow.workflows.schemas import NodeResult, WorkflowNode
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
from deerflow.workflows.security.sql_policy import (
assert_parameters_bound,
assert_read_only,
assert_tables_allowed,
enforce_limit,
)
logger = logging.getLogger(__name__)
# Response headers safe to echo back into the event log.
_SAFE_RESPONSE_HEADERS = ("content-type", "content-length", "date", "etag", "location")
_REDACTED_REQUEST_HEADERS = ("authorization", "cookie", "x-api-key", "proxy-authorization")
class HttpNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.http
state = ctx.state()
url = render_template(str(node.config.get("url") or ""), state).strip()
if not url:
raise WorkflowError("WORKFLOW_HTTP_FAILED", "HTTP 节点缺少 URL", node_id=node.id)
method = str(node.config.get("method") or "GET").upper()
headers = {str(k): str(v) for k, v in (render_value(node.config.get("headers") or {}, state) or {}).items()}
query = render_value(node.config.get("query") or {}, state) or {}
body = render_value(node.config.get("body"), state)
credential_ref = node.config.get("credentialRef") or node.config.get("credential_ref")
if credential_ref:
headers.update(await self._credential_headers(str(credential_ref), node, ctx))
try:
target = await enforce_http_target(url, policy)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
timeout = min(
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
policy.timeout_seconds,
)
max_bytes = min(
int(node.config.get("maxResponseBytes") or node.config.get("max_response_bytes") or policy.max_response_bytes),
policy.max_response_bytes,
)
allow_redirects = bool(node.config.get("allowRedirects") or node.config.get("allow_redirects"))
max_hops = policy.max_redirects if allow_redirects else 0
await ctx.emit_event(
"node.progress",
data={
"phase": "http.request",
"method": method,
"host": target["host"],
"headers": [k for k in headers if k.lower() not in _REDACTED_REQUEST_HEADERS],
},
node_id=node.id,
)
import httpx
current_url = url
hops = 0
raw = b""
truncated = False
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
while True:
ctx.cancel.raise_if_cancelled(node.id)
try:
request = client.build_request(
method,
current_url,
headers=headers,
params=query or None,
json=body if isinstance(body, (dict, list)) else None,
content=body if isinstance(body, (str, bytes)) else None,
)
response = await client.send(request, stream=True)
except httpx.HTTPError as exc:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
"HTTP 请求失败",
retryable=True,
node_id=node.id,
details={"reason": type(exc).__name__},
) from exc
if response.status_code in (301, 302, 303, 307, 308) and hops < max_hops:
location = response.headers.get("location") or ""
await response.aclose()
if not location:
break
current_url = await enforce_redirect(location, current_url, policy)
hops += 1
continue
chunks: list[bytes] = []
size = 0
try:
async for chunk in response.aiter_bytes():
size += len(chunk)
if size > max_bytes:
truncated = True
break
chunks.append(chunk)
finally:
await response.aclose()
raw = b"".join(chunks)
break
text_body = raw.decode("utf-8", errors="replace")
parsed: Any = None
content_type = response.headers.get("content-type", "")
if "json" in content_type.lower():
try:
parsed = json.loads(text_body)
except json.JSONDecodeError:
parsed = None
data = {
"status": response.status_code,
"ok": 200 <= response.status_code < 300,
"headers": {k: v for k, v in response.headers.items() if k.lower() in _SAFE_RESPONSE_HEADERS},
"json": parsed,
"text": None if parsed is not None else text_body,
"truncated": truncated,
"redirects": hops,
}
if not data["ok"]:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
f"HTTP 请求返回 {response.status_code}",
retryable=response.status_code >= 500 or response.status_code == 429,
node_id=node.id,
details={"status": response.status_code, "preview": text_body[:500]},
)
return NodeResult(data=data, metadata={"host": target["host"]})
async def _credential_headers(self, ref: str, node: WorkflowNode, ctx: RunContext) -> dict[str, str]:
if ctx.deps.resolve_credential is None:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"运行环境未配置凭据解析能力",
node_id=node.id,
)
credential = await ctx.deps.resolve_credential(ref)
if not credential:
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"引用的凭据不存在或无权访问",
node_id=node.id,
details={"credentialRef": ref},
)
headers = credential.get("headers")
return {str(k): str(v) for k, v in headers.items()} if isinstance(headers, dict) else {}
class SqlReadNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.sql
if not policy.enabled:
raise WorkflowError("WORKFLOW_SQL_POLICY_DENIED", "SQL 节点已被系统禁用", node_id=node.id)
data_source_id = str(node.config.get("dataSourceId") or node.config.get("data_source_id") or "")
if not data_source_id:
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 节点未选择数据源", node_id=node.id)
if ctx.deps.resolve_data_source is None:
raise WorkflowError("WORKFLOW_RESOURCE_MISSING", "运行环境未配置数据源解析能力", node_id=node.id)
source = await ctx.deps.resolve_data_source(data_source_id)
if not source or not source.get("dsn"):
raise WorkflowError(
"WORKFLOW_RESOURCE_MISSING",
"数据源不存在或无权访问",
node_id=node.id,
details={"dataSourceId": data_source_id},
)
statement = assert_read_only(str(node.config.get("statement") or ""))
parameters = render_value(node.config.get("parameters") or {}, ctx.state()) or {}
if not isinstance(parameters, dict):
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 参数必须是对象", node_id=node.id)
assert_parameters_bound(statement, parameters)
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
max_rows = min(
int(node.config.get("maxRows") or node.config.get("max_rows") or policy.max_rows),
policy.max_rows,
int(source.get("max_rows") or policy.max_rows),
)
statement = enforce_limit(statement, max_rows)
await ctx.emit_event(
"node.progress",
data={"phase": "sql.query", "dataSourceId": data_source_id, "maxRows": max_rows},
node_id=node.id,
)
try:
result = await execute_read_query(
dsn=str(source["dsn"]),
statement=statement,
parameters={str(k): v for k, v in parameters.items()},
max_rows=max_rows,
timeout_seconds=policy.statement_timeout_seconds,
max_cell_chars=policy.max_cell_chars,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
return NodeResult(data=result)
class CodeNodeExecutor:
def __init__(self, config: WorkflowConfig) -> None:
self._config = config
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
policy = self._config.code
if not policy.enabled:
raise WorkflowError(
"WORKFLOW_CODE_SANDBOX_FAILED",
"代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)",
node_id=node.id,
)
source = str(node.config.get("source") or "")
if not source.strip():
raise WorkflowError("WORKFLOW_CODE_SANDBOX_FAILED", "代码节点没有源代码", node_id=node.id)
if len(source) > policy.max_source_chars:
raise WorkflowError(
"WORKFLOW_LIMIT_EXCEEDED",
f"代码长度超过上限 {policy.max_source_chars}",
node_id=node.id,
)
inputs = render_value(node.config.get("inputs") or {}, ctx.state()) or {}
timeout = min(
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
policy.timeout_seconds,
)
allow_network = bool((node.config.get("allowNetwork") or node.config.get("allow_network")) and policy.allow_network)
try:
outcome = await run_python_snippet(
source=source,
inputs=inputs if isinstance(inputs, dict) else {"value": inputs},
timeout_seconds=timeout,
max_output_chars=policy.max_output_chars,
memory_limit_mb=policy.memory_limit_mb,
allow_network=allow_network,
)
except WorkflowError as exc:
exc.node_id = exc.node_id or node.id
raise
return NodeResult(
data=outcome["result"],
metadata={"stdout": outcome["stdout"][-4000:], "stderr": outcome["stderr"][-2000:]},
)
__all__ = ["CodeNodeExecutor", "HttpNodeExecutor", "SqlReadNodeExecutor"]
@@ -0,0 +1,17 @@
"""Leaf constants shared by the scheduler and node executors.
Kept dependency-free so the engine and the node package can both import it
without forming a cycle.
"""
from __future__ import annotations
# Metadata key a node sets to restrict which outgoing ports the engine activates.
ACTIVE_PORTS = "activePorts"
# Reserved port names.
LOOP_CONTINUE_PORT = "continue"
LOOP_DONE_PORT = "done"
CONDITION_DEFAULT_PORT = "else"
__all__ = ["ACTIVE_PORTS", "CONDITION_DEFAULT_PORT", "LOOP_CONTINUE_PORT", "LOOP_DONE_PORT"]
@@ -0,0 +1,23 @@
"""Workflow execution runtime (scheduler, context, event sink)."""
from __future__ import annotations
from deerflow.workflows.runtime.context import (
CancelToken,
RunContext,
WorkflowPaused,
WorkflowRuntimeDeps,
)
from deerflow.workflows.runtime.engine import EngineResult, WorkflowEngine
from deerflow.workflows.runtime.sink import PersistingWorkflowEventSink, envelope_from_row
__all__ = [
"CancelToken",
"EngineResult",
"PersistingWorkflowEventSink",
"RunContext",
"WorkflowEngine",
"WorkflowPaused",
"WorkflowRuntimeDeps",
"envelope_from_row",
]
@@ -0,0 +1,173 @@
"""Constrained Python execution for the ``code`` node.
The snippet runs in a **separate interpreter process** with:
* a scrubbed environment (no API keys, tokens, or DSNs inherited),
* a fresh temporary working directory that is deleted afterwards,
* an address-space cap and no core dumps (POSIX ``setrlimit``),
* sockets disabled unless the node explicitly allows network,
* a hard wall-clock timeout with process-group kill,
* bounded stdout/stderr capture.
This is process isolation, not a container: a determined snippet can still read
world-readable files on the host. That is why ``workflows.code.enabled``
defaults to ``false`` — turn it on only where the container sandbox or an OS
level jail already confines the backend process.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import shutil
import sys
import tempfile
from pathlib import Path
from typing import Any
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
_RUNNER_TEMPLATE = """
import json, os, sys
_ALLOW_NETWORK = {allow_network}
if not _ALLOW_NETWORK:
import socket
def _blocked(*_args, **_kwargs):
raise PermissionError("network access is disabled for this code node")
socket.socket = _blocked
socket.create_connection = _blocked
socket.getaddrinfo = _blocked
try:
import resource
_limit = {memory_bytes}
resource.setrlimit(resource.RLIMIT_AS, (_limit, _limit))
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
except Exception:
pass
inputs = json.loads(os.environ.get("WORKFLOW_NODE_INPUT") or "{{}}")
output = {{}}
_globals = {{"__name__": "__workflow_code__", "inputs": inputs, "output": output}}
with open({source_path!r}, "r", encoding="utf-8") as handle:
_source = handle.read()
exec(compile(_source, "<workflow-code>", "exec"), _globals)
_result = _globals.get("output")
if not isinstance(_result, dict):
_result = {{"value": _result}}
sys.stdout.write("\\n__WORKFLOW_RESULT__" + json.dumps(_result, ensure_ascii=False, default=str))
"""
def _safe_env() -> dict[str, str]:
env = {
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
"LANG": os.environ.get("LANG", "C.UTF-8"),
"PYTHONIOENCODING": "utf-8",
"PYTHONDONTWRITEBYTECODE": "1",
"HOME": "",
}
if sys.platform == "win32":
for key in ("SYSTEMROOT", "COMSPEC", "TEMP", "TMP"):
if os.environ.get(key):
env[key] = os.environ[key]
return env
async def run_python_snippet(
*,
source: str,
inputs: dict[str, Any],
timeout_seconds: int,
max_output_chars: int,
memory_limit_mb: int,
allow_network: bool,
) -> dict[str, Any]:
workdir = Path(tempfile.mkdtemp(prefix="wf-code-"))
try:
source_path = workdir / "snippet.py"
source_path.write_text(source, encoding="utf-8")
runner_path = workdir / "runner.py"
runner_path.write_text(
_RUNNER_TEMPLATE.format(
allow_network=bool(allow_network),
memory_bytes=max(64, int(memory_limit_mb)) * 1024 * 1024,
source_path=str(source_path),
),
encoding="utf-8",
)
env = _safe_env()
env["WORKFLOW_NODE_INPUT"] = json.dumps(inputs, ensure_ascii=False, default=str)
process = await asyncio.create_subprocess_exec(
sys.executable,
"-I",
"-S",
str(runner_path),
cwd=str(workdir),
env=env,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
start_new_session=sys.platform != "win32",
)
try:
stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
except TimeoutError as exc:
_terminate(process)
raise WorkflowError(
"WORKFLOW_TIMEOUT",
f"代码节点执行超时({timeout_seconds}s)",
details={"timeoutSeconds": timeout_seconds},
) from exc
stdout = stdout_bytes.decode("utf-8", errors="replace")
stderr = stderr_bytes.decode("utf-8", errors="replace")[:max_output_chars]
if process.returncode != 0:
raise WorkflowError(
"WORKFLOW_CODE_SANDBOX_FAILED",
"代码节点执行失败",
details={"exitCode": process.returncode, "stderr": stderr[-2000:]},
)
marker = "__WORKFLOW_RESULT__"
index = stdout.rfind(marker)
result: dict[str, Any] = {}
logs = stdout
if index >= 0:
logs = stdout[:index]
try:
parsed = json.loads(stdout[index + len(marker) :])
result = parsed if isinstance(parsed, dict) else {"value": parsed}
except json.JSONDecodeError:
result = {}
return {
"result": result,
"stdout": logs[:max_output_chars],
"stderr": stderr,
}
finally:
shutil.rmtree(workdir, ignore_errors=True)
def _terminate(process: asyncio.subprocess.Process) -> None:
try:
if sys.platform != "win32":
os.killpg(os.getpgid(process.pid), 9)
else:
process.kill()
except (ProcessLookupError, PermissionError, OSError):
logger.debug("code node process already gone", exc_info=True)
__all__ = ["run_python_snippet"]
@@ -0,0 +1,162 @@
"""Run-scoped execution context shared by the engine and node executors."""
from __future__ import annotations
import asyncio
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.events import WorkflowEventType
from deerflow.workflows.schemas import ArtifactRef, WorkflowGraph, WorkflowNode
EmitFn = Callable[..., Awaitable[Any]]
def _public_artifact_payload(ref: ArtifactRef) -> dict[str, Any]:
"""Wire payload for ``artifact.created``.
The sandbox/host ``path`` stays internal: clients address artifacts by id
through the run-scoped content endpoints, never by filesystem path."""
from deerflow.workflows.events import artifact_kind
return {
"artifactId": ref.artifact_id,
"name": ref.name,
"kind": artifact_kind(ref.mime_type),
"mimeType": ref.mime_type,
"sizeBytes": ref.size_bytes,
"preview": ref.preview,
}
class WorkflowPaused(Exception): # noqa: N818 - control-flow signal, not an error
"""Raised by ``human_input`` to suspend the run until a resume arrives."""
def __init__(self, node_id: str, pending_input: dict[str, Any]) -> None:
super().__init__(f"workflow paused at {node_id}")
self.node_id = node_id
self.pending_input = pending_input
class CancelToken:
"""Cooperative cancellation shared between the executor and node code."""
def __init__(self) -> None:
self._event = asyncio.Event()
self._reason = "cancelled"
def cancel(self, reason: str = "cancelled") -> None:
self._reason = reason
self._event.set()
@property
def cancelled(self) -> bool:
return self._event.is_set()
async def wait(self) -> None:
await self._event.wait()
def raise_if_cancelled(self, node_id: str | None = None) -> None:
if self._event.is_set():
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
@dataclass
class WorkflowRuntimeDeps:
"""Injected capabilities. The harness never imports the app layer, so every
integration point (agents, HTTP egress, SQL, sandbox, subworkflows) arrives
as an async callable supplied by the caller. ``None`` means "not available",
which node executors turn into a typed ``WORKFLOW_RESOURCE_MISSING``."""
run_agent: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_http: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_sql: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_code: Callable[..., Awaitable[dict[str, Any]]] | None = None
run_subworkflow: Callable[..., Awaitable[dict[str, Any]]] | None = None
save_artifact: Callable[..., Awaitable[dict[str, Any]]] | None = None
resolve_credential: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
resolve_data_source: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
@dataclass
class RunContext:
"""Everything a node executor is allowed to see."""
run_id: str
workflow_id: str
version_id: str
owner_id: str
graph: WorkflowGraph
inputs: dict[str, Any]
deps: WorkflowRuntimeDeps
cancel: CancelToken
emit: EmitFn
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
loop_state: dict[str, int] = field(default_factory=dict)
resume_payload: dict[str, Any] | None = None
node_run_ids: dict[str, str] = field(default_factory=dict)
current_node_id: str | None = None
env: dict[str, Any] = field(default_factory=dict)
def state(self) -> dict[str, Any]:
"""Whitelisted tree visible to expressions. Credentials never appear."""
return {
"inputs": self.inputs,
"nodes": self.node_results,
"run": {"id": self.run_id, "workflowId": self.workflow_id, "versionId": self.version_id},
"loop": dict(self.loop_state),
"env": self.env,
}
async def emit_event(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
) -> None:
node = node_id or self.current_node_id
await self.emit(
event_type,
data=data or {},
node_id=node,
node_run_id=self.node_run_ids.get(node) if node else None,
)
async def register_artifact(
self,
node: WorkflowNode,
*,
name: str,
mime_type: str = "application/octet-stream",
path: str = "",
size_bytes: int | None = None,
preview: str | None = None,
) -> ArtifactRef | None:
"""Persist an artifact reference and emit ``artifact.created``."""
if self.deps.save_artifact is None:
return None
row = await self.deps.save_artifact(
run_id=self.run_id,
node_id=node.id,
name=name,
mime_type=mime_type,
path=path,
size_bytes=size_bytes,
preview=preview,
)
ref = ArtifactRef(
artifactId=str(row.get("id") or ""),
name=str(row.get("name") or name),
mimeType=str(row.get("mime_type") or mime_type),
path=str(row.get("path") or path),
sizeBytes=row.get("size_bytes"),
preview=row.get("preview"),
)
await self.emit_event("artifact.created", data=_public_artifact_payload(ref), node_id=node.id)
return ref
__all__ = ["CancelToken", "RunContext", "WorkflowPaused", "WorkflowRuntimeDeps"]
@@ -0,0 +1,531 @@
"""Persisted DAG scheduler for workflow runs.
Scheduling model
----------------
Every edge carries a state: ``pending`` → ``active`` (its source chose this
port) or ``pruned`` (the source chose a different port). A node is *ready* when
none of its non-back incoming edges is ``pending`` and at least one is
``active``; a node whose incoming edges all resolved to ``pruned`` is *skipped*
and prunes its own outgoing edges in turn. That single rule gives us fan-out
parallelism, condition branching, and merge joins without a topological
pre-pass, and it terminates because each superstep resolves at least one edge.
Loops are the one cycle we allow: the back edge (``loop`` node, port
``continue``) is excluded from readiness gating, and activating it clears the
body's results so the next iteration re-runs them. Every re-entry is capped by
``settings.max_loop_iterations``.
Durability: node results are written to the run store as they complete, so a
resumed or re-leased run replays only what is missing. ``human_input`` raises
:class:`WorkflowPaused`, which the caller turns into ``awaiting_input``.
"""
from __future__ import annotations
import asyncio
import logging
import time
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any, Protocol
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
from deerflow.workflows.errors import WorkflowError
from deerflow.workflows.ports import ACTIVE_PORTS, LOOP_CONTINUE_PORT
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
from deerflow.workflows.schemas import NodeResult, WorkflowEdge, WorkflowGraph, WorkflowNode
class ExecutorRegistry(Protocol):
"""Structural view of ``deerflow.workflows.nodes.NodeRegistry``.
Declared here rather than imported so the scheduler has no dependency on
the node package (which depends on the runtime context).
"""
def get(self, node_type: str) -> Any: ...
logger = logging.getLogger(__name__)
PENDING = "pending"
ACTIVE = "active"
PRUNED = "pruned"
@dataclass
class EngineResult:
output: dict[str, Any] = field(default_factory=dict)
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
loop_state: dict[str, int] = field(default_factory=dict)
steps: int = 0
def _is_back_edge(edge: WorkflowEdge, nodes: dict[str, WorkflowNode]) -> bool:
source = nodes.get(edge.source)
return bool(source and source.type == "loop" and (edge.source_port or "") == LOOP_CONTINUE_PORT)
def _node_started_event_data(node: WorkflowNode, node_attempt: int) -> dict[str, Any]:
"""Safe presentation metadata for workflow clients.
This stays intentionally small: UI clients need stable agent identity and
an optional explicit parallel group, but never prompt templates, tools,
credentials, or the full node config. A missing group is valid; clients
may still use their backwards-compatible name-based presentation.
"""
data: dict[str, Any] = {
"nodeType": node.type,
"nodeKind": node.type,
"name": node.name,
"attempt": node_attempt,
}
group_id = node.config.get("parallelGroupId") or node.config.get("parallel_group_id")
if isinstance(group_id, str) and group_id.strip():
data["parallelGroupId"] = group_id.strip()
round_index = node.config.get("roundIndex")
if round_index is None:
round_index = node.config.get("round_index")
if isinstance(round_index, int) and round_index >= 0:
data["roundIndex"] = round_index
if node.type == "agent":
agent_id = node.config.get("agentId") or node.config.get("agent_id")
if isinstance(agent_id, str) and agent_id.strip():
agent: dict[str, str] = {
"agentId": agent_id.strip(),
"name": str(
node.config.get("agentName")
or node.config.get("agent_name")
or node.name
or agent_id
),
}
avatar_url = node.config.get("agentAvatarUrl") or node.config.get("agent_avatar_url")
if isinstance(avatar_url, str) and avatar_url.strip():
agent["avatarUrl"] = avatar_url.strip()
data["agent"] = agent
return data
class WorkflowEngine:
def __init__(self, registry: ExecutorRegistry, *, run_store: WorkflowRunStore | None = None) -> None:
self._registry = registry
self._store = run_store
async def run(
self,
ctx: RunContext,
*,
preloaded: dict[str, dict[str, Any]] | None = None,
loop_state: dict[str, int] | None = None,
attempt: int = 1,
) -> EngineResult:
graph: WorkflowGraph = ctx.graph
nodes = graph.node_map()
outgoing: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
incoming: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
for edge in graph.edges:
outgoing[edge.source].append(edge)
incoming[edge.target].append(edge)
edge_state: dict[str, str] = {edge.id: PENDING for edge in graph.edges}
done: set[str] = set()
skipped: set[str] = set()
ctx.loop_state.update(loop_state or {})
if preloaded:
ctx.node_results.update(preloaded)
done |= set(preloaded)
# Re-derive edge states from the ports each replayed node selected.
for node_id in list(done):
self._apply_ports(
node_id,
ctx.node_results.get(node_id, {}).get("metadata") or {},
outgoing,
edge_state,
)
semaphore = asyncio.Semaphore(max(1, graph.settings.max_parallelism))
steps = 0
max_steps = max(1, graph.settings.max_steps)
while True:
ctx.cancel.raise_if_cancelled()
self._propagate_skips(nodes, incoming, outgoing, edge_state, done, skipped)
ready = [node_id for node_id in nodes if node_id not in done and node_id not in skipped and self._readiness(node_id, incoming, edge_state, nodes) == "ready"]
if not ready:
break
steps += len(ready)
if steps > max_steps:
raise WorkflowError(
"WORKFLOW_MAX_STEPS",
f"运行步数超过上限 {max_steps}",
details={"maxSteps": max_steps},
)
results = await asyncio.gather(
*(self._run_one(nodes[node_id], ctx, semaphore, attempt) for node_id in ready),
return_exceptions=True,
)
paused: WorkflowPaused | None = None
failure: BaseException | None = None
for node_id, outcome in zip(ready, results, strict=True):
if isinstance(outcome, WorkflowPaused):
paused = paused or outcome
continue
if isinstance(outcome, BaseException):
failure = failure or outcome
continue
ctx.node_results[node_id] = outcome
done.add(node_id)
metadata = outcome.get("metadata") or {}
self._apply_ports(node_id, metadata, outgoing, edge_state)
if nodes[node_id].type == "loop":
await self._maybe_reset_loop_body(nodes[node_id], ctx, nodes, outgoing, edge_state, done, skipped)
if failure is not None:
raise failure
if paused is not None:
raise paused
return EngineResult(
output=self._collect_output(nodes, ctx),
node_results=dict(ctx.node_results),
loop_state=dict(ctx.loop_state),
steps=steps,
)
# ── scheduling helpers ──────────────────────────────────────────────
def _readiness(
self,
node_id: str,
incoming: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
nodes: dict[str, WorkflowNode],
) -> str:
edges = incoming.get(node_id) or []
if not edges:
return "ready"
back = [e for e in edges if _is_back_edge(e, nodes)]
if any(edge_state[e.id] == ACTIVE for e in back):
return "ready"
normal = [e for e in edges if not _is_back_edge(e, nodes)]
if not normal:
return "blocked"
if any(edge_state[e.id] == PENDING for e in normal):
return "blocked"
if any(edge_state[e.id] == ACTIVE for e in normal):
return "ready"
return "skip"
def _apply_ports(
self,
node_id: str,
metadata: dict[str, Any],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
) -> None:
active_ports = metadata.get(ACTIVE_PORTS)
allowed = {str(p) for p in active_ports} if isinstance(active_ports, list) else None
for edge in outgoing.get(node_id) or []:
port = edge.source_port or ""
if allowed is None or not port or port in allowed:
edge_state[edge.id] = ACTIVE
else:
edge_state[edge.id] = PRUNED
def _propagate_skips(
self,
nodes: dict[str, WorkflowNode],
incoming: dict[str, list[WorkflowEdge]],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
done: set[str],
skipped: set[str],
) -> None:
changed = True
while changed:
changed = False
for node_id in nodes:
if node_id in done or node_id in skipped:
continue
if self._readiness(node_id, incoming, edge_state, nodes) != "skip":
continue
skipped.add(node_id)
for edge in outgoing.get(node_id) or []:
edge_state[edge.id] = PRUNED
changed = True
async def _maybe_reset_loop_body(
self,
loop_node: WorkflowNode,
ctx: RunContext,
nodes: dict[str, WorkflowNode],
outgoing: dict[str, list[WorkflowEdge]],
edge_state: dict[str, str],
done: set[str],
skipped: set[str],
) -> None:
back_edges = [e for e in outgoing.get(loop_node.id) or [] if _is_back_edge(e, nodes) and edge_state[e.id] == ACTIVE]
if not back_edges:
return
body_entry = str(loop_node.config.get("bodyEntry") or loop_node.config.get("body_entry") or "")
if not body_entry or body_entry not in nodes:
raise WorkflowError(
"WORKFLOW_LOOP_FAILED",
"循环节点的 bodyEntry 不存在",
node_id=loop_node.id,
details={"bodyEntry": body_entry},
)
body = self._loop_body(body_entry, loop_node.id, outgoing, nodes)
for node_id in body:
done.discard(node_id)
skipped.discard(node_id)
ctx.node_results.pop(node_id, None)
for edge in outgoing.get(node_id) or []:
if not _is_back_edge(edge, nodes):
edge_state[edge.id] = PENDING
# The back edge stays ACTIVE so the body entry is reachable again; the
# loop node prunes it itself once it selects the ``done`` port.
if self._store is not None and body:
try:
await self._store.invalidate_node_runs(ctx.run_id, sorted(body))
except Exception: # noqa: BLE001 - bookkeeping must not kill the run
logger.warning("failed to invalidate node runs for loop reset", exc_info=True)
await ctx.emit_event(
"node.progress",
data={
"phase": "loop.iteration",
"iteration": ctx.loop_state.get(loop_node.id, 0),
"bodyEntry": body_entry,
},
node_id=loop_node.id,
)
def _loop_body(
self,
body_entry: str,
loop_id: str,
outgoing: dict[str, list[WorkflowEdge]],
nodes: dict[str, WorkflowNode],
) -> set[str]:
"""Nodes reachable from ``body_entry`` up to (and including) the loop node."""
body: set[str] = set()
stack = [body_entry]
while stack:
current = stack.pop()
if current in body:
continue
body.add(current)
if current == loop_id:
continue
for edge in outgoing.get(current) or []:
if not _is_back_edge(edge, nodes):
stack.append(edge.target)
return body
# ── node execution ──────────────────────────────────────────────────
async def _run_one(
self,
node: WorkflowNode,
ctx: RunContext,
semaphore: asyncio.Semaphore,
attempt: int,
) -> dict[str, Any]:
async with semaphore:
ctx.cancel.raise_if_cancelled(node.id)
retry_cfg = node.config.get("retry") if isinstance(node.config.get("retry"), dict) else {}
max_attempts = max(1, min(5, int(retry_cfg.get("maxAttempts") or 1)))
backoff = max(0.0, float(retry_cfg.get("backoffSeconds") or 0.5))
on_error = str(node.config.get("onError") or "fail")
timeout = self._node_timeout(node, ctx)
last_error: WorkflowError | None = None
for node_attempt in range(1, max_attempts + 1):
node_run = await self._begin_node_run(ctx, node, attempt, node_attempt)
started = time.monotonic()
await ctx.emit_event(
"node.started",
data=_node_started_event_data(node, node_attempt),
node_id=node.id,
)
try:
executor = self._registry.get(node.type)
result: NodeResult = await asyncio.wait_for(executor.execute(node, ctx), timeout=timeout)
except WorkflowPaused:
await self._finish_node_run(ctx, node_run, status="queued", output=None)
raise
except TimeoutError as exc:
last_error = WorkflowError(
"WORKFLOW_TIMEOUT",
f"节点执行超时({timeout}s)",
retryable=True,
node_id=node.id,
details={"timeoutSeconds": timeout},
)
await self._fail_node_run(ctx, node_run, last_error, started)
if node_attempt < max_attempts:
await asyncio.sleep(backoff * node_attempt)
continue
if on_error == "continue":
return self._error_result(last_error)
raise last_error from exc
except WorkflowError as exc:
last_error = exc
exc.node_id = exc.node_id or node.id
await self._fail_node_run(ctx, node_run, exc, started)
if exc.retryable and node_attempt < max_attempts:
await asyncio.sleep(backoff * node_attempt)
continue
if on_error == "continue":
return self._error_result(exc)
raise
except asyncio.CancelledError:
raise
except Exception as exc: # noqa: BLE001 - unexpected node bug
logger.exception("workflow node %s crashed", node.id)
last_error = WorkflowError(
"WORKFLOW_INTERNAL",
"节点执行发生内部错误",
node_id=node.id,
details={"reason": type(exc).__name__},
)
await self._fail_node_run(ctx, node_run, last_error, started)
if on_error == "continue":
return self._error_result(last_error)
raise last_error from exc
payload = result.model_dump(by_alias=False)
duration = int((time.monotonic() - started) * 1000)
await self._finish_node_run(ctx, node_run, status="completed", output=payload, duration_ms=duration)
await ctx.emit_event(
"node.completed",
data={
"nodeType": node.type,
"durationMs": duration,
"data": payload.get("data"),
"warnings": payload.get("warnings") or [],
},
node_id=node.id,
)
return payload
raise last_error or WorkflowError("WORKFLOW_INTERNAL", "节点执行失败", node_id=node.id)
def _node_timeout(self, node: WorkflowNode, ctx: RunContext) -> int:
configured = node.config.get("timeoutSeconds") or node.config.get("timeout_seconds")
graph_limit = ctx.graph.settings.node_timeout_seconds
try:
value = int(configured) if configured else graph_limit
except (TypeError, ValueError):
value = graph_limit
return max(1, min(value, graph_limit))
def _error_result(self, error: WorkflowError) -> dict[str, Any]:
return {
"data": {},
"messages": [],
"artifacts": [],
"metadata": {"failed": True, "error": error.to_body().model_dump(by_alias=True)},
"warnings": [error.message],
}
async def _begin_node_run(self, ctx: RunContext, node: WorkflowNode, run_attempt: int, node_attempt: int) -> dict[str, Any] | None:
if self._store is None:
return None
try:
row = await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node.id,
"node_type": node.type,
"status": "running",
"attempt": node_attempt,
"iteration": int(ctx.loop_state.get(node.id, 0)),
"started_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run start for %s", node.id, exc_info=True)
return None
ctx.node_run_ids[node.id] = str(row.get("id") or "")
return row
async def _finish_node_run(
self,
ctx: RunContext,
node_run: dict[str, Any] | None,
*,
status: str,
output: dict[str, Any] | None,
duration_ms: int | None = None,
) -> None:
if self._store is None or node_run is None:
return
try:
await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node_run["node_id"],
"node_type": node_run["node_type"],
"attempt": node_run["attempt"],
"iteration": node_run["iteration"],
"status": status,
"output": output,
"duration_ms": duration_ms,
"finished_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run finish", exc_info=True)
async def _fail_node_run(
self,
ctx: RunContext,
node_run: dict[str, Any] | None,
error: WorkflowError,
started: float,
) -> None:
duration = int((time.monotonic() - started) * 1000)
body = error.to_body().model_dump(by_alias=True)
if self._store is not None and node_run is not None:
try:
await self._store.record_node_run(
{
"run_id": ctx.run_id,
"node_id": node_run["node_id"],
"node_type": node_run["node_type"],
"attempt": node_run["attempt"],
"iteration": node_run["iteration"],
"status": "failed",
"error": body,
"duration_ms": duration,
"finished_at": datetime.now(UTC),
"is_current": True,
}
)
except Exception: # noqa: BLE001
logger.warning("failed to record node run failure", exc_info=True)
await ctx.emit_event(
"node.failed",
data={"error": body, "durationMs": duration},
node_id=error.node_id,
)
def _collect_output(self, nodes: dict[str, WorkflowNode], ctx: RunContext) -> dict[str, Any]:
output: dict[str, Any] = {}
for node_id, node in nodes.items():
if node.type != "output":
continue
result = ctx.node_results.get(node_id)
if isinstance(result, dict) and isinstance(result.get("data"), dict):
output.update(result["data"])
return output
__all__ = ["EngineResult", "WorkflowEngine"]
@@ -0,0 +1,117 @@
"""Persist-then-publish event sink for workflow runs."""
from __future__ import annotations
import logging
from collections.abc import Awaitable, Callable
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.workflows.events import WorkflowEventEnvelope, WorkflowEventType
logger = logging.getLogger(__name__)
PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
class PersistingWorkflowEventSink:
"""Assigns ``seq`` in the database first, then fans out to live listeners.
A persistence failure never aborts the run, but the event is **dropped
rather than published**: SSE clients tail by cursor and rebuild from the
durable log, so a live-only frame with a made-up seq would render once and
then vanish on reconnect (a "ghost event"). Callers can see the drop via
``degraded_count`` / the ``eventPersistenceDegraded`` marker on the returned
envelope; the stream endpoint synthesises a closing frame from the run row
when a terminal event could not be logged.
"""
def __init__(
self,
store: WorkflowEventStore,
*,
run_id: str,
workflow_id: str,
version_id: str,
publish: PublishFn | None = None,
) -> None:
self._store = store
self._run_id = run_id
self._workflow_id = workflow_id
self._version_id = version_id
self._publish = publish
self.degraded_count = 0
async def emit(
self,
event_type: WorkflowEventType,
*,
data: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> WorkflowEventEnvelope:
payload = data or {}
seq = 0
degraded = False
try:
row = await self._store.append(
run_id=self._run_id,
workflow_id=self._workflow_id,
version_id=self._version_id,
event_type=event_type,
payload=payload,
node_id=node_id,
node_run_id=node_run_id,
)
seq = int(row.get("seq") or 0)
except Exception as exc: # noqa: BLE001 - the run must survive a log outage
logger.warning(
"workflow event not persisted (dropped, not published): run=%s event=%s node=%s error=%s:%s",
self._run_id,
event_type,
node_id,
type(exc).__name__,
exc,
)
self.degraded_count += 1
degraded = True
envelope = WorkflowEventEnvelope(
runId=self._run_id,
workflowId=self._workflow_id,
versionId=self._version_id,
seq=seq,
event=event_type,
nodeId=node_id,
nodeRunId=node_run_id,
timestamp=datetime.now(UTC),
data={**payload, "eventPersistenceDegraded": True} if degraded else payload,
)
if degraded:
return envelope
if self._publish is not None:
try:
await self._publish(envelope.to_sse_dict())
except Exception: # noqa: BLE001 - live fan-out is best effort
logger.debug("workflow live publish failed", exc_info=True)
return envelope
def envelope_from_row(row: dict[str, Any]) -> dict[str, Any]:
"""Turn a stored event row into the SSE wire object (replay path)."""
return {
"schemaVersion": "1.0",
"runId": row.get("run_id"),
"workflowId": row.get("workflow_id"),
"versionId": row.get("version_id"),
"seq": int(row.get("seq") or 0),
"event": row.get("event_type"),
"nodeId": row.get("node_id"),
"nodeRunId": row.get("node_run_id"),
"timestamp": row.get("created_at"),
"data": row.get("payload") or {},
}
__all__ = ["PersistingWorkflowEventSink", "envelope_from_row"]
@@ -0,0 +1,107 @@
"""Read-only query execution for the ``sql_read`` node.
Engines are cached per DSN with a tiny pool because a workflow typically hits
the same reporting database many times in one run. Values are bound as
parameters and every cell is truncated before it can reach an event payload.
"""
from __future__ import annotations
import asyncio
import logging
from datetime import date, datetime
from decimal import Decimal
from typing import Any
from sqlalchemy import text
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
from deerflow.workflows.errors import WorkflowError
logger = logging.getLogger(__name__)
_engines: dict[str, AsyncEngine] = {}
_lock = asyncio.Lock()
async def _get_engine(dsn: str) -> AsyncEngine:
async with _lock:
engine = _engines.get(dsn)
if engine is None:
engine = create_async_engine(
dsn,
pool_pre_ping=True,
pool_size=2,
max_overflow=2,
pool_recycle=1800,
)
_engines[dsn] = engine
return engine
async def dispose_engines() -> None:
async with _lock:
for engine in _engines.values():
await engine.dispose()
_engines.clear()
def _coerce(value: Any, max_chars: int) -> Any:
if value is None or isinstance(value, (bool, int, float)):
return value
if isinstance(value, Decimal):
return float(value)
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, (bytes, bytearray, memoryview)):
return f"<binary {len(bytes(value))} bytes>"
text_value = str(value)
return text_value if len(text_value) <= max_chars else text_value[:max_chars] + "…"
async def execute_read_query(
*,
dsn: str,
statement: str,
parameters: dict[str, Any],
max_rows: int,
timeout_seconds: int,
max_cell_chars: int = 4000,
) -> dict[str, Any]:
engine = await _get_engine(dsn)
async def _run() -> dict[str, Any]:
async with engine.connect() as conn:
result = await conn.execute(text(statement), parameters)
columns = list(result.keys())
rows = result.fetchmany(max_rows)
truncated = bool(result.fetchone())
return {
"columns": columns,
"rows": [{col: _coerce(val, max_cell_chars) for col, val in zip(columns, row, strict=False)} for row in rows],
"rowCount": len(rows),
"truncated": truncated,
}
try:
return await asyncio.wait_for(_run(), timeout=timeout_seconds)
except TimeoutError as exc:
raise WorkflowError(
"WORKFLOW_TIMEOUT",
f"数据库查询超时({timeout_seconds}s)",
retryable=True,
details={"timeoutSeconds": timeout_seconds},
) from exc
except WorkflowError:
raise
except Exception as exc: # noqa: BLE001 - never leak DSN/credentials upward
logger.warning("workflow sql_read failed", exc_info=True)
raise WorkflowError(
"WORKFLOW_SQL_FAILED",
"数据库查询执行失败",
retryable=True,
details={"reason": type(exc).__name__},
) from exc
__all__ = ["dispose_engines", "execute_read_query"]
@@ -0,0 +1,54 @@
"""Load frozen workflow sample graphs shipped with the harness."""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from deerflow.workflows.schemas import WorkflowGraph
_SAMPLES_DIR = Path(__file__).resolve().parent
_NODES_DIR = _SAMPLES_DIR / "nodes"
def samples_dir() -> Path:
return _SAMPLES_DIR
def nodes_dir() -> Path:
return _NODES_DIR
@lru_cache(maxsize=32)
def load_sample_graph(name: str = "report_generation") -> WorkflowGraph:
"""Load and validate a sample graph by stem name (without ``.json``)."""
path = _SAMPLES_DIR / f"{name}.json"
if not path.is_file():
raise FileNotFoundError(f"workflow sample not found: {path}")
raw = json.loads(path.read_text(encoding="utf-8"))
return WorkflowGraph.model_validate(raw)
@lru_cache(maxsize=64)
def load_node_sample(node_type: str) -> dict:
"""Load a minimal single-node fixture (raw dict, not a full graph)."""
path = _NODES_DIR / f"{node_type}.json"
if not path.is_file():
raise FileNotFoundError(f"node sample not found: {path}")
return json.loads(path.read_text(encoding="utf-8"))
def list_node_samples() -> list[str]:
if not _NODES_DIR.is_dir():
return []
return sorted(p.stem for p in _NODES_DIR.glob("*.json"))
__all__ = [
"list_node_samples",
"load_node_sample",
"load_sample_graph",
"nodes_dir",
"samples_dir",
]
@@ -0,0 +1,15 @@
{
"schemaVersion": "1.0",
"runId": "run_01",
"workflowId": "wf_report_generation",
"versionId": "wv_03",
"seq": 42,
"event": "node.output.delta",
"nodeId": "agent_write",
"nodeRunId": "nr_09",
"timestamp": "2026-08-26T10:00:00+08:00",
"data": {
"channel": "answer",
"delta": "第一段内容"
}
}
@@ -0,0 +1,13 @@
{
"id": "agent_1",
"type": "agent",
"name": "智能体",
"config": {
"agentId": "report-writer",
"promptTemplate": "根据材料撰写:{{ nodes.merge_1.data }}",
"responseMode": "text",
"threadMode": "isolated_per_node",
"timeoutSeconds": 600,
"skillNames": []
}
}
@@ -0,0 +1,11 @@
{
"id": "code_1",
"type": "code",
"name": "代码(沙箱)",
"config": {
"language": "python",
"source": "def main(inputs):\n rows = inputs.get(\"rows\") or []\n return {\"count\": len(rows), \"titles\": [r.get(\"title\") for r in rows[:10]]}\n",
"timeoutSeconds": 60,
"allowNetwork": false
}
}
@@ -0,0 +1,18 @@
{
"id": "cond_1",
"type": "condition",
"name": "条件",
"config": {
"branches": [
{
"name": "passed",
"expression": "{{ nodes.agent_review.data.passed }} == true"
},
{
"name": "rejected",
"expression": "{{ nodes.agent_review.data.passed }} == false"
}
],
"defaultBranch": "rejected"
}
}
@@ -0,0 +1,16 @@
{
"id": "http_1",
"type": "http",
"name": "HTTP",
"config": {
"method": "GET",
"url": "https://api.example.internal/items/{{ inputs.taskId }}",
"headers": { "Accept": "application/json" },
"query": {},
"body": null,
"credentialRef": "cred_task_api",
"timeoutSeconds": 30,
"maxResponseBytes": 1048576,
"allowRedirects": false
}
}
@@ -0,0 +1,20 @@
{
"id": "human_1",
"type": "human_input",
"name": "人工介入",
"config": {
"formSchema": {
"type": "object",
"required": ["decision"],
"properties": {
"decision": {
"type": "string",
"enum": ["approve", "revise"]
},
"comment": { "type": "string" }
}
},
"prompt": "请确认是否放行当前报告草稿。",
"actions": ["approve", "revise"]
}
}
@@ -0,0 +1,10 @@
{
"id": "loop_1",
"type": "loop",
"name": "受限循环",
"config": {
"maxIterations": 2,
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
"bodyEntry": "agent_write"
}
}
@@ -0,0 +1,9 @@
{
"id": "merge_1",
"type": "merge",
"name": "合并",
"config": {
"strategy": "all",
"sources": ["sql_1", "http_1", "agent_collect"]
}
}
@@ -0,0 +1,11 @@
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": {
"reportMarkdown": "{{ nodes.agent_write.data.answer }}"
},
"llmRepair": false
}
}
@@ -0,0 +1,12 @@
{
"id": "skill_1",
"type": "skill",
"name": "技能(agent_skill)",
"config": {
"mode": "agent_skill",
"agentId": "report-collector",
"skillNames": ["web-research"],
"promptTemplate": "使用指定技能完成:{{ inputs.topic }}",
"timeoutSeconds": 300
}
}
@@ -0,0 +1,13 @@
{
"id": "sql_1",
"type": "sql_read",
"name": "SQL 只读",
"config": {
"dataSourceId": "ds_report_readonly",
"statement": "SELECT id, title FROM report_source WHERE task_id = :task_id",
"parameters": {
"task_id": "{{ inputs.taskId }}"
},
"maxRows": 1000
}
}
@@ -0,0 +1,6 @@
{
"id": "start_1",
"type": "start",
"name": "开始",
"config": {}
}
@@ -0,0 +1,13 @@
{
"id": "sub_1",
"type": "subworkflow",
"name": "子工作流",
"config": {
"workflowId": "wf_collect_pack",
"versionId": "wv_01",
"inputMapping": {
"topic": "{{ inputs.topic }}",
"taskId": "{{ inputs.taskId }}"
}
}
}
@@ -0,0 +1,20 @@
{
"id": "transform_1",
"type": "transform",
"name": "模板/转换",
"config": {
"operations": [
{
"op": "map",
"from": "{{ nodes.sql_1.data.rows }}",
"to": "rows"
},
{
"op": "template",
"template": "# {{ inputs.topic }}\n\n共 {{ nodes.sql_1.metadata.rowCount }} 行。",
"to": "markdown"
}
],
"template": null
}
}
@@ -0,0 +1,260 @@
{
"schemaVersion": "1.0",
"id": "wf_report_generation",
"name": "专题报告生成",
"description": "阶段0验收基线:并行取数 → 合并 → 整理 → 撰写 → 审核 → 条件退回/输出",
"inputSchema": {
"type": "object",
"required": ["topic", "taskId"],
"properties": {
"topic": { "type": "string", "description": "报告主题" },
"taskId": { "type": "string", "description": "业务任务 ID" },
"dateRange": {
"type": "object",
"properties": {
"start": { "type": "string" },
"end": { "type": "string" }
}
}
}
},
"outputSchema": {
"type": "object",
"required": ["reportMarkdown", "passed"],
"properties": {
"reportMarkdown": { "type": "string" },
"passed": { "type": "boolean" },
"artifacts": { "type": "array" },
"issues": { "type": "array" }
}
},
"nodes": [
{
"id": "start_1",
"type": "start",
"name": "开始",
"config": {}
},
{
"id": "sql_1",
"type": "sql_read",
"name": "查询内部结构化数据",
"config": {
"dataSourceId": "ds_report_readonly",
"statement": "SELECT id, title, content, updated_at FROM report_source WHERE task_id = :task_id ORDER BY updated_at DESC",
"parameters": {
"task_id": "{{ inputs.taskId }}"
},
"maxRows": 1000
}
},
{
"id": "http_1",
"type": "http",
"name": "调用只读业务接口",
"config": {
"method": "GET",
"url": "https://api.example.internal/tasks/{{ inputs.taskId }}/summary",
"headers": {
"Accept": "application/json"
},
"credentialRef": "cred_task_api",
"timeoutSeconds": 30,
"maxResponseBytes": 524288,
"allowRedirects": false
}
},
{
"id": "agent_collect",
"type": "agent",
"name": "外部材料采集",
"config": {
"agentId": "report-collector",
"promptTemplate": "围绕主题「{{ inputs.topic }}」搜集外部公开材料,任务 ID:{{ inputs.taskId }}。输出 JSON:{sources:[], notes:string}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["sources", "notes"],
"properties": {
"sources": { "type": "array" },
"notes": { "type": "string" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600,
"skillNames": ["web-research"]
}
},
{
"id": "merge_1",
"type": "merge",
"name": "合并材料并保留来源",
"config": {
"strategy": "all",
"sources": ["sql_1", "http_1", "agent_collect"]
}
},
{
"id": "agent_clean",
"type": "agent",
"name": "数据整理与事实去重",
"config": {
"agentId": "report-cleaner",
"promptTemplate": "整理并去重以下材料,输出严格 JSON:{{ nodes.merge_1.data }}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["facts", "sources"],
"properties": {
"facts": { "type": "array" },
"sources": { "type": "array" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600
}
},
{
"id": "agent_write",
"type": "agent",
"name": "撰写 Markdown 报告",
"config": {
"agentId": "report-writer",
"promptTemplate": "根据整理结果撰写专题报告(Markdown)。主题:{{ inputs.topic }}\n材料:{{ nodes.agent_clean.data }}\n若存在审核意见:{{ nodes.agent_review.data.revisedAdvice }}",
"responseMode": "text",
"threadMode": "isolated_per_node",
"timeoutSeconds": 900
}
},
{
"id": "agent_review",
"type": "agent",
"name": "审核报告",
"config": {
"agentId": "report-reviewer",
"promptTemplate": "审核以下报告,输出 JSON {passed:boolean, issues:array, revisedAdvice:string}:\n{{ nodes.agent_write.data.answer }}",
"responseMode": "json",
"responseSchema": {
"type": "object",
"required": ["passed", "issues", "revisedAdvice"],
"properties": {
"passed": { "type": "boolean" },
"issues": { "type": "array" },
"revisedAdvice": { "type": "string" }
}
},
"threadMode": "isolated_per_node",
"timeoutSeconds": 600
}
},
{
"id": "cond_review",
"type": "condition",
"name": "审核结果分支",
"config": {
"branches": [
{
"name": "passed",
"expression": "{{ nodes.agent_review.data.passed }} == true"
},
{
"name": "rejected",
"expression": "{{ nodes.agent_review.data.passed }} == false"
}
],
"defaultBranch": "rejected"
}
},
{
"id": "loop_revise",
"type": "loop",
"name": "退回修改(最多 2 次)",
"config": {
"maxIterations": 2,
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
"bodyEntry": "agent_write"
}
},
{
"id": "transform_pack",
"type": "transform",
"name": "组装最终字段",
"config": {
"operations": [
{
"op": "map",
"from": "{{ nodes.agent_write.data.answer }}",
"to": "reportMarkdown"
},
{
"op": "map",
"from": "{{ nodes.agent_review.data.passed }}",
"to": "passed"
},
{
"op": "map",
"from": "{{ nodes.agent_review.data.issues }}",
"to": "issues"
}
]
}
},
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": {
"reportMarkdown": "{{ nodes.transform_pack.data.reportMarkdown }}",
"passed": "{{ nodes.transform_pack.data.passed }}",
"issues": "{{ nodes.transform_pack.data.issues }}"
},
"llmRepair": false
}
}
],
"edges": [
{ "id": "e_start_sql", "source": "start_1", "target": "sql_1" },
{ "id": "e_start_http", "source": "start_1", "target": "http_1" },
{ "id": "e_start_collect", "source": "start_1", "target": "agent_collect" },
{ "id": "e_sql_merge", "source": "sql_1", "target": "merge_1" },
{ "id": "e_http_merge", "source": "http_1", "target": "merge_1" },
{ "id": "e_collect_merge", "source": "agent_collect", "target": "merge_1" },
{ "id": "e_merge_clean", "source": "merge_1", "target": "agent_clean" },
{ "id": "e_clean_write", "source": "agent_clean", "target": "agent_write" },
{ "id": "e_write_review", "source": "agent_write", "target": "agent_review" },
{ "id": "e_review_cond", "source": "agent_review", "target": "cond_review" },
{
"id": "e_cond_pass",
"source": "cond_review",
"target": "transform_pack",
"sourcePort": "passed"
},
{
"id": "e_cond_reject",
"source": "cond_review",
"target": "loop_revise",
"sourcePort": "rejected"
},
{
"id": "e_loop_write",
"source": "loop_revise",
"target": "agent_write",
"sourcePort": "continue"
},
{
"id": "e_loop_fail",
"source": "loop_revise",
"target": "transform_pack",
"sourcePort": "exhausted"
},
{ "id": "e_pack_out", "source": "transform_pack", "target": "output_1" }
],
"settings": {
"runTimeoutSeconds": 1800,
"nodeTimeoutSeconds": 300,
"maxSteps": 200,
"maxLoopIterations": 5,
"maxParallelism": 4
}
}
@@ -0,0 +1,41 @@
{
"schemaVersion": "1.0",
"id": "wf_start_output",
"name": "空图:开始→输出",
"description": "阶段2运行骨架用的最小可执行图",
"inputSchema": {
"type": "object",
"properties": {
"message": { "type": "string" }
}
},
"outputSchema": {
"type": "object",
"required": ["message"],
"properties": {
"message": { "type": "string" }
}
},
"nodes": [
{ "id": "start_1", "type": "start", "name": "开始", "config": {} },
{
"id": "output_1",
"type": "output",
"name": "输出",
"config": {
"mapping": { "message": "{{ inputs.message }}" },
"llmRepair": false
}
}
],
"edges": [
{ "id": "e1", "source": "start_1", "target": "output_1" }
],
"settings": {
"runTimeoutSeconds": 60,
"nodeTimeoutSeconds": 30,
"maxSteps": 10,
"maxLoopIterations": 1,
"maxParallelism": 1
}
}
@@ -0,0 +1,73 @@
"""Full JSON Schema validation for workflow input/output values.
The engine used to check only ``required`` keys; everything else (wrong type,
unknown enum value, nested ``properties``/``items`` violations) slipped through
until downstream nodes exploded with confusing errors. This module runs the
graph's ``inputSchema`` / ``outputSchema`` through ``Draft202012Validator`` and
returns structured, frontend-friendly errors (path + message) instead of
leaking Python tracebacks.
Kept as pure functions so both the node executors (``WorkflowError``) and the
HTTP router (400 responses) can share the exact same rules.
"""
from __future__ import annotations
from typing import Any
from jsonschema import Draft202012Validator
# Cap the reported errors so a wildly invalid payload cannot produce a
# multi-megabyte error body.
_MAX_ERRORS = 20
def schema_issues(value: Any, schema: dict[str, Any] | None) -> list[dict[str, Any]]:
"""Validate ``value`` against ``schema``.
Returns ``[]`` when the schema is empty/absent (nothing declared = nothing
enforced). Each issue is ``{"path": [...], "message": str}`` with ``path``
pointing at the offending field, e.g. ``["options", 0, "name"]``.
"""
if not isinstance(schema, dict) or not schema:
return []
try:
validator = Draft202012Validator(schema)
# Construction alone does not check the schema — malformed keywords
# (e.g. a non-string ``pattern``) only explode during validation.
errors = sorted(validator.iter_errors(value), key=lambda err: list(err.absolute_path))
except Exception: # noqa: BLE001 - a malformed user schema must not 500
return [{"path": [], "message": "schema 定义本身无效,无法用于校验"}]
out: list[dict[str, Any]] = []
seen: set[tuple[Any, ...]] = set()
for err in errors:
path = [str(p) if not isinstance(p, int) else p for p in err.absolute_path]
key = tuple(path)
if key in seen:
continue
seen.add(key)
out.append({"path": path, "message": _humanize(err)})
if len(out) >= _MAX_ERRORS:
out.append({"path": [], "message": f"……其余问题已省略(共 {len(errors)} 处)"})
break
return out
def _humanize(err: Any) -> str:
"""Map the common jsonschema phrasings to short Chinese messages."""
message = str(err.message)
validator = getattr(err, "validator", "")
path = ".".join(str(p) for p in err.absolute_path) or "输入"
if validator == "required":
missing = message.split("'")[1] if "'" in message else "?"
return f"{path} 缺少必填字段:{missing}"
if validator == "enum":
return f"{path} 的值不在允许范围内:{message}"
if validator == "type":
return f"{path} 类型不正确:{message}"
if validator == "additionalProperties":
return f"{path} 含有未声明的额外字段:{message}"
return f"{path} 不符合 schema:{message}"
__all__ = ["schema_issues"]
@@ -0,0 +1,374 @@
"""Frozen workflow graph schema v1.0 (internal DTO).
Coze canvas DTO is converted into this structure at the Gateway boundary.
The runtime must never depend on Coze-private fields.
Wire JSON uses camelCase aliases; Python code uses snake_case.
"""
from __future__ import annotations
from hashlib import sha256
from typing import Any, Literal
from pydantic import BaseModel, Field, field_validator, model_validator
SCHEMA_VERSION = "1.0"
NodeType = Literal[
"start",
"output",
"agent",
"skill",
"http",
"sql_read",
"code",
"transform",
"condition",
"merge",
"human_input",
"subworkflow",
"loop",
]
ALL_NODE_TYPES: tuple[NodeType, ...] = (
"start",
"output",
"agent",
"skill",
"http",
"sql_read",
"code",
"transform",
"condition",
"merge",
"human_input",
"subworkflow",
"loop",
)
RunStatus = Literal[
"queued",
"running",
"awaiting_input",
"completed",
"failed",
"cancel_requested",
"cancelled",
]
ALL_RUN_STATUSES: tuple[RunStatus, ...] = (
"queued",
"running",
"awaiting_input",
"completed",
"failed",
"cancel_requested",
"cancelled",
)
TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset({"completed", "failed", "cancelled"})
NodeRunStatus = Literal[
"queued",
"running",
"completed",
"failed",
"skipped",
"cancelled",
]
SkillMode = Literal["agent_skill", "callable_skill"]
MergeStrategy = Literal["all", "first_success", "any_success"]
HttpMethod = Literal["GET", "POST", "PUT", "PATCH", "DELETE"]
AgentResponseMode = Literal["text", "json"]
AgentThreadMode = Literal["isolated_per_node", "shared_run_thread"]
ExecutionMode = Literal["normal"]
class WorkflowGraphSettings(BaseModel):
"""Per-graph resource limits (still capped by system limits at publish)."""
run_timeout_seconds: int = Field(default=1800, ge=1, alias="runTimeoutSeconds")
node_timeout_seconds: int = Field(default=300, ge=1, alias="nodeTimeoutSeconds")
max_steps: int = Field(default=200, ge=1, alias="maxSteps")
max_loop_iterations: int = Field(default=5, ge=1, alias="maxLoopIterations")
max_parallelism: int = Field(default=4, ge=1, alias="maxParallelism")
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowNode(BaseModel):
"""A single node on the internal standard graph."""
id: str = Field(min_length=1)
type: NodeType
name: str = ""
config: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
@field_validator("id")
@classmethod
def _id_not_blank(cls, value: str) -> str:
stripped = value.strip()
if not stripped:
raise ValueError("node id must be non-empty")
return stripped
class WorkflowEdge(BaseModel):
"""Directed edge. ``source_port`` names a condition / loop outlet when set."""
id: str = Field(min_length=1)
source: str
target: str
source_port: str | None = Field(default=None, alias="sourcePort")
model_config = {"extra": "forbid", "populate_by_name": True}
class WorkflowGraph(BaseModel):
"""Internal standard workflow graph (publish / run snapshot)."""
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
id: str = Field(min_length=1)
name: str = ""
description: str = ""
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
nodes: list[WorkflowNode] = Field(default_factory=list)
edges: list[WorkflowEdge] = Field(default_factory=list)
settings: WorkflowGraphSettings = Field(default_factory=WorkflowGraphSettings)
model_config = {"extra": "forbid", "populate_by_name": True}
@model_validator(mode="after")
def _unique_ids(self) -> WorkflowGraph:
node_ids = [n.id for n in self.nodes]
if len(node_ids) != len(set(node_ids)):
raise ValueError("duplicate node id")
edge_ids = [e.id for e in self.edges]
if len(edge_ids) != len(set(edge_ids)):
raise ValueError("duplicate edge id")
known = set(node_ids)
for edge in self.edges:
if edge.source not in known:
raise ValueError(f"edge {edge.id} source not found: {edge.source}")
if edge.target not in known:
raise ValueError(f"edge {edge.id} target not found: {edge.target}")
return self
def node_map(self) -> dict[str, WorkflowNode]:
return {node.id: node for node in self.nodes}
def canonical_json(self) -> str:
"""Deterministic JSON used for ``graph_hash`` (sorted keys, no whitespace)."""
return self.model_dump_json(by_alias=True, exclude_none=True)
def graph_hash(self) -> str:
return sha256(self.canonical_json().encode("utf-8")).hexdigest()
class ArtifactRef(BaseModel):
"""Reference to a large/binary result; full bytes live outside events."""
artifact_id: str = Field(alias="artifactId")
name: str
mime_type: str = Field(default="application/octet-stream", alias="mimeType")
path: str = ""
size_bytes: int | None = Field(default=None, alias="sizeBytes")
preview: str | None = None
model_config = {"extra": "forbid", "populate_by_name": True}
class NodeResult(BaseModel):
"""Uniform result returned by every node executor."""
data: dict[str, Any] = Field(default_factory=dict)
messages: list[dict[str, Any]] = Field(default_factory=list)
artifacts: list[ArtifactRef] = Field(default_factory=list)
metadata: dict[str, Any] = Field(default_factory=dict)
warnings: list[str] = Field(default_factory=list)
model_config = {"extra": "forbid", "populate_by_name": True}
class StartRunRequest(BaseModel):
"""Body for ``POST /api/workflows/{workflow_id}/runs``."""
version_id: str = Field(alias="versionId")
inputs: dict[str, Any] = Field(default_factory=dict)
idempotency_key: str | None = Field(default=None, alias="idempotencyKey")
execution_mode: ExecutionMode = Field(default="normal", alias="executionMode")
model_config = {"extra": "forbid", "populate_by_name": True}
class ResumeRunRequest(BaseModel):
"""Body for ``POST /api/workflows/runs/{run_id}/resume``."""
resume_token: str = Field(alias="resumeToken")
action: str
payload: dict[str, Any] = Field(default_factory=dict)
model_config = {"extra": "forbid", "populate_by_name": True}
# ── Typed node config helpers (optional; validators may use later) ───────────
class AgentNodeConfig(BaseModel):
agent_id: str = Field(alias="agentId")
prompt_template: str = Field(default="", alias="promptTemplate")
response_mode: AgentResponseMode = Field(default="text", alias="responseMode")
response_schema: dict[str, Any] = Field(default_factory=dict, alias="responseSchema")
thread_mode: AgentThreadMode = Field(default="isolated_per_node", alias="threadMode")
timeout_seconds: int = Field(default=600, ge=1, alias="timeoutSeconds")
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
model_config = {"extra": "forbid", "populate_by_name": True}
class SkillNodeConfig(BaseModel):
mode: SkillMode = "agent_skill"
agent_id: str | None = Field(default=None, alias="agentId")
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
prompt_template: str = Field(default="", alias="promptTemplate")
# callable_skill fields (phase 4+; accepted in schema, not executed in phase 0)
entry: str | None = None
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
timeout_seconds: int = Field(default=300, ge=1, alias="timeoutSeconds")
model_config = {"extra": "forbid", "populate_by_name": True}
class HttpNodeConfig(BaseModel):
method: HttpMethod = "GET"
url: str
headers: dict[str, str] = Field(default_factory=dict)
query: dict[str, Any] = Field(default_factory=dict)
body: Any = None
credential_ref: str | None = Field(default=None, alias="credentialRef")
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
max_response_bytes: int = Field(default=1_048_576, ge=1, alias="maxResponseBytes")
allow_redirects: bool = Field(default=False, alias="allowRedirects")
model_config = {"extra": "forbid", "populate_by_name": True}
class SqlReadNodeConfig(BaseModel):
data_source_id: str = Field(alias="dataSourceId")
statement: str
parameters: dict[str, Any] = Field(default_factory=dict)
max_rows: int = Field(default=1000, ge=1, alias="maxRows")
model_config = {"extra": "forbid", "populate_by_name": True}
class CodeNodeConfig(BaseModel):
language: Literal["python"] = "python"
source: str
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
allow_network: bool = Field(default=False, alias="allowNetwork")
model_config = {"extra": "forbid", "populate_by_name": True}
class TransformNodeConfig(BaseModel):
operations: list[dict[str, Any]] = Field(default_factory=list)
template: str | None = None
model_config = {"extra": "forbid", "populate_by_name": True}
class ConditionBranch(BaseModel):
name: str
expression: str
model_config = {"extra": "forbid", "populate_by_name": True}
class ConditionNodeConfig(BaseModel):
branches: list[ConditionBranch] = Field(default_factory=list)
default_branch: str | None = Field(default=None, alias="defaultBranch")
model_config = {"extra": "forbid", "populate_by_name": True}
class MergeNodeConfig(BaseModel):
strategy: MergeStrategy = "all"
sources: list[str] = Field(default_factory=list)
model_config = {"extra": "forbid", "populate_by_name": True}
class HumanInputNodeConfig(BaseModel):
form_schema: dict[str, Any] = Field(default_factory=dict, alias="formSchema")
prompt: str = ""
actions: list[str] = Field(default_factory=lambda: ["submit"])
model_config = {"extra": "forbid", "populate_by_name": True}
class SubworkflowNodeConfig(BaseModel):
workflow_id: str = Field(alias="workflowId")
version_id: str = Field(alias="versionId")
input_mapping: dict[str, str] = Field(default_factory=dict, alias="inputMapping")
model_config = {"extra": "forbid", "populate_by_name": True}
class LoopNodeConfig(BaseModel):
max_iterations: int = Field(default=5, ge=1, alias="maxIterations")
continue_when: str = Field(default="", alias="continueWhen")
body_entry: str = Field(alias="bodyEntry")
model_config = {"extra": "forbid", "populate_by_name": True}
class OutputNodeConfig(BaseModel):
mapping: dict[str, str] = Field(default_factory=dict)
llm_repair: bool = Field(default=False, alias="llmRepair")
model_config = {"extra": "forbid", "populate_by_name": True}
__all__ = [
"SCHEMA_VERSION",
"ALL_NODE_TYPES",
"ALL_RUN_STATUSES",
"TERMINAL_RUN_STATUSES",
"AgentNodeConfig",
"AgentResponseMode",
"AgentThreadMode",
"ArtifactRef",
"CodeNodeConfig",
"ConditionBranch",
"ConditionNodeConfig",
"ExecutionMode",
"HttpMethod",
"HttpNodeConfig",
"HumanInputNodeConfig",
"LoopNodeConfig",
"MergeNodeConfig",
"MergeStrategy",
"NodeResult",
"NodeRunStatus",
"NodeType",
"OutputNodeConfig",
"ResumeRunRequest",
"RunStatus",
"SkillMode",
"SkillNodeConfig",
"SqlReadNodeConfig",
"StartRunRequest",
"SubworkflowNodeConfig",
"TransformNodeConfig",
"WorkflowEdge",
"WorkflowGraph",
"WorkflowGraphSettings",
"WorkflowNode",
]
@@ -0,0 +1,18 @@
"""Security policies for external-effect workflow nodes."""
from __future__ import annotations
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
from deerflow.workflows.security.sql_policy import (
assert_parameters_bound,
assert_read_only,
enforce_limit,
)
__all__ = [
"assert_parameters_bound",
"assert_read_only",
"enforce_http_target",
"enforce_limit",
"enforce_redirect",
]
@@ -0,0 +1,114 @@
"""Egress policy for the ``http`` node (SSRF defence).
Checks, in order: scheme → host allow/deny lists → port → every resolved IP
address. Resolving the host and validating *all* A/AAAA answers is what stops
`http://internal.example.com` from pointing at `169.254.169.254` or `127.0.0.1`.
Known limitation: we validate the resolved addresses and then let httpx dial the
hostname again, so a DNS answer that changes between the two lookups is not
covered (classic DNS-rebinding TOCTOU). Closing that fully requires dialing the
pinned IP with SNI/Host overrides; for now the deployment guidance is to keep
``allow_private_networks`` false and prefer an explicit ``allowed_hosts`` list.
"""
from __future__ import annotations
import asyncio
import ipaddress
import socket
from urllib.parse import urlparse, urlsplit
from deerflow.config.workflow_config import WorkflowHttpPolicyConfig
from deerflow.workflows.errors import WorkflowError
_ALLOWED_SCHEMES = ("http", "https")
_DEFAULT_PORTS = {"http": 80, "https": 443}
# Cloud metadata endpoints get their own explicit deny so a permissive
# intranet configuration cannot accidentally expose instance credentials.
_METADATA_HOSTS = frozenset({"169.254.169.254", "metadata.google.internal", "metadata"})
def _denied(message: str, **details: object) -> WorkflowError:
return WorkflowError("WORKFLOW_HTTP_POLICY_DENIED", message, details=dict(details))
def _host_matches(host: str, patterns: list[str]) -> bool:
target = host.lower().rstrip(".")
for raw in patterns:
pattern = str(raw).strip().lower().rstrip(".")
if not pattern:
continue
if pattern.startswith("."):
if target == pattern[1:] or target.endswith(pattern):
return True
elif pattern.startswith("*."):
if target == pattern[2:] or target.endswith(pattern[1:]):
return True
elif target == pattern:
return True
return False
def _is_private(address: str) -> bool:
try:
ip = ipaddress.ip_address(address)
except ValueError:
return False
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified
async def _resolve(host: str, port: int) -> list[str]:
loop = asyncio.get_running_loop()
try:
infos = await loop.getaddrinfo(host, port, type=socket.SOCK_STREAM)
except OSError as exc:
raise WorkflowError(
"WORKFLOW_HTTP_FAILED",
"无法解析目标域名",
retryable=True,
details={"host": host, "reason": type(exc).__name__},
) from exc
return sorted({str(info[4][0]) for info in infos})
async def enforce_http_target(url: str, policy: WorkflowHttpPolicyConfig) -> dict[str, object]:
"""Validate a target URL. Returns the resolved metadata or raises."""
if not policy.enabled:
raise _denied("HTTP 节点已被系统禁用")
parsed = urlparse(url)
scheme = (parsed.scheme or "").lower()
if scheme not in _ALLOWED_SCHEMES:
raise _denied("仅允许 http/https 协议", scheme=scheme or "")
host = (parsed.hostname or "").lower()
if not host:
raise _denied("URL 缺少主机名")
port = parsed.port or _DEFAULT_PORTS[scheme]
if host in _METADATA_HOSTS:
raise _denied("目标为云元数据地址,已拒绝", host=host)
if _host_matches(host, policy.blocked_hosts):
raise _denied("目标主机在禁止名单中", host=host)
if policy.allowed_hosts and not _host_matches(host, policy.allowed_hosts):
raise _denied("目标主机不在允许名单中", host=host)
if policy.allowed_ports and port not in policy.allowed_ports:
raise _denied("目标端口不被允许", port=port)
addresses = await _resolve(host, port)
if not policy.allow_private_networks:
private = [addr for addr in addresses if _is_private(addr)]
if private:
raise _denied("目标解析到内网/环回地址,已拒绝", host=host)
if any(addr in _METADATA_HOSTS for addr in addresses):
raise _denied("目标解析到云元数据地址,已拒绝", host=host)
return {"host": host, "port": port, "scheme": scheme, "addresses": addresses}
async def enforce_redirect(location: str, base_url: str, policy: WorkflowHttpPolicyConfig) -> str:
"""Validate a redirect hop, resolving relative locations against the base."""
if location.startswith("/"):
base = urlsplit(base_url)
location = f"{base.scheme}://{base.netloc}{location}"
await enforce_http_target(location, policy)
return location
__all__ = ["enforce_http_target", "enforce_redirect"]
@@ -0,0 +1,97 @@
"""Envelope encryption for workflow secrets (data-source DSNs, credentials).
The key comes from ``WORKFLOW_SECRET_KEY`` (or ``DEERFLOW_SECRET_KEY``) and is
stretched with SHA-256 into a Fernet key, so operators can supply any
sufficiently long passphrase.
When no key is configured we refuse to encrypt rather than silently storing
plaintext: the caller decides whether to reject the write or fall back. That
choice is deliberate — a silent plaintext fallback is how credentials end up in
database backups without anyone noticing.
"""
from __future__ import annotations
import base64
import hashlib
import logging
import os
logger = logging.getLogger(__name__)
_ENV_KEYS = ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY")
_PREFIX = "enc:v1:"
class SecretKeyMissingError(RuntimeError):
def __init__(self) -> None:
super().__init__("未配置 WORKFLOW_SECRET_KEY,无法加密保存敏感连接信息")
def _fernet():
key_material = ""
for name in _ENV_KEYS:
value = os.environ.get(name)
if value and value.strip():
key_material = value.strip()
break
if not key_material:
raise SecretKeyMissingError()
try:
from cryptography.fernet import Fernet
except ImportError as exc: # pragma: no cover - dependency is declared
raise SecretKeyMissingError() from exc
digest = hashlib.sha256(key_material.encode("utf-8")).digest()
return Fernet(base64.urlsafe_b64encode(digest))
def secrets_available() -> bool:
try:
_fernet()
except Exception: # noqa: BLE001 - any failure means "cannot encrypt"
return False
return True
def encrypt_secret(plaintext: str) -> str:
token = _fernet().encrypt(plaintext.encode("utf-8")).decode("ascii")
return f"{_PREFIX}{token}"
def decrypt_secret(stored: str) -> str:
if not stored:
return ""
if not stored.startswith(_PREFIX):
# Legacy / externally seeded plaintext row.
return stored
return _fernet().decrypt(stored[len(_PREFIX) :].encode("ascii")).decode("utf-8")
def is_encrypted(stored: str) -> bool:
return bool(stored) and stored.startswith(_PREFIX)
def mask_dsn(dsn: str) -> str:
"""Human-readable summary that never reveals user/password/host details."""
if not dsn:
return ""
scheme, _, rest = dsn.partition("://")
if not rest:
return f"{scheme[:24]}://***"
host_part = rest.split("@")[-1]
host = host_part.split("/")[0]
database = host_part.split("/", 1)[1].split("?")[0] if "/" in host_part else ""
masked_host = host.split(":")[0]
if len(masked_host) > 4:
masked_host = f"{masked_host[:2]}***{masked_host[-2:]}"
return f"{scheme}://***@{masked_host}/{database}"
__all__ = [
"SecretKeyMissingError",
"decrypt_secret",
"encrypt_secret",
"is_encrypted",
"mask_dsn",
"secrets_available",
]
@@ -0,0 +1,136 @@
"""Read-only statement policy for the ``sql_read`` node.
The node is *only* allowed to run a single read query. We reject anything that
is not a `SELECT`/`WITH`, anything containing a statement separator, and any
write/DDL/administrative keyword — after stripping comments, because
``SELECT 1 -- ; DROP TABLE t`` and ``/*!*/`` style tricks are exactly how these
filters get bypassed.
Parameters are always bound (never interpolated), so this policy is a second
line of defence rather than the only one. Production deployments should also
point the data source at a read-only database account.
"""
from __future__ import annotations
import re
from deerflow.workflows.errors import WorkflowError
_FORBIDDEN_KEYWORDS = (
"insert",
"update",
"delete",
"drop",
"alter",
"create",
"truncate",
"replace into",
"grant",
"revoke",
"merge",
"call",
"execute",
"exec",
"copy",
"load_file",
"outfile",
"dumpfile",
"into outfile",
"pg_read_file",
"pg_sleep",
"sleep",
"benchmark",
"shutdown",
"vacuum",
"attach",
"detach",
"pragma",
"set ",
"lock",
"do ",
)
_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
_LINE_COMMENT = re.compile(r"(--|#)[^\n]*")
_PARAM_RE = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)")
def _denied(message: str, **details: object) -> WorkflowError:
return WorkflowError("WORKFLOW_SQL_POLICY_DENIED", message, details=dict(details))
def strip_comments(statement: str) -> str:
return _LINE_COMMENT.sub(" ", _BLOCK_COMMENT.sub(" ", statement))
def assert_read_only(statement: str) -> str:
"""Validate and return the normalised single-statement query."""
raw = (statement or "").strip()
if not raw:
raise _denied("SQL 语句为空")
cleaned = strip_comments(raw).strip()
if not cleaned:
raise _denied("SQL 语句仅包含注释")
body = cleaned.rstrip().rstrip(";").strip()
if ";" in body:
raise _denied("只允许执行单条语句")
normalized = re.sub(r"\s+", " ", body.lower()).strip()
if not (normalized.startswith("select") or normalized.startswith("with")):
raise _denied("只允许 SELECT / WITH 查询")
for keyword in _FORBIDDEN_KEYWORDS:
token = keyword.strip()
if re.search(rf"\b{re.escape(token)}\b", normalized):
raise _denied("语句包含被禁止的关键字", keyword=token)
return body
def enforce_limit(statement: str, max_rows: int) -> str:
"""Append a LIMIT when the query does not already bound its result size."""
lowered = statement.lower()
if re.search(r"\blimit\b", lowered) or re.search(r"\bfetch\s+first\b", lowered):
return statement
return f"{statement} LIMIT {int(max_rows)}"
_FROM_JOIN_RE = re.compile(r"\b(?:from|join)\s+([A-Za-z_][\w.]*)", re.IGNORECASE)
def _table_name(raw: str) -> str:
return raw.split(".")[-1].strip("`\"[]").lower()
def assert_tables_allowed(statement: str, allowed_tables: list[str]) -> None:
"""When the data source has a table allowlist, every FROM/JOIN target must
appear on it. An empty allowlist means no extra restriction."""
if not allowed_tables:
return
allowed = {item.lower() for item in allowed_tables} | {_table_name(item) for item in allowed_tables}
for name in _FROM_JOIN_RE.findall(strip_comments(statement)):
short = _table_name(name)
if short not in allowed and name.lower() not in allowed:
raise _denied("查询引用了未授权的表", table=name, allowedTables=list(allowed_tables))
def declared_parameters(statement: str) -> set[str]:
return set(_PARAM_RE.findall(strip_comments(statement)))
def assert_parameters_bound(statement: str, parameters: dict[str, object]) -> None:
missing = sorted(declared_parameters(statement) - set(parameters))
if missing:
raise WorkflowError(
"WORKFLOW_SQL_FAILED",
f"缺少查询参数:{'、'.join(missing)}",
details={"missing": missing},
)
__all__ = [
"assert_parameters_bound",
"assert_read_only",
"assert_tables_allowed",
"declared_parameters",
"enforce_limit",
"strip_comments",
]
@@ -0,0 +1,335 @@
"""Publish-time structural validator for workflow graphs (phase 1).
Full resource-permission and SQL EXPLAIN checks land in later phases.
This module never imports ``app.*``.
"""
from __future__ import annotations
from collections import defaultdict, deque
from collections.abc import Callable
from typing import Any
from pydantic import ValidationError
from deerflow.workflows.errors import WorkflowError, WorkflowErrorBody
from deerflow.workflows.expressions import collect_referenced_nodes
from deerflow.workflows.schemas import ALL_NODE_TYPES, WorkflowGraph
ResourceChecker = Callable[[str, dict[str, Any]], "str | dict[str, Any] | None"]
"""Return an error message when a node resource is missing/forbidden.
May instead return ``{"message": ..., "field": "config.agentId"}`` so API
responses can point the canvas form at the exact offending field. ``None``
means the node's resources check out."""
class ValidationIssue(WorkflowErrorBody):
"""Alias kept for readability in API responses."""
def _collect_reachable(start_ids: set[str], adjacency: dict[str, list[str]]) -> set[str]:
seen: set[str] = set()
queue: deque[str] = deque(start_ids)
while queue:
node_id = queue.popleft()
if node_id in seen:
continue
seen.add(node_id)
for nxt in adjacency.get(node_id, []):
if nxt not in seen:
queue.append(nxt)
return seen
def _has_uncontrolled_cycle(graph: WorkflowGraph) -> bool:
"""Detect cycles that are not entered via an explicit ``loop`` node.
Phase-1 heuristic: if the condensation of the graph (ignoring edges whose
source is a ``loop`` node) still has a back-edge, reject.
"""
loop_ids = {n.id for n in graph.nodes if n.type == "loop"}
adjacency: dict[str, list[str]] = defaultdict(list)
for edge in graph.edges:
if edge.source in loop_ids:
continue
adjacency[edge.source].append(edge.target)
visiting: set[str] = set()
visited: set[str] = set()
def dfs(node_id: str) -> bool:
visiting.add(node_id)
for nxt in adjacency.get(node_id, []):
if nxt in visiting:
return True
if nxt not in visited and dfs(nxt):
return True
visiting.discard(node_id)
visited.add(node_id)
return False
for node in graph.nodes:
if node.id not in visited and dfs(node.id):
return True
return False
def validate_workflow_graph(
raw: dict[str, Any] | WorkflowGraph,
*,
system_max_steps: int = 500,
system_max_loop_iterations: int = 20,
system_max_parallelism: int = 16,
resource_checker: ResourceChecker | None = None,
) -> list[ValidationIssue]:
"""Validate a graph. Empty list means publishable."""
issues: list[ValidationIssue] = []
try:
graph = raw if isinstance(raw, WorkflowGraph) else WorkflowGraph.model_validate(raw)
except ValidationError as exc:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="图结构不符合 schema v1.0",
details={"errors": exc.errors()},
)
)
return issues
starts = [n for n in graph.nodes if n.type == "start"]
outputs = [n for n in graph.nodes if n.type == "output"]
if len(starts) != 1:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message=f"工作流必须恰好有 1 个开始节点,当前为 {len(starts)}",
details={"count": len(starts)},
)
)
if len(outputs) < 1:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="工作流至少需要 1 个输出节点",
)
)
unknown = [n for n in graph.nodes if n.type not in ALL_NODE_TYPES]
for node in unknown:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message=f"未知节点类型: {node.type}",
node_id=node.id,
)
)
if _has_uncontrolled_cycle(graph):
issues.append(
ValidationIssue(
code="WORKFLOW_GRAPH_CYCLE",
message="检测到未受控环;循环必须通过显式 loop 节点",
)
)
if starts:
adjacency: dict[str, list[str]] = defaultdict(list)
for edge in graph.edges:
adjacency[edge.source].append(edge.target)
reachable = _collect_reachable({starts[0].id}, adjacency)
for node in graph.nodes:
if node.id not in reachable:
issues.append(
ValidationIssue(
code="WORKFLOW_NODE_NOT_FOUND",
message=f"节点不可从开始节点到达: {node.id}",
node_id=node.id,
)
)
for out in outputs:
if out.id not in reachable:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message=f"输出节点不可达: {out.id}",
node_id=out.id,
)
)
settings = graph.settings
if settings.max_steps > system_max_steps:
issues.append(
ValidationIssue(
code="WORKFLOW_LIMIT_EXCEEDED",
message=f"maxSteps 超过系统上限 {system_max_steps}",
details={"maxSteps": settings.max_steps, "limit": system_max_steps},
)
)
if settings.max_loop_iterations > system_max_loop_iterations:
issues.append(
ValidationIssue(
code="WORKFLOW_LIMIT_EXCEEDED",
message=f"maxLoopIterations 超过系统上限 {system_max_loop_iterations}",
details={
"maxLoopIterations": settings.max_loop_iterations,
"limit": system_max_loop_iterations,
},
)
)
if settings.max_parallelism > system_max_parallelism:
issues.append(
ValidationIssue(
code="WORKFLOW_LIMIT_EXCEEDED",
message=f"maxParallelism 超过系统上限 {system_max_parallelism}",
details={"maxParallelism": settings.max_parallelism, "limit": system_max_parallelism},
)
)
for node in graph.nodes:
if node.type == "condition":
branches = node.config.get("branches") or []
ports = {str(b.get("name")) for b in branches if isinstance(b, dict) and b.get("name")}
default_branch = node.config.get("defaultBranch") or node.config.get("default_branch")
if default_branch:
ports.add(str(default_branch))
edge_ports = {e.source_port for e in graph.edges if e.source == node.id and e.source_port}
missing = edge_ports - ports
for port in missing:
issues.append(
ValidationIssue(
code="WORKFLOW_EDGE_INVALID",
message=f"条件节点出口未声明: {port}",
node_id=node.id,
details={"sourcePort": port},
)
)
if node.type == "loop":
max_iter = node.config.get("maxIterations") or node.config.get("max_iterations")
if max_iter is None:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="loop 节点必须配置 maxIterations",
node_id=node.id,
)
)
body_entry = node.config.get("bodyEntry") or node.config.get("body_entry")
if not body_entry:
issues.append(
ValidationIssue(
code="WORKFLOW_SCHEMA_INVALID",
message="loop 节点必须配置 bodyEntry",
node_id=node.id,
)
)
elif body_entry not in graph.node_map():
issues.append(
ValidationIssue(
code="WORKFLOW_NODE_NOT_FOUND",
message=f"loop bodyEntry 不存在: {body_entry}",
node_id=node.id,
)
)
if resource_checker is not None:
verdict = resource_checker(node.type, node.config or {})
if verdict:
message = verdict if isinstance(verdict, str) else str(verdict.get("message") or "节点资源配置无效")
field = verdict.get("field") if isinstance(verdict, dict) else None
issues.append(
ValidationIssue(
code="WORKFLOW_RESOURCE_MISSING",
message=message,
node_id=node.id,
details={"field": field} if field else None,
)
)
issues.extend(_check_upstream_references(graph))
return issues
def _predecessors(graph: WorkflowGraph) -> dict[str, set[str]]:
preds: dict[str, set[str]] = defaultdict(set)
for edge in graph.edges:
preds[edge.target].add(edge.source)
return preds
def _ancestors(node_id: str, preds: dict[str, set[str]]) -> set[str]:
seen: set[str] = set()
stack = list(preds.get(node_id) or ())
while stack:
current = stack.pop()
if current in seen:
continue
seen.add(current)
stack.extend(preds.get(current) or ())
return seen
def _check_upstream_references(graph: WorkflowGraph) -> list[ValidationIssue]:
"""§8.1: node input references must come from upstream nodes."""
issues: list[ValidationIssue] = []
known = graph.node_map()
preds = _predecessors(graph)
for node in graph.nodes:
refs = collect_referenced_nodes(node.config)
ancestors = _ancestors(node.id, preds)
for ref in sorted(refs):
if ref not in known:
issues.append(
ValidationIssue(
code="WORKFLOW_NODE_NOT_FOUND",
message=f"表达式引用了不存在的节点: {ref}",
node_id=node.id,
details={"referencedNodeId": ref},
)
)
continue
if ref == node.id and node.type != "loop":
issues.append(
ValidationIssue(
code="WORKFLOW_EXPRESSION_INVALID",
message="节点不能引用自身输出",
node_id=node.id,
details={"referencedNodeId": ref},
)
)
continue
if ref != node.id and ref not in ancestors:
issues.append(
ValidationIssue(
code="WORKFLOW_EXPRESSION_INVALID",
message=f"节点引用必须来自上游,{ref} 不在 {node.id} 的上游",
node_id=node.id,
details={"referencedNodeId": ref},
)
)
return issues
def assert_publishable(raw: dict[str, Any] | WorkflowGraph, **kwargs: Any) -> WorkflowGraph:
"""Validate and return the typed graph, or raise ``WorkflowError``."""
issues = validate_workflow_graph(raw, **kwargs)
if issues:
first = issues[0]
raise WorkflowError(
first.code,
first.message,
node_id=first.node_id,
details={"issues": [i.model_dump(by_alias=True) for i in issues]},
)
return raw if isinstance(raw, WorkflowGraph) else WorkflowGraph.model_validate(raw)
__all__ = [
"ResourceChecker",
"ValidationIssue",
"assert_publishable",
"validate_workflow_graph",
]