代码提交
This commit is contained in:
1 parent
4ac7bc21cd
commit
17cf849361
110 files changed
+17007
-518
No files matched your search
@@ -42,6 +42,7 @@ from deerflow.config.token_usage_config import TokenUsageConfig
|
||||
from deerflow.config.tool_config import ToolConfig, ToolGroupConfig
|
||||
from deerflow.config.tool_search_config import ToolSearchConfig, load_tool_search_config_from_dict
|
||||
from deerflow.config.vision_config import VisionConfig
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.config.y_log_config import YLogConfig
|
||||
|
||||
load_dotenv()
|
||||
@@ -149,6 +150,7 @@ class AppConfig(BaseModel):
|
||||
knowledge_ingest: KnowledgeIngestConfig = Field(default_factory=KnowledgeIngestConfig, description="External knowledge-ingest endpoint configuration")
|
||||
task_deeplink: TaskDeeplinkConfig = Field(default_factory=TaskDeeplinkConfig, description="任务深链(rwfx)目的树闸门与跳转地址配置(前端经 /api/public/config/task-deeplink 读取)")
|
||||
y_log: YLogConfig = Field(default_factory=YLogConfig, description="操作日志上报配置(前端经 /api/public/config/y-log 读取)")
|
||||
workflows: WorkflowConfig = Field(default_factory=WorkflowConfig, description="Workflow Studio limits / iframe origins")
|
||||
sentiment_agent: SentimentAgentConfig = Field(
|
||||
default_factory=SentimentAgentConfig,
|
||||
description="舆情分析虚拟智能体上游代理超时(地址由前端 runtime-config.js 随请求传入)",
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
"""Workflow Studio configuration (limits, egress policy, sandbox, iframe origins)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class WorkflowEmbedConfig(BaseModel):
|
||||
"""iframe ticket / origin policy."""
|
||||
|
||||
allowed_origins: list[str] = Field(
|
||||
default_factory=list,
|
||||
description="Allowed parent origins for workflow embed tickets (exact match)",
|
||||
)
|
||||
ticket_ttl_seconds: int = Field(default=60, ge=5, le=600)
|
||||
|
||||
|
||||
class WorkflowHttpPolicyConfig(BaseModel):
|
||||
"""Egress policy for the ``http`` node.
|
||||
|
||||
Default-deny on private address space. ``allowed_hosts`` is an explicit
|
||||
allowlist (exact host or ``.suffix`` match); when it is empty every public
|
||||
host is allowed but private/loopback/link-local targets are still refused,
|
||||
which is what stops SSRF against cloud metadata and internal services.
|
||||
"""
|
||||
|
||||
enabled: bool = Field(default=True)
|
||||
allowed_hosts: list[str] = Field(default_factory=list)
|
||||
blocked_hosts: list[str] = Field(default_factory=list)
|
||||
allow_private_networks: bool = Field(
|
||||
default=False,
|
||||
description="Allow loopback/private/link-local targets (only for trusted intranet deployments)",
|
||||
)
|
||||
allowed_ports: list[int] = Field(default_factory=lambda: [80, 443, 8080, 8443])
|
||||
max_response_bytes: int = Field(default=2_097_152, ge=1024)
|
||||
max_redirects: int = Field(default=0, ge=0, le=5)
|
||||
timeout_seconds: int = Field(default=60, ge=1)
|
||||
|
||||
|
||||
class WorkflowSqlPolicyConfig(BaseModel):
|
||||
"""Read-only policy for the ``sql_read`` node."""
|
||||
|
||||
enabled: bool = Field(default=True)
|
||||
max_rows: int = Field(default=1000, ge=1)
|
||||
statement_timeout_seconds: int = Field(default=30, ge=1)
|
||||
max_cell_chars: int = Field(default=4000, ge=64)
|
||||
|
||||
|
||||
class WorkflowCodeNodeConfig(BaseModel):
|
||||
"""``code`` node execution. Off by default: the local runner is a hardened
|
||||
subprocess, not a container, so operators must opt in knowingly."""
|
||||
|
||||
enabled: bool = Field(default=False)
|
||||
timeout_seconds: int = Field(default=30, ge=1, le=600)
|
||||
max_source_chars: int = Field(default=20_000, ge=64)
|
||||
max_output_chars: int = Field(default=100_000, ge=256)
|
||||
memory_limit_mb: int = Field(default=512, ge=64)
|
||||
allow_network: bool = Field(default=False)
|
||||
|
||||
|
||||
class WorkflowRetentionConfig(BaseModel):
|
||||
event_retention_days: int = Field(default=30, ge=1)
|
||||
run_retention_days: int = Field(default=90, ge=1)
|
||||
|
||||
|
||||
class WorkflowConfig(BaseModel):
|
||||
"""System-level workflow limits and policies."""
|
||||
|
||||
enabled: bool = Field(default=True, description="Master switch for workflow studio APIs")
|
||||
max_steps: int = Field(default=500, ge=1)
|
||||
max_loop_iterations: int = Field(default=20, ge=1)
|
||||
max_parallelism: int = Field(default=16, ge=1)
|
||||
run_timeout_seconds: int = Field(default=3600, ge=1)
|
||||
node_timeout_seconds: int = Field(default=600, ge=1)
|
||||
max_concurrent_runs_per_user: int = Field(default=5, ge=1)
|
||||
lease_ttl_seconds: int = Field(default=60, ge=10)
|
||||
embed: WorkflowEmbedConfig = Field(default_factory=WorkflowEmbedConfig)
|
||||
http: WorkflowHttpPolicyConfig = Field(default_factory=WorkflowHttpPolicyConfig)
|
||||
sql: WorkflowSqlPolicyConfig = Field(default_factory=WorkflowSqlPolicyConfig)
|
||||
code: WorkflowCodeNodeConfig = Field(default_factory=WorkflowCodeNodeConfig)
|
||||
retention: WorkflowRetentionConfig = Field(default_factory=WorkflowRetentionConfig)
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
"""Create workflow_definitions / workflow_versions tables.
|
||||
|
||||
Revision ID: 20260828_01
|
||||
Revises: 20260825_01
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_01"
|
||||
down_revision = "20260825_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
op.create_table(
|
||||
"workflow_definitions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
|
||||
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("draft_graph_json", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_workflow_definitions_owner_id", "workflow_definitions", ["owner_id"])
|
||||
op.create_index("ix_workflow_definitions_status", "workflow_definitions", ["status"])
|
||||
|
||||
if not inspector.has_table("workflow_versions"):
|
||||
op.create_table(
|
||||
"workflow_versions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_number", sa.Integer(), nullable=False),
|
||||
sa.Column("graph_json", sa.Text(), nullable=False),
|
||||
sa.Column("graph_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("input_schema", sa.Text(), nullable=True),
|
||||
sa.Column("output_schema", sa.Text(), nullable=True),
|
||||
sa.Column("published_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
|
||||
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
|
||||
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
|
||||
)
|
||||
op.create_index("ix_workflow_versions_workflow_id", "workflow_versions", ["workflow_id"])
|
||||
op.create_index("ix_workflow_versions_graph_hash", "workflow_versions", ["graph_hash"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("workflow_versions"):
|
||||
op.drop_table("workflow_versions")
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
op.drop_table("workflow_definitions")
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""Create workflow run / node-run / event / artifact tables.
|
||||
|
||||
Revision ID: 20260828_02
|
||||
Revises: 20260828_01
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_02"
|
||||
down_revision = "20260828_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if not inspector.has_table("workflow_runs"):
|
||||
op.create_table(
|
||||
"workflow_runs",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("workflow_version_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
|
||||
sa.Column("input_json", sa.Text(), nullable=True),
|
||||
sa.Column("output_json", sa.Text(), nullable=True),
|
||||
sa.Column("error_json", sa.Text(), nullable=True),
|
||||
sa.Column("context_json", sa.Text(), nullable=True),
|
||||
sa.Column("pending_input_json", sa.Text(), nullable=True),
|
||||
sa.Column("resume_token", sa.String(length=64), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(length=64), nullable=True),
|
||||
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("retry_of_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),
|
||||
)
|
||||
op.create_index("ix_workflow_runs_workflow_id", "workflow_runs", ["workflow_id"])
|
||||
op.create_index("ix_workflow_runs_workflow_version_id", "workflow_runs", ["workflow_version_id"])
|
||||
op.create_index("ix_workflow_runs_owner_id", "workflow_runs", ["owner_id"])
|
||||
op.create_index("ix_workflow_runs_status", "workflow_runs", ["status"])
|
||||
op.create_index("ix_workflow_runs_resume_token", "workflow_runs", ["resume_token"])
|
||||
|
||||
if not inspector.has_table("workflow_node_runs"):
|
||||
op.create_table(
|
||||
"workflow_node_runs",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("node_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("iteration", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("is_current", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("input_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("output_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("error_json", sa.Text(), nullable=True),
|
||||
sa.Column("parent_node_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint(
|
||||
"run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_workflow_node_runs_run_id", "workflow_node_runs", ["run_id"])
|
||||
op.create_index("ix_workflow_node_runs_status", "workflow_node_runs", ["status"])
|
||||
|
||||
if not inspector.has_table("workflow_run_events"):
|
||||
op.create_table(
|
||||
"workflow_run_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("seq", sa.Integer(), nullable=False),
|
||||
sa.Column("event_type", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("node_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("payload_json", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
)
|
||||
op.create_index("ix_workflow_run_events_run_id", "workflow_run_events", ["run_id"])
|
||||
op.create_index("ix_workflow_run_events_run_seq", "workflow_run_events", ["run_id", "seq"])
|
||||
|
||||
if not inspector.has_table("workflow_run_artifacts"):
|
||||
op.create_table(
|
||||
"workflow_run_artifacts",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("name", sa.String(length=512), nullable=False),
|
||||
sa.Column("mime_type", sa.String(length=191), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=True),
|
||||
sa.Column("preview", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_workflow_run_artifacts_run_id", "workflow_run_artifacts", ["run_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
for table in (
|
||||
"workflow_run_artifacts",
|
||||
"workflow_run_events",
|
||||
"workflow_node_runs",
|
||||
"workflow_runs",
|
||||
):
|
||||
if inspector.has_table(table):
|
||||
op.drop_table(table)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Create workflow_data_sources table.
|
||||
|
||||
Revision ID: 20260828_03
|
||||
Revises: 20260828_02
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_03"
|
||||
down_revision = "20260828_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
op.create_table(
|
||||
"workflow_data_sources",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("name", sa.String(length=191), nullable=False),
|
||||
sa.Column("description", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False, server_default="sql"),
|
||||
sa.Column("driver", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
|
||||
sa.Column("secret_encrypted", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("masked_target", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("max_rows", sa.Integer(), nullable=False, server_default="1000"),
|
||||
sa.Column("allowed_tables", sa.Text(), nullable=True),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("created_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("name", name="uq_workflow_data_sources_name"),
|
||||
)
|
||||
op.create_index("ix_workflow_data_sources_owner_id", "workflow_data_sources", ["owner_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if sa.inspect(bind).has_table("workflow_data_sources"):
|
||||
op.drop_table("workflow_data_sources")
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Add workflow counter columns: version allocator + event seq counter.
|
||||
|
||||
Revision ID: 20260828_04
|
||||
Revises: 20260828_03
|
||||
Create Date: 2026-08-28
|
||||
|
||||
Functional-robustness remediation (audit items P1-02 / P1-03):
|
||||
|
||||
* ``workflow_definitions.next_version_number`` — publish allocates the version
|
||||
number by CAS-bumping this counter instead of ``MAX(version_number) + 1``,
|
||||
so two concurrent publishes get adjacent distinct numbers instead of a
|
||||
unique-constraint 500.
|
||||
* ``workflow_runs.next_event_seq`` — the event log draws each event's ``seq``
|
||||
from this counter in the insert's own transaction instead of ``MAX(seq)+1``.
|
||||
|
||||
Both columns carry a ``server_default`` so existing databases get them via the
|
||||
startup ``_ensure_orm_columns_sync`` sweep even without running Alembic; rows
|
||||
keep the default (1 / 0) and the first write self-heals from the existing
|
||||
maximum on unique-conflict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_04"
|
||||
down_revision = "20260828_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
|
||||
if "next_version_number" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("next_version_number", sa.Integer(), nullable=False, server_default="1"),
|
||||
)
|
||||
|
||||
if inspector.has_table("workflow_runs"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
|
||||
if "next_event_seq" not in existing:
|
||||
op.add_column(
|
||||
"workflow_runs",
|
||||
sa.Column("next_event_seq", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
|
||||
if "next_version_number" in existing:
|
||||
op.drop_column("workflow_definitions", "next_version_number")
|
||||
|
||||
if inspector.has_table("workflow_runs"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
|
||||
if "next_event_seq" in existing:
|
||||
op.drop_column("workflow_runs", "next_event_seq")
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Add non-secret HTTP metadata to workflow data sources.
|
||||
|
||||
Revision ID: 20260829_01
|
||||
Revises: 20260828_04
|
||||
Create Date: 2026-08-29
|
||||
|
||||
HTTP request headers remain in the encrypted secret blob. The endpoint and
|
||||
method allowlist are intentionally separate non-secret fields so the canvas
|
||||
can show a reusable interface resource without exposing credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260829_01"
|
||||
down_revision = "20260828_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
|
||||
if "base_url" not in existing:
|
||||
op.add_column(
|
||||
"workflow_data_sources",
|
||||
sa.Column("base_url", sa.String(length=2048), nullable=False, server_default=""),
|
||||
)
|
||||
if "allowed_methods" not in existing:
|
||||
op.add_column(
|
||||
"workflow_data_sources",
|
||||
sa.Column("allowed_methods", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
|
||||
if "allowed_methods" in existing:
|
||||
op.drop_column("workflow_data_sources", "allowed_methods")
|
||||
if "base_url" in existing:
|
||||
op.drop_column("workflow_data_sources", "base_url")
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""Add Coze canvas draft column to workflow_definitions.
|
||||
|
||||
Revision ID: 20260829_02
|
||||
Revises: 20260829_01
|
||||
Create Date: 2026-08-29
|
||||
|
||||
The canvas document (positions, Coze form internals) is stored separately from
|
||||
the executable WorkflowGraph so neither representation can impersonate the
|
||||
other. ``published_canvas_schema_hash`` records which canvas the current
|
||||
published version was compiled from, for audit only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260829_02"
|
||||
down_revision = "20260829_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
|
||||
if "draft_canvas_schema_json" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
|
||||
)
|
||||
if "published_canvas_schema_hash" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("published_canvas_schema_hash", sa.String(length=64), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
|
||||
if "published_canvas_schema_hash" in existing:
|
||||
op.drop_column("workflow_definitions", "published_canvas_schema_hash")
|
||||
if "draft_canvas_schema_json" in existing:
|
||||
op.drop_column("workflow_definitions", "draft_canvas_schema_json")
|
||||
+15
@@ -50,6 +50,14 @@ from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
@@ -128,6 +136,13 @@ __all__ = [
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"LightAppRow",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDefinitionRow",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunEventRow",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowVersionRow",
|
||||
"LlmCallMetricRow",
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Workflow data-source persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.persistence.workflow_data_sources.store import (
|
||||
MemoryWorkflowDataSourceStore,
|
||||
SqlWorkflowDataSourceStore,
|
||||
WorkflowDataSourceStore,
|
||||
make_workflow_data_source_store,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowDataSourceStore",
|
||||
"SqlWorkflowDataSourceStore",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDataSourceStore",
|
||||
"make_workflow_data_source_store",
|
||||
]
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""ORM model for workflow SQL data sources and HTTP credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowDataSourceRow(Base):
|
||||
"""A read-only database connection usable by ``sql_read`` nodes.
|
||||
|
||||
``secret_ciphertext`` holds the DSN encrypted with the workflow secret key;
|
||||
it is never returned by any API. ``owner_id`` NULL means the data source is
|
||||
shared (admin-managed)."""
|
||||
|
||||
__tablename__ = "workflow_data_sources"
|
||||
__table_args__ = (UniqueConstraint("name", name="uq_workflow_data_sources_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False)
|
||||
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="sql")
|
||||
driver: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
base_url: Mapped[str] = mapped_column(
|
||||
String(2048),
|
||||
nullable=False,
|
||||
default="",
|
||||
server_default=sa_text("''"),
|
||||
)
|
||||
# Nullable so the startup column synchronizer can safely patch existing databases.
|
||||
allowed_methods: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
secret_ciphertext: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
secret_encrypted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
masked_target: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
max_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=1000, server_default=sa_text("1000"))
|
||||
allowed_tables: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
"""Workflow data-source store (SQL + in-memory).
|
||||
|
||||
Secrets are encrypted before they reach the database and only ever decrypted
|
||||
inside :meth:`resolve_dsn`, which the run executor calls. No read path returns
|
||||
the ciphertext.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.workflows.security.secret_box import decrypt_secret, encrypt_secret, mask_dsn
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _public_dict(row: WorkflowDataSourceRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"description": row.description or "",
|
||||
"kind": row.kind,
|
||||
"driver": row.driver or "",
|
||||
"base_url": row.base_url or "",
|
||||
"allowed_methods": list(row.allowed_methods or []),
|
||||
"masked_target": row.masked_target or "",
|
||||
"secret_encrypted": bool(row.secret_encrypted),
|
||||
"max_rows": row.max_rows,
|
||||
"allowed_tables": list(row.allowed_tables or []),
|
||||
"enabled": bool(row.enabled),
|
||||
"owner_id": row.owner_id,
|
||||
"created_by": row.created_by,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class WorkflowDataSourceStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
"""Decrypted connection info for the executor. Never expose via HTTP."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SqlWorkflowDataSourceStore(WorkflowDataSourceStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowDataSourceRow).order_by(WorkflowDataSourceRow.name.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
visible = []
|
||||
for row in rows:
|
||||
if row.owner_id is None and include_shared:
|
||||
visible.append(row)
|
||||
elif owner_id is not None and row.owner_id == owner_id:
|
||||
visible.append(row)
|
||||
return [_public_dict(row) for row in visible]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
return _public_dict(row) if row else None
|
||||
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
dsn = str(data.get("dsn") or "")
|
||||
kind = str(data.get("kind") or "sql")
|
||||
base_url = str(data.get("base_url") or "")
|
||||
row = WorkflowDataSourceRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
name=str(data["name"])[:191],
|
||||
description=str(data.get("description") or "")[:512],
|
||||
kind=kind,
|
||||
driver=str(
|
||||
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
|
||||
)[:64],
|
||||
base_url=base_url[:2048],
|
||||
allowed_methods=list(data.get("allowed_methods") or []),
|
||||
secret_ciphertext=encrypt_secret(dsn) if dsn else "",
|
||||
secret_encrypted=bool(dsn),
|
||||
masked_target=(base_url if kind == "http" else mask_dsn(dsn))[:512],
|
||||
max_rows=int(data.get("max_rows") or 1000),
|
||||
allowed_tables=list(data.get("allowed_tables") or []),
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
owner_id=data.get("owner_id"),
|
||||
created_by=data.get("created_by"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = _public_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None:
|
||||
return None
|
||||
if data.get("name"):
|
||||
row.name = str(data["name"])[:191]
|
||||
if "description" in data:
|
||||
row.description = str(data.get("description") or "")[:512]
|
||||
if "base_url" in data:
|
||||
row.base_url = str(data.get("base_url") or "")[:2048]
|
||||
if row.kind == "http":
|
||||
row.masked_target = row.base_url[:512]
|
||||
if "allowed_methods" in data:
|
||||
row.allowed_methods = list(data.get("allowed_methods") or [])
|
||||
if data.get("dsn"):
|
||||
dsn = str(data["dsn"])
|
||||
row.secret_ciphertext = encrypt_secret(dsn)
|
||||
row.secret_encrypted = True
|
||||
if row.kind == "http":
|
||||
row.driver = "HTTP API"
|
||||
else:
|
||||
row.masked_target = mask_dsn(dsn)[:512]
|
||||
row.driver = dsn.split("://", 1)[0][:64]
|
||||
if data.get("max_rows") is not None:
|
||||
row.max_rows = int(data["max_rows"])
|
||||
if "allowed_tables" in data:
|
||||
row.allowed_tables = list(data.get("allowed_tables") or [])
|
||||
if "enabled" in data:
|
||||
row.enabled = bool(data["enabled"])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
out = _public_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None or not row.enabled or not row.secret_ciphertext:
|
||||
return None
|
||||
return {
|
||||
"id": row.id,
|
||||
"dsn": decrypt_secret(row.secret_ciphertext),
|
||||
"kind": row.kind,
|
||||
"base_url": row.base_url or "",
|
||||
"allowed_methods": list(row.allowed_methods or []),
|
||||
"max_rows": row.max_rows,
|
||||
"allowed_tables": list(row.allowed_tables or []),
|
||||
"owner_id": row.owner_id,
|
||||
}
|
||||
|
||||
|
||||
class MemoryWorkflowDataSourceStore(WorkflowDataSourceStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._secrets: dict[str, str] = {}
|
||||
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for row in self._rows.values():
|
||||
if row["owner_id"] is None and include_shared:
|
||||
out.append(row)
|
||||
elif owner_id is not None and row["owner_id"] == owner_id:
|
||||
out.append(row)
|
||||
return [copy.deepcopy(r) for r in sorted(out, key=lambda r: r["name"])]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
return copy.deepcopy(row) if row else None
|
||||
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source_id = str(data.get("id") or uuid.uuid4())
|
||||
dsn = str(data.get("dsn") or "")
|
||||
kind = str(data.get("kind") or "sql")
|
||||
base_url = str(data.get("base_url") or "")
|
||||
now = _iso(datetime.now(UTC))
|
||||
row = {
|
||||
"id": source_id,
|
||||
"name": str(data["name"]),
|
||||
"description": str(data.get("description") or ""),
|
||||
"kind": kind,
|
||||
"driver": str(
|
||||
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
|
||||
),
|
||||
"base_url": base_url,
|
||||
"allowed_methods": list(data.get("allowed_methods") or []),
|
||||
"masked_target": base_url if kind == "http" else mask_dsn(dsn),
|
||||
"secret_encrypted": bool(dsn),
|
||||
"max_rows": int(data.get("max_rows") or 1000),
|
||||
"allowed_tables": list(data.get("allowed_tables") or []),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"owner_id": data.get("owner_id"),
|
||||
"created_by": data.get("created_by"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[source_id] = row
|
||||
self._secrets[source_id] = dsn
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in (
|
||||
"name",
|
||||
"description",
|
||||
"base_url",
|
||||
"allowed_methods",
|
||||
"max_rows",
|
||||
"allowed_tables",
|
||||
"enabled",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
row[key] = data[key]
|
||||
if "base_url" in data and row["kind"] == "http":
|
||||
row["masked_target"] = str(row["base_url"])
|
||||
if data.get("dsn"):
|
||||
self._secrets[source_id] = str(data["dsn"])
|
||||
if row["kind"] == "http":
|
||||
row["driver"] = "HTTP API"
|
||||
else:
|
||||
row["masked_target"] = mask_dsn(str(data["dsn"]))
|
||||
row["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
self._secrets.pop(source_id, None)
|
||||
return self._rows.pop(source_id, None) is not None
|
||||
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
if row is None or not row["enabled"]:
|
||||
return None
|
||||
return {
|
||||
"id": source_id,
|
||||
"dsn": self._secrets.get(source_id, ""),
|
||||
"kind": row["kind"],
|
||||
"base_url": row.get("base_url", ""),
|
||||
"allowed_methods": list(row.get("allowed_methods") or []),
|
||||
"max_rows": row["max_rows"],
|
||||
"allowed_tables": row["allowed_tables"],
|
||||
"owner_id": row["owner_id"],
|
||||
}
|
||||
|
||||
|
||||
def make_workflow_data_source_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowDataSourceStore()
|
||||
return SqlWorkflowDataSourceStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowDataSourceStore",
|
||||
"SqlWorkflowDataSourceStore",
|
||||
"WorkflowDataSourceStore",
|
||||
"make_workflow_data_source_store",
|
||||
]
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Workflow run event-log persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowEventStore",
|
||||
"SqlWorkflowEventStore",
|
||||
"WorkflowEventStore",
|
||||
"WorkflowRunEventRow",
|
||||
"make_workflow_event_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_event_store(session_factory, run_store=None):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowEventStore(run_store=run_store)
|
||||
return SqlWorkflowEventStore(session_factory)
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Abstract workflow run event store (durable, replayable)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowEventStore(ABC):
|
||||
@abstractmethod
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, assigning the next per-run ``seq``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
"""Retention purge by event age; returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
"""Delete every event belonging to the given runs (used when the runs
|
||||
themselves are retained away). Returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""In-memory workflow event log (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
|
||||
|
||||
class MemoryWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
|
||||
self._events: dict[str, list[dict[str, Any]]] = {}
|
||||
self._max_per_run = max_per_run
|
||||
self._run_store = run_store
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
rows = self._events.setdefault(run_id, [])
|
||||
event = {
|
||||
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
|
||||
"run_id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_id": version_id,
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"node_run_id": node_run_id,
|
||||
"payload": payload or {},
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
rows.append(event)
|
||||
if len(rows) > self._max_per_run:
|
||||
del rows[: len(rows) - self._max_per_run]
|
||||
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
|
||||
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
|
||||
# where the counter bump and the insert share one transaction).
|
||||
try:
|
||||
await self._run_store.note_event_seq(run_id, event["seq"])
|
||||
except Exception: # noqa: BLE001 - counter sync is best effort
|
||||
pass
|
||||
return copy.deepcopy(event)
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
|
||||
return [copy.deepcopy(r) for r in rows[:limit]]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
rows = self._events.get(run_id, [])
|
||||
return int(rows[-1]["seq"]) if rows else 0
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id, rows in list(self._events.items()):
|
||||
kept: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
created = _parse_dt(row.get("created_at"))
|
||||
if created is not None and created < cutoff:
|
||||
deleted += 1
|
||||
else:
|
||||
kept.append(row)
|
||||
if kept:
|
||||
self._events[run_id] = kept
|
||||
else:
|
||||
del self._events[run_id]
|
||||
return deleted
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id in run_ids:
|
||||
rows = self._events.pop(run_id, None)
|
||||
if rows:
|
||||
deleted += len(rows)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""ORM model for the durable workflow run event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunEventRow(Base):
|
||||
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
|
||||
|
||||
__tablename__ = "workflow_run_events"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
|
||||
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
"""SQLAlchemy-backed workflow event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
|
||||
|
||||
_MAX_PAYLOAD_CHARS = 64_000
|
||||
_APPEND_ATTEMPTS = 5
|
||||
|
||||
|
||||
def _dump_payload(payload: dict[str, Any] | None) -> str:
|
||||
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
if len(text) > _MAX_PAYLOAD_CHARS:
|
||||
return json.dumps(
|
||||
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(row.payload_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
return {
|
||||
"seq": row.seq,
|
||||
"run_id": row.run_id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_id": row.version_id,
|
||||
"event_type": row.event_type,
|
||||
"node_id": row.node_id,
|
||||
"node_run_id": row.node_run_id,
|
||||
"payload": payload if isinstance(payload, dict) else {},
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
max_retries: int = _APPEND_ATTEMPTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, taking its ``seq`` from the run's own counter.
|
||||
|
||||
``MAX(seq) + 1`` let two appenders draw the same number and burn a
|
||||
unique-constraint retry; here the counter increment and the insert share
|
||||
one transaction, and the row write-lock held by the UPDATE serialises
|
||||
concurrent appenders until commit — so seqs are strictly increasing,
|
||||
unique and gap-free per run, in commit order.
|
||||
"""
|
||||
text = _dump_payload(payload)
|
||||
for _ in range(max_retries):
|
||||
async with self._sf() as session:
|
||||
seq = await self._take_seq(session, run_id)
|
||||
row = WorkflowRunEventRow(
|
||||
run_id=run_id,
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
seq=seq,
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
payload_json=text,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
# Only reachable for legacy rows whose counter was backfilled
|
||||
# to 0 while events already existed. Re-sync the counter to
|
||||
# MAX(seq) in its own committed transaction, then retry.
|
||||
await session.rollback()
|
||||
await self._resync_counter(run_id)
|
||||
continue
|
||||
out = event_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
|
||||
|
||||
async def _take_seq(self, session: Any, run_id: str) -> int:
|
||||
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
|
||||
missing (event store used standalone in tests)."""
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
|
||||
)
|
||||
if result.rowcount:
|
||||
# Our own uncommitted increment; the row lock guards it until commit.
|
||||
current = (
|
||||
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
|
||||
).scalar()
|
||||
return int(current or 1)
|
||||
current = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar()
|
||||
return int(current or 0) + 1
|
||||
|
||||
async def _resync_counter(self, run_id: str) -> None:
|
||||
"""Point a legacy/stale run counter back at MAX(seq)."""
|
||||
async with self._sf() as session:
|
||||
max_seq = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar() or 0
|
||||
await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
|
||||
.values(next_event_seq=int(max_seq))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [event_to_dict(r) for r in rows]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
|
||||
return int(value or 0)
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""Workflow run / node-run / artifact persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunConflictError,
|
||||
WorkflowRunNotFoundError,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
|
||||
|
||||
__all__ = [
|
||||
"CANCELLABLE_RUN_STATUSES",
|
||||
"MemoryWorkflowRunStore",
|
||||
"SqlWorkflowRunStore",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunConflictError",
|
||||
"WorkflowRunNotFoundError",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowRunStore",
|
||||
"make_workflow_run_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_run_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowRunStore()
|
||||
return SqlWorkflowRunStore(session_factory)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Abstract workflow run / node-run / artifact store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
|
||||
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
|
||||
|
||||
|
||||
class WorkflowRunNotFoundError(Exception):
|
||||
def __init__(self, run_id: str) -> None:
|
||||
super().__init__(f"workflow run not found: {run_id}")
|
||||
self.run_id = run_id
|
||||
|
||||
|
||||
class WorkflowRunConflictError(Exception):
|
||||
"""Raised when a state transition is not legal for the current status."""
|
||||
|
||||
def __init__(self, run_id: str, status: str) -> None:
|
||||
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
|
||||
self.run_id = run_id
|
||||
self.status = status
|
||||
|
||||
|
||||
class WorkflowRunStore(ABC):
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Create a run. Returns ``(run, created)``; an existing
|
||||
``idempotency_key`` yields the stored run with ``created=False``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""CAS update. When ``lease_owner`` is given the row must still be leased
|
||||
by that owner. Terminal statuses clear the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pause atomically: status + token + pending descriptor (+ optional run
|
||||
context) in one conditional update, so a concurrent resume can never be
|
||||
overwritten by a follow-up context write."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
|
||||
``queued``. Returns None when the token is wrong/already used."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomic human-input resume in a single conditional update.
|
||||
|
||||
Validates the token, stores ``payload`` as ``context.resume_payload``
|
||||
(replacing the whole run context with ``context`` when given), clears the
|
||||
token/pending descriptor and flips the run to ``queued`` — all under
|
||||
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
|
||||
the resumed run on success, ``None`` when the token is wrong, already
|
||||
used, or the run is no longer awaiting input (caller maps that to 409).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
"""Mark node runs stale (used when a loop re-enters its body)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch one artifact row scoped to its run (authorization key)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
"""Delete finished runs older than the cutoff, with their node runs and
|
||||
artifact rows. Only terminal runs are eligible. Returns deleted run ids
|
||||
(bounded per call so a sweeper can loop)."""
|
||||
raise NotImplementedError
|
||||
+388
@@ -0,0 +1,388 @@
|
||||
"""In-memory workflow run store (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
class MemoryWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self) -> None:
|
||||
self._runs: dict[str, dict[str, Any]] = {}
|
||||
self._node_runs: dict[str, list[dict[str, Any]]] = {}
|
||||
self._artifacts: dict[str, list[dict[str, Any]]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
async with self._lock:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
for run in self._runs.values():
|
||||
if run.get("idempotency_key") == str(key):
|
||||
return copy.deepcopy(run), False
|
||||
now = datetime.now(UTC)
|
||||
run_id = str(data.get("id") or uuid.uuid4())
|
||||
run = {
|
||||
"id": run_id,
|
||||
"workflow_id": str(data["workflow_id"]),
|
||||
"workflow_version_id": str(data["workflow_version_id"]),
|
||||
"status": str(data.get("status") or "queued"),
|
||||
"owner_id": str(data["owner_id"]),
|
||||
"idempotency_key": str(key) if key else None,
|
||||
"input": data.get("input") or {},
|
||||
"output": None,
|
||||
"error": None,
|
||||
"context": data.get("context") or {},
|
||||
"pending_input": None,
|
||||
"resume_token": None,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"heartbeat_at": None,
|
||||
"attempt": 0,
|
||||
"max_attempts": int(data.get("max_attempts") or 3),
|
||||
"version": 0,
|
||||
"retry_of_run_id": data.get("retry_of_run_id"),
|
||||
"next_event_seq": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"_created": now,
|
||||
}
|
||||
self._runs[run_id] = run
|
||||
return copy.deepcopy(run), True
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
return copy.deepcopy(run) if run else None
|
||||
|
||||
async def note_event_seq(self, run_id: str, seq: int) -> None:
|
||||
"""Mirror the SQL counter semantics for no-DB mode: the memory event
|
||||
store reports each handed-out seq so run views can expose lastSeq."""
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
|
||||
run["next_event_seq"] = int(seq)
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
|
||||
if workflow_id:
|
||||
rows = [r for r in rows if r["workflow_id"] == workflow_id]
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
out: list[str] = []
|
||||
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
|
||||
if run["status"] == "queued":
|
||||
out.append(run["id"])
|
||||
elif run["status"] == "running" and run["lease_until"]:
|
||||
if datetime.fromisoformat(run["lease_until"]) < now:
|
||||
out.append(run["id"])
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
|
||||
if not claimable:
|
||||
return None
|
||||
run.update(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=_iso(lease_until),
|
||||
heartbeat_at=_iso(now),
|
||||
attempt=run["attempt"] + 1,
|
||||
version=run["version"] + 1,
|
||||
started_at=run["started_at"] or _iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
|
||||
return False
|
||||
run["lease_until"] = _iso(lease_until)
|
||||
run["heartbeat_at"] = _iso(datetime.now(UTC))
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
mapping = {
|
||||
"output_json": "output",
|
||||
"error_json": "error",
|
||||
"context_json": "context",
|
||||
"pending_input_json": "pending_input",
|
||||
}
|
||||
for key, value in fields.items():
|
||||
run[mapping.get(key, key)] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
run["lease_owner"] = None
|
||||
run["lease_until"] = None
|
||||
run["resume_token"] = None
|
||||
run["pending_input"] = None
|
||||
run["finished_at"] = run.get("finished_at") or _iso(now)
|
||||
for key in ("started_at", "finished_at"):
|
||||
if isinstance(run.get(key), datetime):
|
||||
run[key] = _iso(run[key])
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(now)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if run["status"] in ("cancel_requested", "cancelled"):
|
||||
return copy.deepcopy(run)
|
||||
if run["status"] not in CANCELLABLE_RUN_STATUSES:
|
||||
return None
|
||||
run["status"] = "cancel_requested"
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "cancel_requested":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
finished_at=_iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "running":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="awaiting_input",
|
||||
pending_input=copy.deepcopy(pending_input),
|
||||
resume_token=resume_token,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
if context is not None:
|
||||
run["context"] = copy.deepcopy(context)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
run.update(status="queued", resume_token=None, pending_input=None)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
base_context = context if context is not None else run.get("context") or {}
|
||||
run.update(
|
||||
context={**base_context, "resume_payload": copy.deepcopy(payload)},
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
rows = self._node_runs.setdefault(run_id, [])
|
||||
existing = next(
|
||||
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
|
||||
None,
|
||||
)
|
||||
if existing is None:
|
||||
existing = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": run_id,
|
||||
"node_id": node_id,
|
||||
"node_type": str(data.get("node_type") or "unknown"),
|
||||
"status": "queued",
|
||||
"attempt": attempt,
|
||||
"iteration": iteration,
|
||||
"is_current": True,
|
||||
"input": None,
|
||||
"output": None,
|
||||
"error": None,
|
||||
"parent_node_run_id": None,
|
||||
"duration_ms": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
rows.append(existing)
|
||||
for key in (
|
||||
"status",
|
||||
"node_type",
|
||||
"input",
|
||||
"output",
|
||||
"error",
|
||||
"parent_node_run_id",
|
||||
"duration_ms",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
|
||||
if "is_current" in data:
|
||||
existing["is_current"] = bool(data["is_current"])
|
||||
return copy.deepcopy(existing)
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
rows = self._node_runs.get(run_id, [])
|
||||
if current_only:
|
||||
rows = [r for r in rows if r["is_current"]]
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
count = 0
|
||||
for row in self._node_runs.get(run_id, []):
|
||||
if row["node_id"] in node_ids and row["is_current"]:
|
||||
row["is_current"] = False
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": str(data["run_id"]),
|
||||
"node_id": str(data.get("node_id") or ""),
|
||||
"name": str(data.get("name") or ""),
|
||||
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
|
||||
"path": str(data.get("path") or ""),
|
||||
"size_bytes": data.get("size_bytes"),
|
||||
"preview": data.get("preview"),
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
self._artifacts.setdefault(row["run_id"], []).append(row)
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
for row in self._artifacts.get(run_id, []):
|
||||
if row["id"] == artifact_id:
|
||||
return copy.deepcopy(row)
|
||||
return None
|
||||
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted: list[str] = []
|
||||
for run_id, run in list(self._runs.items()):
|
||||
if run["status"] not in TERMINAL_RUN_STATUSES:
|
||||
continue
|
||||
finished = _parse_dt(run.get("finished_at"))
|
||||
if finished is None or finished >= cutoff:
|
||||
continue
|
||||
deleted.append(run_id)
|
||||
for run_id in deleted:
|
||||
self._runs.pop(run_id, None)
|
||||
self._node_runs.pop(run_id, None)
|
||||
self._artifacts.pop(run_id, None)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""ORM models for workflow runs and per-node runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunRow(Base):
|
||||
"""One workflow execution bound to an immutable published version."""
|
||||
|
||||
__tablename__ = "workflow_runs"
|
||||
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
# Non-sensitive run context (never credentials).
|
||||
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
# Pending human-input descriptor when status == awaiting_input.
|
||||
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
# Last seq handed out by the event log for this run (0 = none yet). The
|
||||
# event store increments it in the same transaction that inserts the event
|
||||
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
|
||||
# first append self-heals from MAX(seq) on conflict.
|
||||
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowNodeRunRow(Base):
|
||||
"""Per-node execution record (also the resume checkpoint for a paused run)."""
|
||||
|
||||
__tablename__ = "workflow_node_runs"
|
||||
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# queued | running | completed | failed | skipped | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
|
||||
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class WorkflowRunArtifactRow(Base):
|
||||
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
|
||||
|
||||
__tablename__ = "workflow_run_artifacts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
|
||||
_RUN_UPDATABLE = {
|
||||
"status",
|
||||
"output_json",
|
||||
"error_json",
|
||||
"context_json",
|
||||
"pending_input_json",
|
||||
"resume_token",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
}
|
||||
|
||||
# Retention deletes in bounded batches so a long-neglected table cannot produce
|
||||
# one enormous transaction.
|
||||
_PURGE_BATCH = 500
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _load_text_json(raw: str | None) -> Any:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _dump_text_json(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"workflow_version_id": row.workflow_version_id,
|
||||
"status": row.status,
|
||||
"owner_id": row.owner_id,
|
||||
"idempotency_key": row.idempotency_key,
|
||||
"input": row.input_json or {},
|
||||
"output": row.output_json,
|
||||
"error": row.error_json,
|
||||
"context": row.context_json or {},
|
||||
"pending_input": row.pending_input_json,
|
||||
"resume_token": row.resume_token,
|
||||
"lease_owner": row.lease_owner,
|
||||
"lease_until": _iso(row.lease_until),
|
||||
"heartbeat_at": _iso(row.heartbeat_at),
|
||||
"attempt": row.attempt,
|
||||
"max_attempts": row.max_attempts,
|
||||
"version": row.version,
|
||||
"retry_of_run_id": row.retry_of_run_id,
|
||||
# Last seq handed out by the event log (0 = no events yet).
|
||||
"next_event_seq": int(row.next_event_seq or 0),
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"node_type": row.node_type,
|
||||
"status": row.status,
|
||||
"attempt": row.attempt,
|
||||
"iteration": row.iteration,
|
||||
"is_current": bool(row.is_current),
|
||||
"input": _load_text_json(row.input_snapshot),
|
||||
"output": _load_text_json(row.output_snapshot),
|
||||
"error": row.error_json,
|
||||
"parent_node_run_id": row.parent_node_run_id,
|
||||
"duration_ms": row.duration_ms,
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"name": row.name,
|
||||
"mime_type": row.mime_type,
|
||||
"path": row.path,
|
||||
"size_bytes": row.size_bytes,
|
||||
"preview": row.preview,
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
row = WorkflowRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
workflow_id=str(data["workflow_id"]),
|
||||
workflow_version_id=str(data["workflow_version_id"]),
|
||||
status=str(data.get("status") or "queued"),
|
||||
owner_id=str(data["owner_id"]),
|
||||
idempotency_key=str(key) if key else None,
|
||||
input_json=data.get("input") or {},
|
||||
context_json=data.get("context") or {},
|
||||
max_attempts=int(data.get("max_attempts") or 3),
|
||||
retry_of_run_id=data.get("retry_of_run_id"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
raise
|
||||
out = run_to_dict(row)
|
||||
await session.commit()
|
||||
return out, True
|
||||
|
||||
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunRow)
|
||||
if workflow_id:
|
||||
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowRunRow.status == status)
|
||||
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [run_to_dict(r) for r in rows]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
stmt = (
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
)
|
||||
)
|
||||
.order_by(WorkflowRunRow.created_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
heartbeat_at=now,
|
||||
attempt=WorkflowRunRow.attempt + 1,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
started_at=func.coalesce(WorkflowRunRow.started_at, now),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.lease_owner == lease_owner,
|
||||
WorkflowRunRow.status == "running",
|
||||
)
|
||||
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return False
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
|
||||
for key, value in fields.items():
|
||||
if key in _RUN_UPDATABLE:
|
||||
values[key] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
values["lease_owner"] = None
|
||||
values["lease_until"] = None
|
||||
values["resume_token"] = None
|
||||
values["pending_input_json"] = None
|
||||
values.setdefault("finished_at", now)
|
||||
where = [WorkflowRunRow.id == run_id]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
|
||||
)
|
||||
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
finished_at=now,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
values: dict[str, Any] = {
|
||||
"status": "awaiting_input",
|
||||
"pending_input_json": pending_input,
|
||||
"resume_token": resume_token,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"version": WorkflowRunRow.version + 1,
|
||||
"updated_at": now,
|
||||
}
|
||||
if context is not None:
|
||||
values["context_json"] = context
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
if context is None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
if row is None:
|
||||
return None
|
||||
context = dict(row.context_json or {})
|
||||
new_context = {**context, "resume_payload": payload}
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
context_json=new_context,
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(WorkflowNodeRunRow).where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id == node_id,
|
||||
WorkflowNodeRunRow.attempt == attempt,
|
||||
WorkflowNodeRunRow.iteration == iteration,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = WorkflowNodeRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
node_id=node_id,
|
||||
node_type=str(data.get("node_type") or "unknown"),
|
||||
attempt=attempt,
|
||||
iteration=iteration,
|
||||
)
|
||||
session.add(row)
|
||||
row.status = str(data.get("status") or row.status)
|
||||
row.node_type = str(data.get("node_type") or row.node_type)
|
||||
row.is_current = bool(data.get("is_current", True))
|
||||
if "input" in data:
|
||||
row.input_snapshot = _dump_text_json(data.get("input"))
|
||||
if "output" in data:
|
||||
row.output_snapshot = _dump_text_json(data.get("output"))
|
||||
if "error" in data:
|
||||
row.error_json = data.get("error")
|
||||
if data.get("parent_node_run_id"):
|
||||
row.parent_node_run_id = str(data["parent_node_run_id"])
|
||||
if data.get("duration_ms") is not None:
|
||||
row.duration_ms = int(data["duration_ms"])
|
||||
if data.get("started_at") is not None:
|
||||
row.started_at = data["started_at"]
|
||||
if data.get("finished_at") is not None:
|
||||
row.finished_at = data["finished_at"]
|
||||
await session.flush()
|
||||
out = node_run_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
|
||||
if current_only:
|
||||
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
|
||||
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [node_run_to_dict(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
if not node_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowNodeRunRow)
|
||||
.where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id.in_(node_ids),
|
||||
WorkflowNodeRunRow.is_current.is_(True),
|
||||
)
|
||||
.values(is_current=False)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = WorkflowRunArtifactRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=str(data["run_id"]),
|
||||
node_id=str(data.get("node_id") or ""),
|
||||
name=str(data.get("name") or "")[:512],
|
||||
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
|
||||
path=str(data.get("path") or ""),
|
||||
size_bytes=data.get("size_bytes"),
|
||||
preview=data.get("preview"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = artifact_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [artifact_to_dict(r) for r in rows]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(WorkflowRunArtifactRow).where(
|
||||
WorkflowRunArtifactRow.id == artifact_id,
|
||||
WorkflowRunArtifactRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return artifact_to_dict(row) if row is not None else None
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
run_ids = (
|
||||
(
|
||||
await session.execute(
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
|
||||
WorkflowRunRow.finished_at.is_not(None),
|
||||
WorkflowRunRow.finished_at < cutoff,
|
||||
)
|
||||
.limit(_PURGE_BATCH)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not run_ids:
|
||||
return []
|
||||
ids = list(run_ids)
|
||||
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
|
||||
await session.commit()
|
||||
return ids
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""Workflow definition / version persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
WorkflowVersionNotFoundError,
|
||||
)
|
||||
from deerflow.persistence.workflows.memory import MemoryWorkflowStore
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
from deerflow.persistence.workflows.sql import WorkflowRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowStore",
|
||||
"WorkflowDefinitionRow",
|
||||
"WorkflowDraftConflictError",
|
||||
"WorkflowNotFoundError",
|
||||
"WorkflowRepository",
|
||||
"WorkflowStore",
|
||||
"WorkflowVersionNotFoundError",
|
||||
"WorkflowVersionRow",
|
||||
"make_workflow_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowStore()
|
||||
return WorkflowRepository(session_factory)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Abstract workflow definition / version store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowDraftConflictError(Exception):
|
||||
"""Raised when ``expected_revision`` does not match the stored draft revision."""
|
||||
|
||||
def __init__(self, workflow_id: str, current_revision: int) -> None:
|
||||
super().__init__(f"draft conflict for {workflow_id}: current={current_revision}")
|
||||
self.workflow_id = workflow_id
|
||||
self.current_revision = current_revision
|
||||
|
||||
|
||||
class WorkflowNotFoundError(Exception):
|
||||
def __init__(self, workflow_id: str) -> None:
|
||||
super().__init__(f"workflow not found: {workflow_id}")
|
||||
self.workflow_id = workflow_id
|
||||
|
||||
|
||||
class WorkflowVersionNotFoundError(Exception):
|
||||
def __init__(self, version_id: str) -> None:
|
||||
super().__init__(f"workflow version not found: {version_id}")
|
||||
self.version_id = version_id
|
||||
|
||||
|
||||
class WorkflowStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""CAS update of ``draft_graph_json``. Raises ``WorkflowDraftConflictError``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically CAS-update BOTH the execution graph and the canvas document.
|
||||
|
||||
A single conditional UPDATE bumps ``draft_revision`` and writes
|
||||
``draft_graph_json`` + ``draft_canvas_schema_json`` together, so the two
|
||||
representations can never diverge across revisions. Raises
|
||||
``WorkflowDraftConflictError`` on a stale ``expected_revision``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Lightweight list for subworkflow resource catalog."""
|
||||
raise NotImplementedError
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"""In-memory workflow store for tests / no-DB mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryWorkflowStore(WorkflowStore):
|
||||
def __init__(self) -> None:
|
||||
self._definitions: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = list(self._definitions.values())
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
elif not include_archived:
|
||||
rows = [r for r in rows if r["status"] != "archived"]
|
||||
rows.sort(key=lambda r: r.get("updated_at") or "", reverse=True)
|
||||
return [
|
||||
{k: v for k, v in r.items() if k not in ("draft_graph", "draft_canvas_schema", "next_version_number")}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
out = copy.deepcopy(row)
|
||||
if not include_draft:
|
||||
out.pop("draft_graph", None)
|
||||
return out
|
||||
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
|
||||
graph = copy.deepcopy(data.get("draft_graph") or data.get("graph") or {
|
||||
"schemaVersion": "1.0",
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or "未命名工作流",
|
||||
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
|
||||
"edges": [],
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
"outputSchema": {"type": "object", "properties": {}},
|
||||
"settings": {},
|
||||
})
|
||||
graph.setdefault("id", workflow_id)
|
||||
now = _now()
|
||||
row = {
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or graph.get("name") or "未命名工作流",
|
||||
"description": data.get("description") or "",
|
||||
"owner_id": data["owner_id"],
|
||||
"status": data.get("status") or "draft",
|
||||
"draft_revision": 0,
|
||||
"next_version_number": 1,
|
||||
"draft_graph": graph,
|
||||
"draft_canvas_schema": str(data.get("draft_canvas_schema") or "{}"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._definitions[workflow_id] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data and data["name"] is not None:
|
||||
row["name"] = data["name"]
|
||||
if "description" in data and data["description"] is not None:
|
||||
row["description"] = data["description"]
|
||||
if "status" in data and data["status"] is not None:
|
||||
row["status"] = data["status"]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
return await self.update_definition(workflow_id, {"status": "archived"})
|
||||
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row["draft_revision"] != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
|
||||
payload = copy.deepcopy(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
row["draft_graph"] = payload
|
||||
row["draft_revision"] = expected_revision + 1
|
||||
if payload.get("name"):
|
||||
row["name"] = str(payload["name"])[:255]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row["draft_revision"] != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
|
||||
payload = copy.deepcopy(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
row["draft_graph"] = payload
|
||||
row["draft_canvas_schema"] = canvas_schema or "{}"
|
||||
row["draft_revision"] = expected_revision + 1
|
||||
if payload.get("name"):
|
||||
row["name"] = str(payload["name"])[:255]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
definition = self._definitions.get(workflow_id)
|
||||
if definition is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
existing = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
|
||||
# Allocate from the definition's own counter (mirrors the SQL store) so
|
||||
# the number never goes backwards even if events are purged/replayed.
|
||||
counter = int(definition.get("next_version_number") or 1)
|
||||
version_number = max(counter, max((v["version_number"] for v in existing), default=0) + 1)
|
||||
definition["next_version_number"] = version_number + 1
|
||||
version_id = f"wv_{uuid.uuid4().hex[:12]}"
|
||||
version = {
|
||||
"id": version_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_number": version_number,
|
||||
"graph": copy.deepcopy(graph),
|
||||
"graph_hash": graph_hash,
|
||||
"input_schema": graph.get("inputSchema") or graph.get("input_schema") or {},
|
||||
"output_schema": graph.get("outputSchema") or graph.get("output_schema") or {},
|
||||
"published_by": published_by,
|
||||
"published_at": _now(),
|
||||
"change_note": change_note or "",
|
||||
}
|
||||
self._versions[version_id] = version
|
||||
definition["status"] = "active"
|
||||
if canvas_schema_hash:
|
||||
definition["published_canvas_schema_hash"] = str(canvas_schema_hash)[:64]
|
||||
definition["updated_at"] = _now()
|
||||
return copy.deepcopy(version)
|
||||
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
rows = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
|
||||
rows.sort(key=lambda r: r["version_number"], reverse=True)
|
||||
return [{k: v for k, v in r.items() if k != "graph"} for r in rows]
|
||||
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
row = self._versions.get(version_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
by_wf: dict[str, dict[str, Any]] = {}
|
||||
for version in self._versions.values():
|
||||
wf_id = version["workflow_id"]
|
||||
if exclude_workflow_id and wf_id == exclude_workflow_id:
|
||||
continue
|
||||
definition = self._definitions.get(wf_id)
|
||||
if definition is None or definition.get("status") == "archived":
|
||||
continue
|
||||
current = by_wf.get(wf_id)
|
||||
if current is None or version["version_number"] > current["version_number"]:
|
||||
by_wf[wf_id] = {
|
||||
"workflow_id": wf_id,
|
||||
"name": definition["name"],
|
||||
"version_id": version["id"],
|
||||
"version_number": version["version_number"],
|
||||
"graph_hash": version["graph_hash"],
|
||||
"published_at": version["published_at"],
|
||||
}
|
||||
return sorted(by_wf.values(), key=lambda r: r.get("published_at") or "", reverse=True)
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
"""ORM models for workflow definitions and immutable published versions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowDefinitionRow(Base):
|
||||
"""Mutable workflow metadata + current draft canvas."""
|
||||
|
||||
__tablename__ = "workflow_definitions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# draft | active | archived
|
||||
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
|
||||
draft_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
# Monotonic version allocator. "1" = no version published yet; publish takes
|
||||
# the value then CAS-bumps it, replacing the old MAX(version_number)+1 race.
|
||||
next_version_number: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=1, server_default=sa_text("1")
|
||||
)
|
||||
draft_graph_json: Mapped[str] = mapped_column(PortableLongText(), default="{}", nullable=False)
|
||||
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
|
||||
# the executable WorkflowGraph lives in draft_graph_json above.
|
||||
draft_canvas_schema_json: Mapped[str] = mapped_column(
|
||||
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
|
||||
)
|
||||
# sha256 of the canvas document the current published version was compiled
|
||||
# from (audit trail only; publish always freezes the execution graph).
|
||||
published_canvas_schema_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowVersionRow(Base):
|
||||
"""Immutable published graph snapshot. Runs bind to version id."""
|
||||
|
||||
__tablename__ = "workflow_versions"
|
||||
__table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("workflow_definitions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
graph_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
graph_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
input_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
published_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
change_note: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
+385
@@ -0,0 +1,385 @@
|
||||
"""SQLAlchemy-backed workflow definition / version store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
|
||||
# Concurrent publishers CAS-retry against the workflow's own counter; a handful
|
||||
# of attempts is far more than any realistic contention window needs.
|
||||
_PUBLISH_ATTEMPTS = 6
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _parse_graph(raw: str | dict[str, Any] | None) -> dict[str, Any]:
|
||||
if raw is None:
|
||||
return {}
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
text = str(raw).strip()
|
||||
if not text:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _dump_graph(graph: dict[str, Any]) -> str:
|
||||
return json.dumps(graph, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def _definition_dict(row: WorkflowDefinitionRow, *, include_draft: bool) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"description": row.description or "",
|
||||
"owner_id": row.owner_id,
|
||||
"status": row.status,
|
||||
"draft_revision": row.draft_revision,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
if include_draft:
|
||||
data["draft_graph"] = _parse_graph(row.draft_graph_json)
|
||||
# Canvas document as stored (opaque JSON text; "{}" when never saved).
|
||||
data["draft_canvas_schema"] = row.draft_canvas_schema_json or "{}"
|
||||
return data
|
||||
|
||||
|
||||
def _version_dict(row: WorkflowVersionRow, *, include_graph: bool = True) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_number": row.version_number,
|
||||
"graph_hash": row.graph_hash,
|
||||
"input_schema": row.input_schema or {},
|
||||
"output_schema": row.output_schema or {},
|
||||
"published_by": row.published_by,
|
||||
"published_at": _iso(row.published_at),
|
||||
"change_note": row.change_note or "",
|
||||
}
|
||||
if include_graph:
|
||||
data["graph"] = _parse_graph(row.graph_json)
|
||||
return data
|
||||
|
||||
|
||||
class WorkflowRepository(WorkflowStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowDefinitionRow)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.status == status)
|
||||
elif not include_archived:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.status != "archived")
|
||||
stmt = stmt.order_by(WorkflowDefinitionRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_definition_dict(row, include_draft=False) for row in result.scalars()]
|
||||
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
return _definition_dict(row, include_draft=include_draft) if row is not None else None
|
||||
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
|
||||
graph = data.get("draft_graph") or data.get("graph") or {
|
||||
"schemaVersion": "1.0",
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or "未命名工作流",
|
||||
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
|
||||
"edges": [],
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
"outputSchema": {"type": "object", "properties": {}},
|
||||
"settings": {},
|
||||
}
|
||||
if isinstance(graph, dict) and not graph.get("id"):
|
||||
graph = {**graph, "id": workflow_id}
|
||||
if isinstance(graph, dict) and data.get("name") and not graph.get("name"):
|
||||
graph = {**graph, "name": data["name"]}
|
||||
row = WorkflowDefinitionRow(
|
||||
id=workflow_id,
|
||||
name=data.get("name") or graph.get("name") or "未命名工作流",
|
||||
description=data.get("description") or "",
|
||||
owner_id=data["owner_id"],
|
||||
status=data.get("status") or "draft",
|
||||
draft_revision=0,
|
||||
draft_graph_json=_dump_graph(graph),
|
||||
draft_canvas_schema_json=str(data.get("draft_canvas_schema") or "{}"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = _definition_dict(row, include_draft=True)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data and data["name"] is not None:
|
||||
row.name = data["name"]
|
||||
if "description" in data and data["description"] is not None:
|
||||
row.description = data["description"]
|
||||
if "status" in data and data["status"] is not None:
|
||||
row.status = data["status"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
out = _definition_dict(row, include_draft=True)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
return await self.update_definition(workflow_id, {"status": "archived"})
|
||||
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by # audit reserved for later
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row.draft_revision != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
|
||||
payload = dict(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
if row.name and not payload.get("name"):
|
||||
payload["name"] = row.name
|
||||
# Conditional UPDATE on the revision itself: two sessions that both
|
||||
# read the same revision cannot both win — the loser's WHERE clause
|
||||
# matches zero rows and surfaces a deterministic conflict instead of
|
||||
# silently overwriting the other editor's draft.
|
||||
values: dict[str, Any] = {
|
||||
"draft_graph_json": _dump_graph(payload),
|
||||
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
if payload.get("name"):
|
||||
values["name"] = str(payload["name"])[:255]
|
||||
result = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.draft_revision == expected_revision,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
current = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if current is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
|
||||
await session.commit()
|
||||
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
assert fresh is not None
|
||||
return _definition_dict(fresh, include_draft=True)
|
||||
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by # audit reserved for later
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row.draft_revision != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
|
||||
payload = dict(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
if row.name and not payload.get("name"):
|
||||
payload["name"] = row.name
|
||||
# One conditional UPDATE writes graph + canvas + revision together:
|
||||
# the two representations are always from the same save, and a
|
||||
# concurrent editor loses deterministically instead of interleaving
|
||||
# a new canvas onto someone else's graph.
|
||||
values: dict[str, Any] = {
|
||||
"draft_graph_json": _dump_graph(payload),
|
||||
"draft_canvas_schema_json": canvas_schema or "{}",
|
||||
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
if payload.get("name"):
|
||||
values["name"] = str(payload["name"])[:255]
|
||||
result = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.draft_revision == expected_revision,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
current = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if current is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
|
||||
await session.commit()
|
||||
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
assert fresh is not None
|
||||
return _definition_dict(fresh, include_draft=True)
|
||||
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Allocate the version number from the workflow's own counter.
|
||||
|
||||
``MAX(version_number) + 1`` races between concurrent publishes; here the
|
||||
number is claimed by CAS-bumping ``next_version_number`` inside the same
|
||||
transaction that inserts the version row, and the unique
|
||||
``(workflow_id, version_number)`` constraint is the final safety net.
|
||||
Concurrent publishers get adjacent, distinct numbers.
|
||||
"""
|
||||
last_error: Exception | None = None
|
||||
for _ in range(_PUBLISH_ATTEMPTS):
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
definition = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if definition is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
counter_seen = int(definition.next_version_number or 1)
|
||||
# Legacy rows got the column auto-added with server_default 1
|
||||
# while versions already exist — never allocate below the max.
|
||||
latest = (
|
||||
await session.execute(
|
||||
select(func.max(WorkflowVersionRow.version_number)).where(
|
||||
WorkflowVersionRow.workflow_id == workflow_id
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
number = max(counter_seen, int(latest) + 1)
|
||||
claimed = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.next_version_number == counter_seen,
|
||||
)
|
||||
.values(next_version_number=number + 1)
|
||||
)
|
||||
if claimed.rowcount == 0:
|
||||
# Another publish landed first — release nothing (the
|
||||
# counter bump never happened) and retry with fresh state.
|
||||
await session.rollback()
|
||||
last_error = None
|
||||
continue
|
||||
version = WorkflowVersionRow(
|
||||
id=f"wv_{uuid.uuid4().hex[:12]}",
|
||||
workflow_id=workflow_id,
|
||||
version_number=number,
|
||||
graph_json=_dump_graph(graph),
|
||||
graph_hash=graph_hash,
|
||||
input_schema=graph.get("inputSchema") or graph.get("input_schema") or {},
|
||||
output_schema=graph.get("outputSchema") or graph.get("output_schema") or {},
|
||||
published_by=published_by,
|
||||
change_note=change_note or "",
|
||||
)
|
||||
definition.status = "active"
|
||||
definition.updated_at = datetime.now(UTC)
|
||||
if canvas_schema_hash:
|
||||
definition.published_canvas_schema_hash = canvas_schema_hash[:64]
|
||||
session.add(version)
|
||||
await session.flush()
|
||||
out = _version_dict(version, include_graph=True)
|
||||
await session.commit()
|
||||
return out
|
||||
except IntegrityError as exc: # pragma: no cover - CAS makes this near-impossible
|
||||
last_error = exc
|
||||
continue
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError(f"failed to allocate a workflow version number for {workflow_id}")
|
||||
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(WorkflowVersionRow)
|
||||
.where(WorkflowVersionRow.workflow_id == workflow_id)
|
||||
.order_by(WorkflowVersionRow.version_number.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_version_dict(row, include_graph=False) for row in result.scalars()]
|
||||
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowVersionRow, version_id)
|
||||
return _version_dict(row, include_graph=True) if row is not None else None
|
||||
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(WorkflowVersionRow, WorkflowDefinitionRow)
|
||||
.join(WorkflowDefinitionRow, WorkflowDefinitionRow.id == WorkflowVersionRow.workflow_id)
|
||||
.where(WorkflowDefinitionRow.status != "archived")
|
||||
.order_by(WorkflowVersionRow.published_at.desc())
|
||||
)
|
||||
if exclude_workflow_id:
|
||||
stmt = stmt.where(WorkflowVersionRow.workflow_id != exclude_workflow_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for version, definition in result.all():
|
||||
# Latest version per workflow only.
|
||||
if version.workflow_id in seen:
|
||||
continue
|
||||
seen.add(version.workflow_id)
|
||||
out.append(
|
||||
{
|
||||
"workflow_id": definition.id,
|
||||
"name": definition.name,
|
||||
"version_id": version.id,
|
||||
"version_number": version.version_number,
|
||||
"graph_hash": version.graph_hash,
|
||||
"published_at": _iso(version.published_at),
|
||||
}
|
||||
)
|
||||
return out
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Workflow Studio protocol package (phase 0 — frozen contracts).
|
||||
|
||||
Execution, persistence, and Gateway routers land in later phases.
|
||||
This package must never import ``app.*``.
|
||||
"""
|
||||
|
||||
from deerflow.workflows.errors import (
|
||||
ALL_WORKFLOW_ERROR_CODES,
|
||||
WorkflowError,
|
||||
WorkflowErrorBody,
|
||||
WorkflowErrorCode,
|
||||
)
|
||||
from deerflow.workflows.events import (
|
||||
ALL_WORKFLOW_EVENT_TYPES,
|
||||
TERMINAL_EVENT_TYPES,
|
||||
WorkflowEventEnvelope,
|
||||
WorkflowEventSink,
|
||||
WorkflowEventType,
|
||||
)
|
||||
from deerflow.workflows.schemas import (
|
||||
ALL_NODE_TYPES,
|
||||
ALL_RUN_STATUSES,
|
||||
SCHEMA_VERSION,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
ArtifactRef,
|
||||
NodeResult,
|
||||
NodeType,
|
||||
ResumeRunRequest,
|
||||
RunStatus,
|
||||
StartRunRequest,
|
||||
WorkflowEdge,
|
||||
WorkflowGraph,
|
||||
WorkflowGraphSettings,
|
||||
WorkflowNode,
|
||||
)
|
||||
from deerflow.workflows.validator import assert_publishable, validate_workflow_graph
|
||||
|
||||
__all__ = [
|
||||
"ALL_NODE_TYPES",
|
||||
"ALL_RUN_STATUSES",
|
||||
"ALL_WORKFLOW_ERROR_CODES",
|
||||
"ALL_WORKFLOW_EVENT_TYPES",
|
||||
"SCHEMA_VERSION",
|
||||
"TERMINAL_EVENT_TYPES",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"ArtifactRef",
|
||||
"NodeResult",
|
||||
"NodeType",
|
||||
"ResumeRunRequest",
|
||||
"RunStatus",
|
||||
"StartRunRequest",
|
||||
"WorkflowEdge",
|
||||
"WorkflowError",
|
||||
"WorkflowErrorBody",
|
||||
"WorkflowErrorCode",
|
||||
"WorkflowEventEnvelope",
|
||||
"WorkflowEventSink",
|
||||
"WorkflowEventType",
|
||||
"WorkflowGraph",
|
||||
"WorkflowGraphSettings",
|
||||
"WorkflowNode",
|
||||
"assert_publishable",
|
||||
"validate_workflow_graph",
|
||||
]
|
||||
@@ -0,0 +1,150 @@
|
||||
"""Stable workflow error codes and exception types (protocol v1).
|
||||
|
||||
User-visible ``message`` must never include stack traces, SQL passwords,
|
||||
full response bodies, or internal host paths. Full exceptions go to
|
||||
controlled server logs only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
WorkflowErrorCode = Literal[
|
||||
# graph / publish
|
||||
"WORKFLOW_SCHEMA_INVALID",
|
||||
"WORKFLOW_GRAPH_CYCLE",
|
||||
"WORKFLOW_NODE_NOT_FOUND",
|
||||
"WORKFLOW_EDGE_INVALID",
|
||||
"WORKFLOW_EXPRESSION_INVALID",
|
||||
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"WORKFLOW_RESOURCE_FORBIDDEN",
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"WORKFLOW_DRAFT_CONFLICT",
|
||||
"WORKFLOW_VERSION_NOT_FOUND",
|
||||
# run lifecycle
|
||||
"WORKFLOW_RUN_NOT_FOUND",
|
||||
"WORKFLOW_RUN_CONFLICT",
|
||||
"WORKFLOW_RUN_NOT_RESUMABLE",
|
||||
"WORKFLOW_RESUME_TOKEN_INVALID",
|
||||
"WORKFLOW_CANCELLED",
|
||||
"WORKFLOW_TIMEOUT",
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
"WORKFLOW_MAX_LOOP_ITERATIONS",
|
||||
"WORKFLOW_BUDGET_EXCEEDED",
|
||||
# node categories
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
"WORKFLOW_AGENT_FAILED",
|
||||
"WORKFLOW_SKILL_FAILED",
|
||||
"WORKFLOW_HTTP_POLICY_DENIED",
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"WORKFLOW_SQL_POLICY_DENIED",
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
"WORKFLOW_CONDITION_FAILED",
|
||||
"WORKFLOW_MERGE_FAILED",
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
# auth / embed
|
||||
"WORKFLOW_FORBIDDEN",
|
||||
"WORKFLOW_EMBED_TICKET_INVALID",
|
||||
"WORKFLOW_EMBED_ORIGIN_DENIED",
|
||||
# generic
|
||||
"WORKFLOW_INTERNAL",
|
||||
]
|
||||
|
||||
ALL_WORKFLOW_ERROR_CODES: tuple[WorkflowErrorCode, ...] = (
|
||||
"WORKFLOW_SCHEMA_INVALID",
|
||||
"WORKFLOW_GRAPH_CYCLE",
|
||||
"WORKFLOW_NODE_NOT_FOUND",
|
||||
"WORKFLOW_EDGE_INVALID",
|
||||
"WORKFLOW_EXPRESSION_INVALID",
|
||||
"WORKFLOW_EXPRESSION_UPSTREAM_REQUIRED",
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"WORKFLOW_RESOURCE_FORBIDDEN",
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"WORKFLOW_DRAFT_CONFLICT",
|
||||
"WORKFLOW_VERSION_NOT_FOUND",
|
||||
"WORKFLOW_RUN_NOT_FOUND",
|
||||
"WORKFLOW_RUN_CONFLICT",
|
||||
"WORKFLOW_RUN_NOT_RESUMABLE",
|
||||
"WORKFLOW_RESUME_TOKEN_INVALID",
|
||||
"WORKFLOW_CANCELLED",
|
||||
"WORKFLOW_TIMEOUT",
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
"WORKFLOW_MAX_LOOP_ITERATIONS",
|
||||
"WORKFLOW_BUDGET_EXCEEDED",
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
"WORKFLOW_AGENT_FAILED",
|
||||
"WORKFLOW_SKILL_FAILED",
|
||||
"WORKFLOW_HTTP_POLICY_DENIED",
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"WORKFLOW_SQL_POLICY_DENIED",
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
"WORKFLOW_CONDITION_FAILED",
|
||||
"WORKFLOW_MERGE_FAILED",
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
"WORKFLOW_FORBIDDEN",
|
||||
"WORKFLOW_EMBED_TICKET_INVALID",
|
||||
"WORKFLOW_EMBED_ORIGIN_DENIED",
|
||||
"WORKFLOW_INTERNAL",
|
||||
)
|
||||
|
||||
|
||||
class WorkflowErrorBody(BaseModel):
|
||||
"""Wire-format error body shared by API responses and ``run.failed`` events."""
|
||||
|
||||
code: WorkflowErrorCode
|
||||
message: str
|
||||
retryable: bool = False
|
||||
node_id: str | None = Field(default=None, alias="nodeId")
|
||||
details: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class WorkflowError(Exception):
|
||||
"""Raised by validators / node executors; convertible to :class:`WorkflowErrorBody`."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
code: WorkflowErrorCode,
|
||||
message: str,
|
||||
*,
|
||||
retryable: bool = False,
|
||||
node_id: str | None = None,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.retryable = retryable
|
||||
self.node_id = node_id
|
||||
self.details = details or {}
|
||||
|
||||
def to_body(self) -> WorkflowErrorBody:
|
||||
return WorkflowErrorBody(
|
||||
code=self.code,
|
||||
message=self.message,
|
||||
retryable=self.retryable,
|
||||
node_id=self.node_id,
|
||||
details=self.details,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ALL_WORKFLOW_ERROR_CODES",
|
||||
"WorkflowError",
|
||||
"WorkflowErrorBody",
|
||||
"WorkflowErrorCode",
|
||||
]
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Workflow SSE event protocol v1.0.
|
||||
|
||||
Invariant: **persist first, then publish**. ``seq`` is monotonic per run.
|
||||
Heartbeat comments (``: heartbeat``) are live-only and never stored.
|
||||
|
||||
Canonical ``data`` payloads (frozen contract — REST replay and SSE must agree):
|
||||
|
||||
node.tool.started {"name": "web_search", "toolCallId": "call_001",
|
||||
"title": "联网检索", "args": {"path": "..."}}
|
||||
node.tool.finished {"name": "web_search", "toolCallId": "call_001",
|
||||
"status": "succeeded" | "failed", "summary": "获得 12 条结果"}
|
||||
run.awaiting_input {"resumeToken": "one-time-token", "prompt": "请确认是否发布",
|
||||
"formSchema": {...}, "actions": ["approve", "reject"]}
|
||||
artifact.created {"artifactId": "artifact_001", "name": "weekly-report.docx",
|
||||
"kind": "file" | "markdown" | "json" | "table",
|
||||
"mimeType": "...", "sizeBytes": 1234, "preview": "..."}
|
||||
node.started {"nodeType": ..., "name": ..., "attempt": 1,
|
||||
"agent": {...}, "parallelGroupId": "...", "roundIndex": 2}
|
||||
|
||||
Rules: tool identity is ``name`` (never ``toolName``); outcomes are ``status``
|
||||
(never ``ok``); human-input descriptors are flattened (never nested under
|
||||
``pendingInput``); artifacts never carry filesystem paths; agent nodes expose
|
||||
display metadata only (no system prompts, skill whitelists or credentials).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
|
||||
WorkflowEventType = Literal[
|
||||
"run.created",
|
||||
"run.queued",
|
||||
"run.started",
|
||||
"node.queued",
|
||||
"node.started",
|
||||
"node.progress",
|
||||
"node.output.delta",
|
||||
"node.tool.started",
|
||||
"node.tool.finished",
|
||||
"artifact.created",
|
||||
"node.completed",
|
||||
"node.failed",
|
||||
"run.awaiting_input",
|
||||
"run.resumed",
|
||||
"run.cancel_requested",
|
||||
"run.cancelled",
|
||||
"run.completed",
|
||||
"run.failed",
|
||||
]
|
||||
|
||||
ALL_WORKFLOW_EVENT_TYPES: tuple[WorkflowEventType, ...] = (
|
||||
"run.created",
|
||||
"run.queued",
|
||||
"run.started",
|
||||
"node.queued",
|
||||
"node.started",
|
||||
"node.progress",
|
||||
"node.output.delta",
|
||||
"node.tool.started",
|
||||
"node.tool.finished",
|
||||
"artifact.created",
|
||||
"node.completed",
|
||||
"node.failed",
|
||||
"run.awaiting_input",
|
||||
"run.resumed",
|
||||
"run.cancel_requested",
|
||||
"run.cancelled",
|
||||
"run.completed",
|
||||
"run.failed",
|
||||
)
|
||||
|
||||
TERMINAL_EVENT_TYPES: frozenset[WorkflowEventType] = frozenset({"run.completed", "run.failed", "run.cancelled"})
|
||||
|
||||
_KIND_BY_MIME: tuple[tuple[tuple[str, ...], str], ...] = (
|
||||
(("text/markdown", "text/x-markdown"), "markdown"),
|
||||
(("application/json", "text/json", "application/javascript"), "json"),
|
||||
(("text/csv", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), "table"),
|
||||
)
|
||||
|
||||
|
||||
def artifact_kind(mime_type: str) -> str:
|
||||
"""Coarse wire-level artifact category: file | markdown | json | table."""
|
||||
lowered = (mime_type or "").lower()
|
||||
for prefixes, kind in _KIND_BY_MIME:
|
||||
if any(lowered.startswith(prefix) for prefix in prefixes):
|
||||
return kind
|
||||
return "file"
|
||||
|
||||
|
||||
class WorkflowEventEnvelope(BaseModel):
|
||||
"""Common envelope for every persisted / SSE workflow event."""
|
||||
|
||||
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
|
||||
run_id: str = Field(alias="runId")
|
||||
workflow_id: str = Field(alias="workflowId")
|
||||
version_id: str = Field(alias="versionId")
|
||||
seq: int = 0
|
||||
event_type: WorkflowEventType = Field(alias="event")
|
||||
node_id: str | None = Field(default=None, alias="nodeId")
|
||||
node_run_id: str | None = Field(default=None, alias="nodeRunId")
|
||||
timestamp: datetime = Field(default_factory=lambda: datetime.now(UTC))
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
def to_sse_dict(self) -> dict[str, Any]:
|
||||
"""JSON object for the SSE ``data:`` line (camelCase wire fields)."""
|
||||
return {
|
||||
"schemaVersion": self.schema_version,
|
||||
"runId": self.run_id,
|
||||
"workflowId": self.workflow_id,
|
||||
"versionId": self.version_id,
|
||||
"seq": self.seq,
|
||||
"event": self.event_type,
|
||||
"nodeId": self.node_id,
|
||||
"nodeRunId": self.node_run_id,
|
||||
"timestamp": self.timestamp.isoformat() if self.timestamp else None,
|
||||
"data": self.data,
|
||||
}
|
||||
|
||||
def to_sse_frame(self) -> str:
|
||||
"""Full SSE frame including ``id`` / ``event`` / ``data`` lines."""
|
||||
import json
|
||||
|
||||
payload = json.dumps(self.to_sse_dict(), ensure_ascii=False, separators=(",", ":"))
|
||||
return f"id: {self.seq}\nevent: {self.event_type}\ndata: {payload}\n\n"
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class WorkflowEventSink(Protocol):
|
||||
"""Persist-then-publish sink used by the future executor."""
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> WorkflowEventEnvelope:
|
||||
"""Persist event (assign ``seq``), then fan out to live subscribers."""
|
||||
...
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ALL_WORKFLOW_EVENT_TYPES",
|
||||
"SCHEMA_VERSION",
|
||||
"TERMINAL_EVENT_TYPES",
|
||||
"WorkflowEventEnvelope",
|
||||
"WorkflowEventSink",
|
||||
"WorkflowEventType",
|
||||
"artifact_kind",
|
||||
]
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Safe expression / template evaluation for workflow node configs.
|
||||
|
||||
Two surfaces, both intentionally tiny:
|
||||
|
||||
* ``render_template("… {{ nodes.a.data.title }} …", state)`` — string
|
||||
interpolation over a whitelisted state tree.
|
||||
* ``evaluate_condition("nodes.a.data.score > 0.5", state)`` — boolean
|
||||
predicate over the same tree.
|
||||
|
||||
Neither uses :func:`eval`. The condition grammar is a single comparison or a
|
||||
chain joined by ``and`` / ``or``; operands are either a state path or a JSON
|
||||
literal. Anything else raises ``WORKFLOW_EXPRESSION_INVALID`` so a malformed
|
||||
graph fails loudly at publish time instead of silently at run time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
_TEMPLATE_RE = re.compile(r"\{\{\s*([^{}]+?)\s*\}\}")
|
||||
_ROOT_KEYS = ("inputs", "nodes", "run", "loop", "env")
|
||||
_PATH_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\])*$")
|
||||
_COMPARATORS = ("==", "!=", ">=", "<=", ">", "<", " in ", " contains ")
|
||||
_MAX_RENDER_CHARS = 200_000
|
||||
|
||||
|
||||
def _invalid(expr: str, reason: str) -> WorkflowError:
|
||||
return WorkflowError(
|
||||
"WORKFLOW_EXPRESSION_INVALID",
|
||||
f"表达式不合法:{reason}",
|
||||
details={"expression": expr[:200]},
|
||||
)
|
||||
|
||||
|
||||
def resolve_path(path: str, state: dict[str, Any]) -> Any:
|
||||
"""Read ``a.b[0].c`` from the state tree. Missing segments yield ``None``."""
|
||||
expr = path.strip()
|
||||
if not _PATH_RE.match(expr):
|
||||
raise _invalid(expr, "路径格式不合法")
|
||||
root = expr.split(".", 1)[0].split("[", 1)[0]
|
||||
if root not in _ROOT_KEYS:
|
||||
raise _invalid(expr, f"根节点必须是 {'/'.join(_ROOT_KEYS)} 之一")
|
||||
current: Any = state
|
||||
for segment in re.findall(r"[A-Za-z_][A-Za-z0-9_\-]*|\[\d+\]", expr):
|
||||
if current is None:
|
||||
return None
|
||||
if segment.startswith("["):
|
||||
index = int(segment[1:-1])
|
||||
if not isinstance(current, (list, tuple)) or index >= len(current):
|
||||
return None
|
||||
current = current[index]
|
||||
elif isinstance(current, dict):
|
||||
current = current.get(segment)
|
||||
else:
|
||||
return None
|
||||
return current
|
||||
|
||||
|
||||
def _stringify(value: Any) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value)
|
||||
return json.dumps(value, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def render_template(template: str, state: dict[str, Any]) -> str:
|
||||
"""Replace every ``{{ path }}`` with its stringified state value."""
|
||||
if not template:
|
||||
return ""
|
||||
|
||||
def _sub(match: re.Match[str]) -> str:
|
||||
return _stringify(resolve_path(match.group(1), state))
|
||||
|
||||
rendered = _TEMPLATE_RE.sub(_sub, template)
|
||||
if len(rendered) > _MAX_RENDER_CHARS:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
"模板渲染结果超出长度上限",
|
||||
details={"limit": _MAX_RENDER_CHARS},
|
||||
)
|
||||
return rendered
|
||||
|
||||
|
||||
def render_value(value: Any, state: dict[str, Any]) -> Any:
|
||||
"""Recursively render templates inside dict / list / str structures.
|
||||
|
||||
A string that is exactly one ``{{ path }}`` keeps the resolved value's
|
||||
native type instead of being stringified.
|
||||
"""
|
||||
if isinstance(value, str):
|
||||
match = _TEMPLATE_RE.fullmatch(value.strip())
|
||||
if match:
|
||||
return resolve_path(match.group(1), state)
|
||||
return render_template(value, state)
|
||||
if isinstance(value, dict):
|
||||
return {k: render_value(v, state) for k, v in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [render_value(v, state) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def _operand(token: str, state: dict[str, Any]) -> Any:
|
||||
text = token.strip()
|
||||
if not text:
|
||||
raise _invalid(token, "操作数为空")
|
||||
match = _TEMPLATE_RE.fullmatch(text)
|
||||
if match:
|
||||
return resolve_path(match.group(1), state)
|
||||
lowered = text.lower()
|
||||
if lowered in ("true", "false"):
|
||||
return lowered == "true"
|
||||
if lowered in ("null", "none"):
|
||||
return None
|
||||
if (text[0], text[-1]) in (('"', '"'), ("'", "'")) and len(text) >= 2:
|
||||
return text[1:-1]
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
if _PATH_RE.match(text) and text.split(".", 1)[0].split("[", 1)[0] in _ROOT_KEYS:
|
||||
return resolve_path(text, state)
|
||||
return text
|
||||
|
||||
|
||||
def _truthy(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() not in ("", "false", "0", "null", "none")
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _compare(left: Any, op: str, right: Any) -> bool:
|
||||
if op == "==":
|
||||
return left == right
|
||||
if op == "!=":
|
||||
return left != right
|
||||
if op == "in":
|
||||
try:
|
||||
return left in right # type: ignore[operator]
|
||||
except TypeError:
|
||||
return False
|
||||
if op == "contains":
|
||||
try:
|
||||
return right in left # type: ignore[operator]
|
||||
except TypeError:
|
||||
return False
|
||||
try:
|
||||
if op == ">":
|
||||
return left > right
|
||||
if op == "<":
|
||||
return left < right
|
||||
if op == ">=":
|
||||
return left >= right
|
||||
if op == "<=":
|
||||
return left <= right
|
||||
except TypeError:
|
||||
return False
|
||||
raise _invalid(op, f"不支持的比较运算符 {op}")
|
||||
|
||||
|
||||
def _split_top_level(expr: str, keyword: str) -> list[str] | None:
|
||||
"""Split on ``keyword`` outside quotes / braces; None when absent."""
|
||||
parts: list[str] = []
|
||||
depth = 0
|
||||
quote: str | None = None
|
||||
buffer: list[str] = []
|
||||
index = 0
|
||||
pattern = f" {keyword} "
|
||||
while index < len(expr):
|
||||
char = expr[index]
|
||||
if quote:
|
||||
buffer.append(char)
|
||||
if char == quote:
|
||||
quote = None
|
||||
index += 1
|
||||
continue
|
||||
if char in ('"', "'"):
|
||||
quote = char
|
||||
buffer.append(char)
|
||||
index += 1
|
||||
continue
|
||||
if char in "[{(":
|
||||
depth += 1
|
||||
elif char in "]})":
|
||||
depth -= 1
|
||||
if depth == 0 and expr[index : index + len(pattern)] == pattern:
|
||||
parts.append("".join(buffer))
|
||||
buffer = []
|
||||
index += len(pattern)
|
||||
continue
|
||||
buffer.append(char)
|
||||
index += 1
|
||||
if not parts:
|
||||
return None
|
||||
parts.append("".join(buffer))
|
||||
return parts
|
||||
|
||||
|
||||
def evaluate_condition(expression: str, state: dict[str, Any]) -> bool:
|
||||
"""Evaluate a boolean predicate without ``eval``."""
|
||||
expr = (expression or "").strip()
|
||||
if not expr:
|
||||
return False
|
||||
|
||||
or_parts = _split_top_level(expr, "or")
|
||||
if or_parts:
|
||||
return any(evaluate_condition(part, state) for part in or_parts)
|
||||
and_parts = _split_top_level(expr, "and")
|
||||
if and_parts:
|
||||
return all(evaluate_condition(part, state) for part in and_parts)
|
||||
|
||||
if expr.startswith("not "):
|
||||
return not evaluate_condition(expr[4:], state)
|
||||
if expr.startswith("(") and expr.endswith(")"):
|
||||
return evaluate_condition(expr[1:-1], state)
|
||||
|
||||
for comparator in _COMPARATORS:
|
||||
op = comparator.strip()
|
||||
index = expr.find(comparator)
|
||||
if index <= 0:
|
||||
continue
|
||||
left = _operand(expr[:index], state)
|
||||
right = _operand(expr[index + len(comparator) :], state)
|
||||
return _compare(left, op, right)
|
||||
|
||||
return _truthy(_operand(expr, state))
|
||||
|
||||
|
||||
_NODES_IDENT_RE = re.compile(r"(?:\{\{\s*)?nodes\.([A-Za-z_][A-Za-z0-9_\-]*)")
|
||||
|
||||
|
||||
def collect_referenced_nodes(value: Any) -> set[str]:
|
||||
"""Node ids referenced through ``nodes.<id>`` anywhere in a config tree.
|
||||
|
||||
Covers both ``{{ nodes.x.data }}`` templates and bare condition paths
|
||||
such as ``nodes.x.data.score > 0.5``.
|
||||
"""
|
||||
found: set[str] = set()
|
||||
if isinstance(value, str):
|
||||
found.update(_NODES_IDENT_RE.findall(value))
|
||||
elif isinstance(value, dict):
|
||||
for item in value.values():
|
||||
found |= collect_referenced_nodes(item)
|
||||
elif isinstance(value, list):
|
||||
for item in value:
|
||||
found |= collect_referenced_nodes(item)
|
||||
return found
|
||||
|
||||
|
||||
__all__ = [
|
||||
"collect_referenced_nodes",
|
||||
"evaluate_condition",
|
||||
"render_template",
|
||||
"render_value",
|
||||
"resolve_path",
|
||||
]
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
"""Node executors for the workflow runtime."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.workflows.nodes.agents import (
|
||||
AgentNodeExecutor,
|
||||
SkillNodeExecutor,
|
||||
SubworkflowNodeExecutor,
|
||||
)
|
||||
from deerflow.workflows.nodes.base import ACTIVE_PORTS, NodeExecutor, NodeRegistry
|
||||
from deerflow.workflows.nodes.core import (
|
||||
ConditionNodeExecutor,
|
||||
LoopNodeExecutor,
|
||||
MergeNodeExecutor,
|
||||
OutputNodeExecutor,
|
||||
StartNodeExecutor,
|
||||
TransformNodeExecutor,
|
||||
)
|
||||
from deerflow.workflows.nodes.human import HumanInputNodeExecutor
|
||||
from deerflow.workflows.nodes.integrations import (
|
||||
CodeNodeExecutor,
|
||||
HttpNodeExecutor,
|
||||
SqlReadNodeExecutor,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ACTIVE_PORTS",
|
||||
"AgentNodeExecutor",
|
||||
"CodeNodeExecutor",
|
||||
"ConditionNodeExecutor",
|
||||
"HttpNodeExecutor",
|
||||
"HumanInputNodeExecutor",
|
||||
"LoopNodeExecutor",
|
||||
"MergeNodeExecutor",
|
||||
"NodeExecutor",
|
||||
"NodeRegistry",
|
||||
"OutputNodeExecutor",
|
||||
"SkillNodeExecutor",
|
||||
"SqlReadNodeExecutor",
|
||||
"StartNodeExecutor",
|
||||
"SubworkflowNodeExecutor",
|
||||
"TransformNodeExecutor",
|
||||
"build_default_registry",
|
||||
]
|
||||
|
||||
|
||||
def build_default_registry(config: WorkflowConfig | None = None) -> NodeRegistry:
|
||||
"""Registry with every executable node type wired up."""
|
||||
cfg = config or WorkflowConfig()
|
||||
registry = NodeRegistry()
|
||||
registry.register("start", StartNodeExecutor())
|
||||
registry.register("output", OutputNodeExecutor())
|
||||
registry.register("transform", TransformNodeExecutor())
|
||||
registry.register("condition", ConditionNodeExecutor())
|
||||
registry.register("merge", MergeNodeExecutor())
|
||||
registry.register("loop", LoopNodeExecutor())
|
||||
registry.register("human_input", HumanInputNodeExecutor())
|
||||
registry.register("http", HttpNodeExecutor(cfg))
|
||||
registry.register("sql_read", SqlReadNodeExecutor(cfg))
|
||||
registry.register("code", CodeNodeExecutor(cfg))
|
||||
registry.register("agent", AgentNodeExecutor())
|
||||
registry.register("skill", SkillNodeExecutor())
|
||||
registry.register("subworkflow", SubworkflowNodeExecutor())
|
||||
return registry
|
||||
@@ -0,0 +1,361 @@
|
||||
"""LLM-backed nodes: ``agent``, ``skill``, ``subworkflow``.
|
||||
|
||||
These are the only nodes that call the DeerFlow LangGraph lead agent. The agent
|
||||
invocation itself is injected (``deps.run_agent``) so the harness keeps its
|
||||
import firewall; this module owns prompt rendering, streaming fan-out, JSON
|
||||
response coercion, and error typing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import render_template, render_value
|
||||
from deerflow.workflows.nodes.base import require_dep
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.schemas import ArtifactRef, NodeResult, WorkflowNode
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JSON_BLOCK = re.compile(r"```(?:json)?\s*(.+?)\s*```", re.DOTALL)
|
||||
_DELTA_FLUSH_SECONDS = 0.05
|
||||
_DELTA_FLUSH_CHARS = 1024
|
||||
|
||||
|
||||
class _DeltaCoalescer:
|
||||
"""Merge model tokens into 50ms / 1KB event payloads (§9.5)."""
|
||||
|
||||
def __init__(self, ctx: RunContext, node_id: str) -> None:
|
||||
self._ctx = ctx
|
||||
self._node_id = node_id
|
||||
self._parts: list[str] = []
|
||||
self._chars = 0
|
||||
self._task: asyncio.Task | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def push(self, chunk: str) -> None:
|
||||
if not chunk:
|
||||
return
|
||||
async with self._lock:
|
||||
self._parts.append(chunk)
|
||||
self._chars += len(chunk)
|
||||
if self._chars >= _DELTA_FLUSH_CHARS:
|
||||
await self._flush_unlocked()
|
||||
return
|
||||
if self._task is None or self._task.done():
|
||||
self._task = asyncio.create_task(self._delayed())
|
||||
|
||||
async def _delayed(self) -> None:
|
||||
await asyncio.sleep(_DELTA_FLUSH_SECONDS)
|
||||
async with self._lock:
|
||||
await self._flush_unlocked()
|
||||
|
||||
async def _flush_unlocked(self) -> None:
|
||||
if not self._parts:
|
||||
return
|
||||
text = "".join(self._parts)
|
||||
self._parts.clear()
|
||||
self._chars = 0
|
||||
await self._ctx.emit_event(
|
||||
"node.output.delta",
|
||||
data={"channel": "answer", "delta": text, "text": text},
|
||||
node_id=self._node_id,
|
||||
)
|
||||
|
||||
async def flush(self) -> None:
|
||||
task = self._task
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
async with self._lock:
|
||||
await self._flush_unlocked()
|
||||
|
||||
|
||||
def _extract_json(text: str) -> Any:
|
||||
candidate = text.strip()
|
||||
match = _JSON_BLOCK.search(candidate)
|
||||
if match:
|
||||
candidate = match.group(1).strip()
|
||||
else:
|
||||
start = min((i for i in (candidate.find("{"), candidate.find("[")) if i >= 0), default=-1)
|
||||
if start > 0:
|
||||
candidate = candidate[start:]
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
_SAFE_ARG_STRING_CHARS = 160
|
||||
_SAFE_ARG_MAX_KEYS = 12
|
||||
|
||||
|
||||
def _safe_tool_args(args: Any) -> dict[str, Any]:
|
||||
"""Minimal safe projection of tool-call args for the event log.
|
||||
|
||||
Scalars only (strings truncated), no nested containers, bounded key count —
|
||||
enough for the canvas to label the step without leaking full payloads such
|
||||
as entire documents, prompts or credentials that a tool may carry."""
|
||||
if not isinstance(args, dict):
|
||||
return {}
|
||||
out: dict[str, Any] = {}
|
||||
for key, value in list(args.items())[:_SAFE_ARG_MAX_KEYS]:
|
||||
if isinstance(value, bool) or not isinstance(value, (str, int, float)):
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
value = value[:_SAFE_ARG_STRING_CHARS]
|
||||
out[str(key)[:64]] = value
|
||||
return out
|
||||
|
||||
|
||||
class _AgentInvocation:
|
||||
"""Shared plumbing for ``agent`` and ``skill`` (agent_skill mode)."""
|
||||
|
||||
async def _invoke(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
*,
|
||||
agent_id: str,
|
||||
prompt: str,
|
||||
skill_names: list[str],
|
||||
response_mode: str,
|
||||
response_schema: dict[str, Any],
|
||||
thread_mode: str,
|
||||
failure_code: str,
|
||||
) -> NodeResult:
|
||||
run_agent = require_dep(ctx.deps.run_agent, node=node, what="智能体执行")
|
||||
coalescer = _DeltaCoalescer(ctx, node.id)
|
||||
|
||||
async def on_delta(chunk: str) -> None:
|
||||
await coalescer.push(chunk)
|
||||
|
||||
async def on_tool(event: dict[str, Any]) -> None:
|
||||
phase = str(event.get("phase") or "started")
|
||||
name = str(event.get("name") or "")
|
||||
call_id = str(event.get("id") or "")
|
||||
if phase == "started":
|
||||
await ctx.emit_event(
|
||||
"node.tool.started",
|
||||
data={
|
||||
"name": name,
|
||||
"toolCallId": call_id,
|
||||
"title": str(event.get("title") or name),
|
||||
"args": _safe_tool_args(event.get("args")),
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
return
|
||||
ok = event.get("ok")
|
||||
await ctx.emit_event(
|
||||
"node.tool.finished",
|
||||
data={
|
||||
"name": name,
|
||||
"toolCallId": call_id,
|
||||
"status": "succeeded" if ok in (True, None) else "failed",
|
||||
"summary": event.get("summary"),
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
|
||||
try:
|
||||
try:
|
||||
outcome = await run_agent(
|
||||
run_id=ctx.run_id,
|
||||
node_id=node.id,
|
||||
owner_id=ctx.owner_id,
|
||||
agent_id=agent_id,
|
||||
prompt=prompt,
|
||||
skill_names=skill_names,
|
||||
thread_mode=thread_mode,
|
||||
cancel=ctx.cancel,
|
||||
on_delta=on_delta,
|
||||
on_tool=on_tool,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - agent failures are expected
|
||||
logger.warning("workflow agent node %s failed", node.id, exc_info=True)
|
||||
raise WorkflowError(
|
||||
failure_code,
|
||||
"智能体执行失败",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
finally:
|
||||
await coalescer.flush()
|
||||
|
||||
text = str(outcome.get("text") or "")
|
||||
data: dict[str, Any] = {"text": text}
|
||||
if response_mode == "json":
|
||||
payload = outcome.get("json")
|
||||
if payload is None:
|
||||
payload = _extract_json(text)
|
||||
if payload is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"智能体未返回可解析的 JSON",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"preview": text[:500]},
|
||||
)
|
||||
required = response_schema.get("required") if isinstance(response_schema, dict) else None
|
||||
if isinstance(required, list) and isinstance(payload, dict):
|
||||
missing = [str(k) for k in required if payload.get(str(k)) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
f"智能体返回缺少字段:{'、'.join(missing)}",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
data["json"] = payload
|
||||
|
||||
artifacts: list[ArtifactRef] = []
|
||||
for raw in outcome.get("artifacts") or []:
|
||||
if not isinstance(raw, dict) or not raw.get("name"):
|
||||
continue
|
||||
ref = await ctx.register_artifact(
|
||||
node,
|
||||
name=str(raw.get("name")),
|
||||
mime_type=str(raw.get("mime_type") or raw.get("mimeType") or "application/octet-stream"),
|
||||
path=str(raw.get("path") or ""),
|
||||
size_bytes=raw.get("size_bytes") or raw.get("sizeBytes"),
|
||||
preview=raw.get("preview"),
|
||||
)
|
||||
if ref is not None:
|
||||
artifacts.append(ref)
|
||||
|
||||
return NodeResult(
|
||||
data=data,
|
||||
messages=[m for m in (outcome.get("messages") or []) if isinstance(m, dict)],
|
||||
artifacts=artifacts,
|
||||
metadata={
|
||||
"agentId": agent_id,
|
||||
"threadId": outcome.get("thread_id"),
|
||||
"usage": outcome.get("usage") or {},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
class AgentNodeExecutor(_AgentInvocation):
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
|
||||
if not agent_id:
|
||||
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点未选择智能体", node_id=node.id)
|
||||
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
|
||||
prompt = render_template(template, ctx.state()).strip()
|
||||
if not prompt:
|
||||
raise WorkflowError("WORKFLOW_AGENT_FAILED", "智能体节点提示词为空", node_id=node.id)
|
||||
schema = node.config.get("responseSchema") or node.config.get("response_schema") or {}
|
||||
return await self._invoke(
|
||||
node,
|
||||
ctx,
|
||||
agent_id=agent_id,
|
||||
prompt=prompt,
|
||||
skill_names=[str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])],
|
||||
response_mode=str(node.config.get("responseMode") or node.config.get("response_mode") or "text"),
|
||||
response_schema=schema if isinstance(schema, dict) else {},
|
||||
thread_mode=str(node.config.get("threadMode") or node.config.get("thread_mode") or "isolated_per_node"),
|
||||
failure_code="WORKFLOW_AGENT_FAILED",
|
||||
)
|
||||
|
||||
|
||||
class SkillNodeExecutor(_AgentInvocation):
|
||||
"""``agent_skill`` runs the agent with a restricted skill set.
|
||||
``callable_skill`` (direct script invocation) is not executable yet."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
mode = str(node.config.get("mode") or "agent_skill")
|
||||
if mode != "agent_skill":
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SKILL_FAILED",
|
||||
"callable_skill 模式暂未开放执行,请改用 agent_skill",
|
||||
node_id=node.id,
|
||||
details={"mode": mode},
|
||||
)
|
||||
agent_id = str(node.config.get("agentId") or node.config.get("agent_id") or "")
|
||||
skill_names = [str(s) for s in (node.config.get("skillNames") or node.config.get("skill_names") or [])]
|
||||
if not agent_id or not skill_names:
|
||||
raise WorkflowError("WORKFLOW_SKILL_FAILED", "技能节点需要同时指定智能体与技能", node_id=node.id)
|
||||
template = str(node.config.get("promptTemplate") or node.config.get("prompt_template") or "")
|
||||
prompt = render_template(template, ctx.state()).strip() or f"请使用技能 {skill_names[0]} 完成任务。"
|
||||
return await self._invoke(
|
||||
node,
|
||||
ctx,
|
||||
agent_id=agent_id,
|
||||
prompt=prompt,
|
||||
skill_names=skill_names,
|
||||
response_mode="text",
|
||||
response_schema={},
|
||||
thread_mode="isolated_per_node",
|
||||
failure_code="WORKFLOW_SKILL_FAILED",
|
||||
)
|
||||
|
||||
|
||||
class SubworkflowNodeExecutor:
|
||||
"""Runs a published child workflow synchronously and folds in its output."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
run_subworkflow = require_dep(ctx.deps.run_subworkflow, node=node, what="子工作流执行")
|
||||
workflow_id = str(node.config.get("workflowId") or node.config.get("workflow_id") or "")
|
||||
version_id = str(node.config.get("versionId") or node.config.get("version_id") or "")
|
||||
if not workflow_id or not version_id:
|
||||
raise WorkflowError("WORKFLOW_SUBWORKFLOW_FAILED", "子工作流节点缺少 workflowId/versionId", node_id=node.id)
|
||||
mapping = node.config.get("inputMapping") or node.config.get("input_mapping") or {}
|
||||
state = ctx.state()
|
||||
inputs = {str(key): render_value(template, state) for key, template in mapping.items()} if isinstance(mapping, dict) else {}
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={"phase": "subworkflow.start", "workflowId": workflow_id, "versionId": version_id},
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
outcome = await run_subworkflow(
|
||||
parent_run_id=ctx.run_id,
|
||||
node_id=node.id,
|
||||
owner_id=ctx.owner_id,
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
inputs=inputs,
|
||||
cancel=ctx.cancel,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
"子工作流执行失败",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
status = str(outcome.get("status") or "completed")
|
||||
if status != "completed":
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SUBWORKFLOW_FAILED",
|
||||
f"子工作流未成功完成({status})",
|
||||
node_id=node.id,
|
||||
details={"childRunId": outcome.get("run_id"), "status": status},
|
||||
)
|
||||
output = outcome.get("output")
|
||||
return NodeResult(
|
||||
data=output if isinstance(output, dict) else {"value": output},
|
||||
metadata={"childRunId": outcome.get("run_id"), "workflowId": workflow_id},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["AgentNodeExecutor", "SkillNodeExecutor", "SubworkflowNodeExecutor"]
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Node executor protocol and registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.ports import ACTIVE_PORTS
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.schemas import NodeResult, NodeType, WorkflowNode
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class NodeExecutor(Protocol):
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult: ...
|
||||
|
||||
|
||||
class NodeRegistry:
|
||||
"""Maps node type → executor. Unknown types fail with a typed error."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._executors: dict[str, NodeExecutor] = {}
|
||||
|
||||
def register(self, node_type: NodeType | str, executor: NodeExecutor) -> None:
|
||||
self._executors[str(node_type)] = executor
|
||||
|
||||
def get(self, node_type: str) -> NodeExecutor:
|
||||
executor = self._executors.get(node_type)
|
||||
if executor is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
f"节点类型暂不支持执行:{node_type}",
|
||||
details={"nodeType": node_type},
|
||||
)
|
||||
return executor
|
||||
|
||||
def supported(self) -> list[str]:
|
||||
return sorted(self._executors)
|
||||
|
||||
|
||||
def require_dep(value: Any, *, node: WorkflowNode, what: str) -> Any:
|
||||
if value is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
f"运行环境缺少{what}能力,无法执行该节点",
|
||||
node_id=node.id,
|
||||
details={"capability": what},
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
__all__ = ["ACTIVE_PORTS", "NodeExecutor", "NodeRegistry", "require_dep"]
|
||||
@@ -0,0 +1,238 @@
|
||||
"""Deterministic control / data nodes: start, output, transform, condition, merge, loop."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import evaluate_condition, render_template, render_value
|
||||
from deerflow.workflows.nodes.base import ACTIVE_PORTS
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.schema_validation import schema_issues
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowNode
|
||||
|
||||
DEFAULT_PORT = "else"
|
||||
LOOP_CONTINUE_PORT = "continue"
|
||||
LOOP_DONE_PORT = "done"
|
||||
|
||||
|
||||
def _required_keys(schema: dict[str, Any]) -> list[str]:
|
||||
required = schema.get("required")
|
||||
if isinstance(required, list):
|
||||
return [str(k) for k in required]
|
||||
return []
|
||||
|
||||
|
||||
class StartNodeExecutor:
|
||||
"""Validates run inputs against the graph input schema and exposes them."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
schema = ctx.graph.input_schema or {}
|
||||
missing = [key for key in _required_keys(schema) if ctx.inputs.get(key) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
f"缺少必填输入:{'、'.join(missing)}",
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
# Full JSON Schema pass (types / enums / nested objects) beyond the
|
||||
# friendly missing-key check above.
|
||||
issues = schema_issues(ctx.inputs, schema)
|
||||
if issues:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_INPUT_INVALID",
|
||||
"输入不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
|
||||
node_id=node.id,
|
||||
details={"errors": issues},
|
||||
)
|
||||
declared = node.config.get("fields")
|
||||
data = dict(ctx.inputs)
|
||||
if isinstance(declared, list) and declared:
|
||||
names = {str(f.get("name")) for f in declared if isinstance(f, dict) and f.get("name")}
|
||||
data = {k: v for k, v in data.items() if k in names} or data
|
||||
return NodeResult(data=data)
|
||||
|
||||
|
||||
class OutputNodeExecutor:
|
||||
"""Projects the final run output. Empty mapping = merge all upstream data."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
mapping = node.config.get("mapping") or {}
|
||||
state = ctx.state()
|
||||
if isinstance(mapping, dict) and mapping:
|
||||
data = {str(key): render_value(template, state) for key, template in mapping.items()}
|
||||
else:
|
||||
data = {}
|
||||
for result in ctx.node_results.values():
|
||||
payload = result.get("data")
|
||||
if isinstance(payload, dict):
|
||||
data.update(payload)
|
||||
schema = ctx.graph.output_schema or {}
|
||||
missing = [key for key in _required_keys(schema) if data.get(key) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
f"输出缺少必填字段:{'、'.join(missing)}",
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
issues = schema_issues(data, schema)
|
||||
if issues:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_OUTPUT_SCHEMA_MISMATCH",
|
||||
"输出不符合 schema:" + ";".join(i["message"] for i in issues[:3]),
|
||||
node_id=node.id,
|
||||
details={"errors": issues},
|
||||
)
|
||||
return NodeResult(data=data, metadata={"isRunOutput": True})
|
||||
|
||||
|
||||
class TransformNodeExecutor:
|
||||
"""Pure data reshaping. Supported ops: set / pick / omit / json_parse / join."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
state = ctx.state()
|
||||
data: dict[str, Any] = {}
|
||||
template = node.config.get("template")
|
||||
if isinstance(template, str) and template:
|
||||
data["text"] = render_template(template, state)
|
||||
|
||||
for raw in node.config.get("operations") or []:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
op = str(raw.get("op") or "set")
|
||||
target = str(raw.get("target") or raw.get("name") or "value")
|
||||
try:
|
||||
if op == "set":
|
||||
data[target] = render_value(raw.get("value"), state)
|
||||
elif op == "pick":
|
||||
source = render_value(raw.get("source"), state)
|
||||
keys = [str(k) for k in (raw.get("keys") or [])]
|
||||
data[target] = {k: source.get(k) for k in keys} if isinstance(source, dict) else None
|
||||
elif op == "omit":
|
||||
source = render_value(raw.get("source"), state)
|
||||
keys = {str(k) for k in (raw.get("keys") or [])}
|
||||
data[target] = {k: v for k, v in source.items() if k not in keys} if isinstance(source, dict) else None
|
||||
elif op == "json_parse":
|
||||
text = render_value(raw.get("source"), state)
|
||||
data[target] = json.loads(text) if isinstance(text, str) and text.strip() else None
|
||||
elif op == "join":
|
||||
items = render_value(raw.get("source"), state)
|
||||
separator = str(raw.get("separator") or "\n")
|
||||
data[target] = separator.join(str(i) for i in items) if isinstance(items, list) else ""
|
||||
else:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
f"不支持的转换操作:{op}",
|
||||
node_id=node.id,
|
||||
details={"op": op},
|
||||
)
|
||||
except WorkflowError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - surface as typed node failure
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TRANSFORM_FAILED",
|
||||
f"转换操作 {op} 执行失败",
|
||||
node_id=node.id,
|
||||
details={"op": op, "target": target, "reason": type(exc).__name__},
|
||||
) from exc
|
||||
return NodeResult(data=data)
|
||||
|
||||
|
||||
class ConditionNodeExecutor:
|
||||
"""Picks the first matching branch; activates only that outgoing port."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
state = ctx.state()
|
||||
branches = node.config.get("branches") or []
|
||||
if not isinstance(branches, list) or not branches:
|
||||
raise WorkflowError("WORKFLOW_CONDITION_FAILED", "条件节点未配置分支", node_id=node.id)
|
||||
evaluated: list[dict[str, Any]] = []
|
||||
chosen: str | None = None
|
||||
for raw in branches:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
name = str(raw.get("name") or "")
|
||||
expression = str(raw.get("expression") or "")
|
||||
matched = evaluate_condition(expression, state)
|
||||
evaluated.append({"name": name, "matched": matched})
|
||||
if matched and chosen is None:
|
||||
chosen = name
|
||||
if chosen is None:
|
||||
chosen = str(node.config.get("defaultBranch") or node.config.get("default_branch") or DEFAULT_PORT)
|
||||
return NodeResult(
|
||||
data={"branch": chosen, "evaluated": evaluated},
|
||||
metadata={ACTIVE_PORTS: [chosen]},
|
||||
)
|
||||
|
||||
|
||||
class MergeNodeExecutor:
|
||||
"""Collects upstream results per strategy (all / first_success / any_success)."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
strategy = str(node.config.get("strategy") or "all")
|
||||
sources = [str(s) for s in (node.config.get("sources") or [])]
|
||||
if not sources:
|
||||
sources = [edge.source for edge in ctx.graph.edges if edge.target == node.id]
|
||||
collected: dict[str, Any] = {}
|
||||
succeeded: list[str] = []
|
||||
for source in sources:
|
||||
result = ctx.node_results.get(source)
|
||||
if result is None:
|
||||
continue
|
||||
collected[source] = result.get("data")
|
||||
succeeded.append(source)
|
||||
if strategy in ("first_success", "any_success") and not succeeded:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_MERGE_FAILED",
|
||||
"合并节点没有任何成功的上游结果",
|
||||
node_id=node.id,
|
||||
details={"strategy": strategy, "sources": sources},
|
||||
)
|
||||
if strategy == "first_success":
|
||||
first = succeeded[0]
|
||||
return NodeResult(data={"selected": first, "value": collected[first]})
|
||||
return NodeResult(data={"branches": collected, "succeeded": succeeded})
|
||||
|
||||
|
||||
class LoopNodeExecutor:
|
||||
"""Iteration gate. Activates ``continue`` (back-edge) or ``done``.
|
||||
|
||||
Iteration accounting lives in ``ctx.loop_state`` so the cap is enforced even
|
||||
across a resume, and the engine invalidates body node results before the
|
||||
next pass.
|
||||
"""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
iteration = int(ctx.loop_state.get(node.id, 0))
|
||||
configured = int(node.config.get("maxIterations") or node.config.get("max_iterations") or 1)
|
||||
cap = min(configured, ctx.graph.settings.max_loop_iterations)
|
||||
expression = str(node.config.get("continueWhen") or node.config.get("continue_when") or "")
|
||||
should_continue = bool(expression) and evaluate_condition(expression, ctx.state())
|
||||
next_iteration = iteration + 1
|
||||
if should_continue and next_iteration >= cap:
|
||||
should_continue = False
|
||||
port = LOOP_CONTINUE_PORT if should_continue else LOOP_DONE_PORT
|
||||
if should_continue:
|
||||
ctx.loop_state[node.id] = next_iteration
|
||||
return NodeResult(
|
||||
data={
|
||||
"iteration": iteration,
|
||||
"nextIteration": next_iteration if should_continue else iteration,
|
||||
"maxIterations": cap,
|
||||
"continue": should_continue,
|
||||
},
|
||||
metadata={ACTIVE_PORTS: [port]},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ConditionNodeExecutor",
|
||||
"LoopNodeExecutor",
|
||||
"MergeNodeExecutor",
|
||||
"OutputNodeExecutor",
|
||||
"StartNodeExecutor",
|
||||
"TransformNodeExecutor",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""``human_input`` node: suspends the run and validates the resumed payload."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import render_template
|
||||
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowNode
|
||||
|
||||
DEFAULT_ACTIONS = ("submit",)
|
||||
|
||||
|
||||
def _form_schema(node: WorkflowNode) -> dict[str, Any]:
|
||||
schema = node.config.get("formSchema") or node.config.get("form_schema") or {}
|
||||
return schema if isinstance(schema, dict) else {}
|
||||
|
||||
|
||||
def _actions(node: WorkflowNode) -> list[str]:
|
||||
raw = node.config.get("actions")
|
||||
actions = [str(a) for a in raw] if isinstance(raw, list) and raw else list(DEFAULT_ACTIONS)
|
||||
return actions
|
||||
|
||||
|
||||
class HumanInputNodeExecutor:
|
||||
"""First pass raises :class:`WorkflowPaused`; the resume pass consumes the
|
||||
payload the API stored on the context and validates it against the form."""
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
payload = ctx.resume_payload
|
||||
if not payload or str(payload.get("nodeId") or "") != node.id:
|
||||
raise WorkflowPaused(
|
||||
node.id,
|
||||
{
|
||||
"nodeId": node.id,
|
||||
"name": node.name,
|
||||
"prompt": render_template(str(node.config.get("prompt") or ""), ctx.state()),
|
||||
"formSchema": _form_schema(node),
|
||||
"actions": _actions(node),
|
||||
},
|
||||
)
|
||||
|
||||
action = str(payload.get("action") or "submit")
|
||||
allowed = _actions(node)
|
||||
if action not in allowed:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
f"不支持的操作:{action}",
|
||||
node_id=node.id,
|
||||
details={"allowed": allowed},
|
||||
)
|
||||
values = payload.get("values")
|
||||
values = values if isinstance(values, dict) else {}
|
||||
schema = _form_schema(node)
|
||||
required = schema.get("required")
|
||||
if isinstance(required, list) and action != "cancel":
|
||||
missing = [str(key) for key in required if values.get(str(key)) in (None, "")]
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HUMAN_INPUT_INVALID",
|
||||
f"缺少必填项:{'、'.join(missing)}",
|
||||
node_id=node.id,
|
||||
details={"missing": missing},
|
||||
)
|
||||
if action == "cancel":
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CANCELLED",
|
||||
"用户在人工节点选择终止运行",
|
||||
node_id=node.id,
|
||||
)
|
||||
# One-shot: a later human node in the same run must pause again.
|
||||
ctx.resume_payload = None
|
||||
return NodeResult(data={"action": action, "values": values})
|
||||
|
||||
|
||||
__all__ = ["HumanInputNodeExecutor"]
|
||||
+276
@@ -0,0 +1,276 @@
|
||||
"""External-effect nodes: ``http``, ``sql_read``, ``code``."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowConfig
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.expressions import render_template, render_value
|
||||
from deerflow.workflows.runtime.code_runner import run_python_snippet
|
||||
from deerflow.workflows.runtime.context import RunContext
|
||||
from deerflow.workflows.runtime.sql_runner import execute_read_query
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowNode
|
||||
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
|
||||
from deerflow.workflows.security.sql_policy import (
|
||||
assert_parameters_bound,
|
||||
assert_read_only,
|
||||
assert_tables_allowed,
|
||||
enforce_limit,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Response headers safe to echo back into the event log.
|
||||
_SAFE_RESPONSE_HEADERS = ("content-type", "content-length", "date", "etag", "location")
|
||||
_REDACTED_REQUEST_HEADERS = ("authorization", "cookie", "x-api-key", "proxy-authorization")
|
||||
|
||||
|
||||
class HttpNodeExecutor:
|
||||
def __init__(self, config: WorkflowConfig) -> None:
|
||||
self._config = config
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
policy = self._config.http
|
||||
state = ctx.state()
|
||||
url = render_template(str(node.config.get("url") or ""), state).strip()
|
||||
if not url:
|
||||
raise WorkflowError("WORKFLOW_HTTP_FAILED", "HTTP 节点缺少 URL", node_id=node.id)
|
||||
method = str(node.config.get("method") or "GET").upper()
|
||||
headers = {str(k): str(v) for k, v in (render_value(node.config.get("headers") or {}, state) or {}).items()}
|
||||
query = render_value(node.config.get("query") or {}, state) or {}
|
||||
body = render_value(node.config.get("body"), state)
|
||||
|
||||
credential_ref = node.config.get("credentialRef") or node.config.get("credential_ref")
|
||||
if credential_ref:
|
||||
headers.update(await self._credential_headers(str(credential_ref), node, ctx))
|
||||
|
||||
try:
|
||||
target = await enforce_http_target(url, policy)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
|
||||
timeout = min(
|
||||
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
|
||||
policy.timeout_seconds,
|
||||
)
|
||||
max_bytes = min(
|
||||
int(node.config.get("maxResponseBytes") or node.config.get("max_response_bytes") or policy.max_response_bytes),
|
||||
policy.max_response_bytes,
|
||||
)
|
||||
allow_redirects = bool(node.config.get("allowRedirects") or node.config.get("allow_redirects"))
|
||||
max_hops = policy.max_redirects if allow_redirects else 0
|
||||
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={
|
||||
"phase": "http.request",
|
||||
"method": method,
|
||||
"host": target["host"],
|
||||
"headers": [k for k in headers if k.lower() not in _REDACTED_REQUEST_HEADERS],
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
|
||||
import httpx
|
||||
|
||||
current_url = url
|
||||
hops = 0
|
||||
raw = b""
|
||||
truncated = False
|
||||
async with httpx.AsyncClient(timeout=timeout, follow_redirects=False) as client:
|
||||
while True:
|
||||
ctx.cancel.raise_if_cancelled(node.id)
|
||||
try:
|
||||
request = client.build_request(
|
||||
method,
|
||||
current_url,
|
||||
headers=headers,
|
||||
params=query or None,
|
||||
json=body if isinstance(body, (dict, list)) else None,
|
||||
content=body if isinstance(body, (str, bytes)) else None,
|
||||
)
|
||||
response = await client.send(request, stream=True)
|
||||
except httpx.HTTPError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"HTTP 请求失败",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
if response.status_code in (301, 302, 303, 307, 308) and hops < max_hops:
|
||||
location = response.headers.get("location") or ""
|
||||
await response.aclose()
|
||||
if not location:
|
||||
break
|
||||
current_url = await enforce_redirect(location, current_url, policy)
|
||||
hops += 1
|
||||
continue
|
||||
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
try:
|
||||
async for chunk in response.aiter_bytes():
|
||||
size += len(chunk)
|
||||
if size > max_bytes:
|
||||
truncated = True
|
||||
break
|
||||
chunks.append(chunk)
|
||||
finally:
|
||||
await response.aclose()
|
||||
raw = b"".join(chunks)
|
||||
break
|
||||
|
||||
text_body = raw.decode("utf-8", errors="replace")
|
||||
parsed: Any = None
|
||||
content_type = response.headers.get("content-type", "")
|
||||
if "json" in content_type.lower():
|
||||
try:
|
||||
parsed = json.loads(text_body)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
|
||||
data = {
|
||||
"status": response.status_code,
|
||||
"ok": 200 <= response.status_code < 300,
|
||||
"headers": {k: v for k, v in response.headers.items() if k.lower() in _SAFE_RESPONSE_HEADERS},
|
||||
"json": parsed,
|
||||
"text": None if parsed is not None else text_body,
|
||||
"truncated": truncated,
|
||||
"redirects": hops,
|
||||
}
|
||||
if not data["ok"]:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
f"HTTP 请求返回 {response.status_code}",
|
||||
retryable=response.status_code >= 500 or response.status_code == 429,
|
||||
node_id=node.id,
|
||||
details={"status": response.status_code, "preview": text_body[:500]},
|
||||
)
|
||||
return NodeResult(data=data, metadata={"host": target["host"]})
|
||||
|
||||
async def _credential_headers(self, ref: str, node: WorkflowNode, ctx: RunContext) -> dict[str, str]:
|
||||
if ctx.deps.resolve_credential is None:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"运行环境未配置凭据解析能力",
|
||||
node_id=node.id,
|
||||
)
|
||||
credential = await ctx.deps.resolve_credential(ref)
|
||||
if not credential:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"引用的凭据不存在或无权访问",
|
||||
node_id=node.id,
|
||||
details={"credentialRef": ref},
|
||||
)
|
||||
headers = credential.get("headers")
|
||||
return {str(k): str(v) for k, v in headers.items()} if isinstance(headers, dict) else {}
|
||||
|
||||
|
||||
class SqlReadNodeExecutor:
|
||||
def __init__(self, config: WorkflowConfig) -> None:
|
||||
self._config = config
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
policy = self._config.sql
|
||||
if not policy.enabled:
|
||||
raise WorkflowError("WORKFLOW_SQL_POLICY_DENIED", "SQL 节点已被系统禁用", node_id=node.id)
|
||||
data_source_id = str(node.config.get("dataSourceId") or node.config.get("data_source_id") or "")
|
||||
if not data_source_id:
|
||||
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 节点未选择数据源", node_id=node.id)
|
||||
if ctx.deps.resolve_data_source is None:
|
||||
raise WorkflowError("WORKFLOW_RESOURCE_MISSING", "运行环境未配置数据源解析能力", node_id=node.id)
|
||||
source = await ctx.deps.resolve_data_source(data_source_id)
|
||||
if not source or not source.get("dsn"):
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_RESOURCE_MISSING",
|
||||
"数据源不存在或无权访问",
|
||||
node_id=node.id,
|
||||
details={"dataSourceId": data_source_id},
|
||||
)
|
||||
|
||||
statement = assert_read_only(str(node.config.get("statement") or ""))
|
||||
parameters = render_value(node.config.get("parameters") or {}, ctx.state()) or {}
|
||||
if not isinstance(parameters, dict):
|
||||
raise WorkflowError("WORKFLOW_SQL_FAILED", "SQL 参数必须是对象", node_id=node.id)
|
||||
assert_parameters_bound(statement, parameters)
|
||||
assert_tables_allowed(statement, list(source.get("allowed_tables") or []))
|
||||
max_rows = min(
|
||||
int(node.config.get("maxRows") or node.config.get("max_rows") or policy.max_rows),
|
||||
policy.max_rows,
|
||||
int(source.get("max_rows") or policy.max_rows),
|
||||
)
|
||||
statement = enforce_limit(statement, max_rows)
|
||||
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={"phase": "sql.query", "dataSourceId": data_source_id, "maxRows": max_rows},
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
result = await execute_read_query(
|
||||
dsn=str(source["dsn"]),
|
||||
statement=statement,
|
||||
parameters={str(k): v for k, v in parameters.items()},
|
||||
max_rows=max_rows,
|
||||
timeout_seconds=policy.statement_timeout_seconds,
|
||||
max_cell_chars=policy.max_cell_chars,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
return NodeResult(data=result)
|
||||
|
||||
|
||||
class CodeNodeExecutor:
|
||||
def __init__(self, config: WorkflowConfig) -> None:
|
||||
self._config = config
|
||||
|
||||
async def execute(self, node: WorkflowNode, ctx: RunContext) -> NodeResult:
|
||||
policy = self._config.code
|
||||
if not policy.enabled:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"代码节点未启用(需管理员在 config.yaml 中开启 workflows.code.enabled)",
|
||||
node_id=node.id,
|
||||
)
|
||||
source = str(node.config.get("source") or "")
|
||||
if not source.strip():
|
||||
raise WorkflowError("WORKFLOW_CODE_SANDBOX_FAILED", "代码节点没有源代码", node_id=node.id)
|
||||
if len(source) > policy.max_source_chars:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LIMIT_EXCEEDED",
|
||||
f"代码长度超过上限 {policy.max_source_chars}",
|
||||
node_id=node.id,
|
||||
)
|
||||
inputs = render_value(node.config.get("inputs") or {}, ctx.state()) or {}
|
||||
timeout = min(
|
||||
int(node.config.get("timeoutSeconds") or node.config.get("timeout_seconds") or policy.timeout_seconds),
|
||||
policy.timeout_seconds,
|
||||
)
|
||||
allow_network = bool((node.config.get("allowNetwork") or node.config.get("allow_network")) and policy.allow_network)
|
||||
try:
|
||||
outcome = await run_python_snippet(
|
||||
source=source,
|
||||
inputs=inputs if isinstance(inputs, dict) else {"value": inputs},
|
||||
timeout_seconds=timeout,
|
||||
max_output_chars=policy.max_output_chars,
|
||||
memory_limit_mb=policy.memory_limit_mb,
|
||||
allow_network=allow_network,
|
||||
)
|
||||
except WorkflowError as exc:
|
||||
exc.node_id = exc.node_id or node.id
|
||||
raise
|
||||
return NodeResult(
|
||||
data=outcome["result"],
|
||||
metadata={"stdout": outcome["stdout"][-4000:], "stderr": outcome["stderr"][-2000:]},
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["CodeNodeExecutor", "HttpNodeExecutor", "SqlReadNodeExecutor"]
|
||||
@@ -0,0 +1,17 @@
|
||||
"""Leaf constants shared by the scheduler and node executors.
|
||||
|
||||
Kept dependency-free so the engine and the node package can both import it
|
||||
without forming a cycle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# Metadata key a node sets to restrict which outgoing ports the engine activates.
|
||||
ACTIVE_PORTS = "activePorts"
|
||||
|
||||
# Reserved port names.
|
||||
LOOP_CONTINUE_PORT = "continue"
|
||||
LOOP_DONE_PORT = "done"
|
||||
CONDITION_DEFAULT_PORT = "else"
|
||||
|
||||
__all__ = ["ACTIVE_PORTS", "CONDITION_DEFAULT_PORT", "LOOP_CONTINUE_PORT", "LOOP_DONE_PORT"]
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
"""Workflow execution runtime (scheduler, context, event sink)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.workflows.runtime.context import (
|
||||
CancelToken,
|
||||
RunContext,
|
||||
WorkflowPaused,
|
||||
WorkflowRuntimeDeps,
|
||||
)
|
||||
from deerflow.workflows.runtime.engine import EngineResult, WorkflowEngine
|
||||
from deerflow.workflows.runtime.sink import PersistingWorkflowEventSink, envelope_from_row
|
||||
|
||||
__all__ = [
|
||||
"CancelToken",
|
||||
"EngineResult",
|
||||
"PersistingWorkflowEventSink",
|
||||
"RunContext",
|
||||
"WorkflowEngine",
|
||||
"WorkflowPaused",
|
||||
"WorkflowRuntimeDeps",
|
||||
"envelope_from_row",
|
||||
]
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
"""Constrained Python execution for the ``code`` node.
|
||||
|
||||
The snippet runs in a **separate interpreter process** with:
|
||||
|
||||
* a scrubbed environment (no API keys, tokens, or DSNs inherited),
|
||||
* a fresh temporary working directory that is deleted afterwards,
|
||||
* an address-space cap and no core dumps (POSIX ``setrlimit``),
|
||||
* sockets disabled unless the node explicitly allows network,
|
||||
* a hard wall-clock timeout with process-group kill,
|
||||
* bounded stdout/stderr capture.
|
||||
|
||||
This is process isolation, not a container: a determined snippet can still read
|
||||
world-readable files on the host. That is why ``workflows.code.enabled``
|
||||
defaults to ``false`` — turn it on only where the container sandbox or an OS
|
||||
level jail already confines the backend process.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_RUNNER_TEMPLATE = """
|
||||
import json, os, sys
|
||||
|
||||
_ALLOW_NETWORK = {allow_network}
|
||||
if not _ALLOW_NETWORK:
|
||||
import socket
|
||||
|
||||
def _blocked(*_args, **_kwargs):
|
||||
raise PermissionError("network access is disabled for this code node")
|
||||
|
||||
socket.socket = _blocked
|
||||
socket.create_connection = _blocked
|
||||
socket.getaddrinfo = _blocked
|
||||
|
||||
try:
|
||||
import resource
|
||||
|
||||
_limit = {memory_bytes}
|
||||
resource.setrlimit(resource.RLIMIT_AS, (_limit, _limit))
|
||||
resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
inputs = json.loads(os.environ.get("WORKFLOW_NODE_INPUT") or "{{}}")
|
||||
output = {{}}
|
||||
_globals = {{"__name__": "__workflow_code__", "inputs": inputs, "output": output}}
|
||||
|
||||
with open({source_path!r}, "r", encoding="utf-8") as handle:
|
||||
_source = handle.read()
|
||||
|
||||
exec(compile(_source, "<workflow-code>", "exec"), _globals)
|
||||
|
||||
_result = _globals.get("output")
|
||||
if not isinstance(_result, dict):
|
||||
_result = {{"value": _result}}
|
||||
sys.stdout.write("\\n__WORKFLOW_RESULT__" + json.dumps(_result, ensure_ascii=False, default=str))
|
||||
"""
|
||||
|
||||
|
||||
def _safe_env() -> dict[str, str]:
|
||||
env = {
|
||||
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||
"LANG": os.environ.get("LANG", "C.UTF-8"),
|
||||
"PYTHONIOENCODING": "utf-8",
|
||||
"PYTHONDONTWRITEBYTECODE": "1",
|
||||
"HOME": "",
|
||||
}
|
||||
if sys.platform == "win32":
|
||||
for key in ("SYSTEMROOT", "COMSPEC", "TEMP", "TMP"):
|
||||
if os.environ.get(key):
|
||||
env[key] = os.environ[key]
|
||||
return env
|
||||
|
||||
|
||||
async def run_python_snippet(
|
||||
*,
|
||||
source: str,
|
||||
inputs: dict[str, Any],
|
||||
timeout_seconds: int,
|
||||
max_output_chars: int,
|
||||
memory_limit_mb: int,
|
||||
allow_network: bool,
|
||||
) -> dict[str, Any]:
|
||||
workdir = Path(tempfile.mkdtemp(prefix="wf-code-"))
|
||||
try:
|
||||
source_path = workdir / "snippet.py"
|
||||
source_path.write_text(source, encoding="utf-8")
|
||||
runner_path = workdir / "runner.py"
|
||||
runner_path.write_text(
|
||||
_RUNNER_TEMPLATE.format(
|
||||
allow_network=bool(allow_network),
|
||||
memory_bytes=max(64, int(memory_limit_mb)) * 1024 * 1024,
|
||||
source_path=str(source_path),
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
env = _safe_env()
|
||||
env["WORKFLOW_NODE_INPUT"] = json.dumps(inputs, ensure_ascii=False, default=str)
|
||||
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable,
|
||||
"-I",
|
||||
"-S",
|
||||
str(runner_path),
|
||||
cwd=str(workdir),
|
||||
env=env,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
start_new_session=sys.platform != "win32",
|
||||
)
|
||||
try:
|
||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(process.communicate(), timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
_terminate(process)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"代码节点执行超时({timeout_seconds}s)",
|
||||
details={"timeoutSeconds": timeout_seconds},
|
||||
) from exc
|
||||
|
||||
stdout = stdout_bytes.decode("utf-8", errors="replace")
|
||||
stderr = stderr_bytes.decode("utf-8", errors="replace")[:max_output_chars]
|
||||
if process.returncode != 0:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_CODE_SANDBOX_FAILED",
|
||||
"代码节点执行失败",
|
||||
details={"exitCode": process.returncode, "stderr": stderr[-2000:]},
|
||||
)
|
||||
|
||||
marker = "__WORKFLOW_RESULT__"
|
||||
index = stdout.rfind(marker)
|
||||
result: dict[str, Any] = {}
|
||||
logs = stdout
|
||||
if index >= 0:
|
||||
logs = stdout[:index]
|
||||
try:
|
||||
parsed = json.loads(stdout[index + len(marker) :])
|
||||
result = parsed if isinstance(parsed, dict) else {"value": parsed}
|
||||
except json.JSONDecodeError:
|
||||
result = {}
|
||||
return {
|
||||
"result": result,
|
||||
"stdout": logs[:max_output_chars],
|
||||
"stderr": stderr,
|
||||
}
|
||||
finally:
|
||||
shutil.rmtree(workdir, ignore_errors=True)
|
||||
|
||||
|
||||
def _terminate(process: asyncio.subprocess.Process) -> None:
|
||||
try:
|
||||
if sys.platform != "win32":
|
||||
os.killpg(os.getpgid(process.pid), 9)
|
||||
else:
|
||||
process.kill()
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
logger.debug("code node process already gone", exc_info=True)
|
||||
|
||||
|
||||
__all__ = ["run_python_snippet"]
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
"""Run-scoped execution context shared by the engine and node executors."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.events import WorkflowEventType
|
||||
from deerflow.workflows.schemas import ArtifactRef, WorkflowGraph, WorkflowNode
|
||||
|
||||
EmitFn = Callable[..., Awaitable[Any]]
|
||||
|
||||
|
||||
def _public_artifact_payload(ref: ArtifactRef) -> dict[str, Any]:
|
||||
"""Wire payload for ``artifact.created``.
|
||||
|
||||
The sandbox/host ``path`` stays internal: clients address artifacts by id
|
||||
through the run-scoped content endpoints, never by filesystem path."""
|
||||
from deerflow.workflows.events import artifact_kind
|
||||
|
||||
return {
|
||||
"artifactId": ref.artifact_id,
|
||||
"name": ref.name,
|
||||
"kind": artifact_kind(ref.mime_type),
|
||||
"mimeType": ref.mime_type,
|
||||
"sizeBytes": ref.size_bytes,
|
||||
"preview": ref.preview,
|
||||
}
|
||||
|
||||
|
||||
class WorkflowPaused(Exception): # noqa: N818 - control-flow signal, not an error
|
||||
"""Raised by ``human_input`` to suspend the run until a resume arrives."""
|
||||
|
||||
def __init__(self, node_id: str, pending_input: dict[str, Any]) -> None:
|
||||
super().__init__(f"workflow paused at {node_id}")
|
||||
self.node_id = node_id
|
||||
self.pending_input = pending_input
|
||||
|
||||
|
||||
class CancelToken:
|
||||
"""Cooperative cancellation shared between the executor and node code."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._event = asyncio.Event()
|
||||
self._reason = "cancelled"
|
||||
|
||||
def cancel(self, reason: str = "cancelled") -> None:
|
||||
self._reason = reason
|
||||
self._event.set()
|
||||
|
||||
@property
|
||||
def cancelled(self) -> bool:
|
||||
return self._event.is_set()
|
||||
|
||||
async def wait(self) -> None:
|
||||
await self._event.wait()
|
||||
|
||||
def raise_if_cancelled(self, node_id: str | None = None) -> None:
|
||||
if self._event.is_set():
|
||||
raise WorkflowError("WORKFLOW_CANCELLED", "运行已被取消", node_id=node_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
class WorkflowRuntimeDeps:
|
||||
"""Injected capabilities. The harness never imports the app layer, so every
|
||||
integration point (agents, HTTP egress, SQL, sandbox, subworkflows) arrives
|
||||
as an async callable supplied by the caller. ``None`` means "not available",
|
||||
which node executors turn into a typed ``WORKFLOW_RESOURCE_MISSING``."""
|
||||
|
||||
run_agent: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_http: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_sql: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_code: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
run_subworkflow: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
save_artifact: Callable[..., Awaitable[dict[str, Any]]] | None = None
|
||||
resolve_credential: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
|
||||
resolve_data_source: Callable[[str], Awaitable[dict[str, Any] | None]] | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunContext:
|
||||
"""Everything a node executor is allowed to see."""
|
||||
|
||||
run_id: str
|
||||
workflow_id: str
|
||||
version_id: str
|
||||
owner_id: str
|
||||
graph: WorkflowGraph
|
||||
inputs: dict[str, Any]
|
||||
deps: WorkflowRuntimeDeps
|
||||
cancel: CancelToken
|
||||
emit: EmitFn
|
||||
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
loop_state: dict[str, int] = field(default_factory=dict)
|
||||
resume_payload: dict[str, Any] | None = None
|
||||
node_run_ids: dict[str, str] = field(default_factory=dict)
|
||||
current_node_id: str | None = None
|
||||
env: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def state(self) -> dict[str, Any]:
|
||||
"""Whitelisted tree visible to expressions. Credentials never appear."""
|
||||
return {
|
||||
"inputs": self.inputs,
|
||||
"nodes": self.node_results,
|
||||
"run": {"id": self.run_id, "workflowId": self.workflow_id, "versionId": self.version_id},
|
||||
"loop": dict(self.loop_state),
|
||||
"env": self.env,
|
||||
}
|
||||
|
||||
async def emit_event(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
) -> None:
|
||||
node = node_id or self.current_node_id
|
||||
await self.emit(
|
||||
event_type,
|
||||
data=data or {},
|
||||
node_id=node,
|
||||
node_run_id=self.node_run_ids.get(node) if node else None,
|
||||
)
|
||||
|
||||
async def register_artifact(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
*,
|
||||
name: str,
|
||||
mime_type: str = "application/octet-stream",
|
||||
path: str = "",
|
||||
size_bytes: int | None = None,
|
||||
preview: str | None = None,
|
||||
) -> ArtifactRef | None:
|
||||
"""Persist an artifact reference and emit ``artifact.created``."""
|
||||
if self.deps.save_artifact is None:
|
||||
return None
|
||||
row = await self.deps.save_artifact(
|
||||
run_id=self.run_id,
|
||||
node_id=node.id,
|
||||
name=name,
|
||||
mime_type=mime_type,
|
||||
path=path,
|
||||
size_bytes=size_bytes,
|
||||
preview=preview,
|
||||
)
|
||||
ref = ArtifactRef(
|
||||
artifactId=str(row.get("id") or ""),
|
||||
name=str(row.get("name") or name),
|
||||
mimeType=str(row.get("mime_type") or mime_type),
|
||||
path=str(row.get("path") or path),
|
||||
sizeBytes=row.get("size_bytes"),
|
||||
preview=row.get("preview"),
|
||||
)
|
||||
await self.emit_event("artifact.created", data=_public_artifact_payload(ref), node_id=node.id)
|
||||
return ref
|
||||
|
||||
|
||||
__all__ = ["CancelToken", "RunContext", "WorkflowPaused", "WorkflowRuntimeDeps"]
|
||||
+531
@@ -0,0 +1,531 @@
|
||||
"""Persisted DAG scheduler for workflow runs.
|
||||
|
||||
Scheduling model
|
||||
----------------
|
||||
Every edge carries a state: ``pending`` → ``active`` (its source chose this
|
||||
port) or ``pruned`` (the source chose a different port). A node is *ready* when
|
||||
none of its non-back incoming edges is ``pending`` and at least one is
|
||||
``active``; a node whose incoming edges all resolved to ``pruned`` is *skipped*
|
||||
and prunes its own outgoing edges in turn. That single rule gives us fan-out
|
||||
parallelism, condition branching, and merge joins without a topological
|
||||
pre-pass, and it terminates because each superstep resolves at least one edge.
|
||||
|
||||
Loops are the one cycle we allow: the back edge (``loop`` node, port
|
||||
``continue``) is excluded from readiness gating, and activating it clears the
|
||||
body's results so the next iteration re-runs them. Every re-entry is capped by
|
||||
``settings.max_loop_iterations``.
|
||||
|
||||
Durability: node results are written to the run store as they complete, so a
|
||||
resumed or re-leased run replays only what is missing. ``human_input`` raises
|
||||
:class:`WorkflowPaused`, which the caller turns into ``awaiting_input``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import WorkflowRunStore
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
from deerflow.workflows.ports import ACTIVE_PORTS, LOOP_CONTINUE_PORT
|
||||
from deerflow.workflows.runtime.context import RunContext, WorkflowPaused
|
||||
from deerflow.workflows.schemas import NodeResult, WorkflowEdge, WorkflowGraph, WorkflowNode
|
||||
|
||||
|
||||
class ExecutorRegistry(Protocol):
|
||||
"""Structural view of ``deerflow.workflows.nodes.NodeRegistry``.
|
||||
|
||||
Declared here rather than imported so the scheduler has no dependency on
|
||||
the node package (which depends on the runtime context).
|
||||
"""
|
||||
|
||||
def get(self, node_type: str) -> Any: ...
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PENDING = "pending"
|
||||
ACTIVE = "active"
|
||||
PRUNED = "pruned"
|
||||
|
||||
|
||||
@dataclass
|
||||
class EngineResult:
|
||||
output: dict[str, Any] = field(default_factory=dict)
|
||||
node_results: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
loop_state: dict[str, int] = field(default_factory=dict)
|
||||
steps: int = 0
|
||||
|
||||
|
||||
def _is_back_edge(edge: WorkflowEdge, nodes: dict[str, WorkflowNode]) -> bool:
|
||||
source = nodes.get(edge.source)
|
||||
return bool(source and source.type == "loop" and (edge.source_port or "") == LOOP_CONTINUE_PORT)
|
||||
|
||||
|
||||
def _node_started_event_data(node: WorkflowNode, node_attempt: int) -> dict[str, Any]:
|
||||
"""Safe presentation metadata for workflow clients.
|
||||
|
||||
This stays intentionally small: UI clients need stable agent identity and
|
||||
an optional explicit parallel group, but never prompt templates, tools,
|
||||
credentials, or the full node config. A missing group is valid; clients
|
||||
may still use their backwards-compatible name-based presentation.
|
||||
"""
|
||||
|
||||
data: dict[str, Any] = {
|
||||
"nodeType": node.type,
|
||||
"nodeKind": node.type,
|
||||
"name": node.name,
|
||||
"attempt": node_attempt,
|
||||
}
|
||||
group_id = node.config.get("parallelGroupId") or node.config.get("parallel_group_id")
|
||||
if isinstance(group_id, str) and group_id.strip():
|
||||
data["parallelGroupId"] = group_id.strip()
|
||||
round_index = node.config.get("roundIndex")
|
||||
if round_index is None:
|
||||
round_index = node.config.get("round_index")
|
||||
if isinstance(round_index, int) and round_index >= 0:
|
||||
data["roundIndex"] = round_index
|
||||
|
||||
if node.type == "agent":
|
||||
agent_id = node.config.get("agentId") or node.config.get("agent_id")
|
||||
if isinstance(agent_id, str) and agent_id.strip():
|
||||
agent: dict[str, str] = {
|
||||
"agentId": agent_id.strip(),
|
||||
"name": str(
|
||||
node.config.get("agentName")
|
||||
or node.config.get("agent_name")
|
||||
or node.name
|
||||
or agent_id
|
||||
),
|
||||
}
|
||||
avatar_url = node.config.get("agentAvatarUrl") or node.config.get("agent_avatar_url")
|
||||
if isinstance(avatar_url, str) and avatar_url.strip():
|
||||
agent["avatarUrl"] = avatar_url.strip()
|
||||
data["agent"] = agent
|
||||
return data
|
||||
|
||||
|
||||
class WorkflowEngine:
|
||||
def __init__(self, registry: ExecutorRegistry, *, run_store: WorkflowRunStore | None = None) -> None:
|
||||
self._registry = registry
|
||||
self._store = run_store
|
||||
|
||||
async def run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
*,
|
||||
preloaded: dict[str, dict[str, Any]] | None = None,
|
||||
loop_state: dict[str, int] | None = None,
|
||||
attempt: int = 1,
|
||||
) -> EngineResult:
|
||||
graph: WorkflowGraph = ctx.graph
|
||||
nodes = graph.node_map()
|
||||
outgoing: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
|
||||
incoming: dict[str, list[WorkflowEdge]] = {node_id: [] for node_id in nodes}
|
||||
for edge in graph.edges:
|
||||
outgoing[edge.source].append(edge)
|
||||
incoming[edge.target].append(edge)
|
||||
|
||||
edge_state: dict[str, str] = {edge.id: PENDING for edge in graph.edges}
|
||||
done: set[str] = set()
|
||||
skipped: set[str] = set()
|
||||
|
||||
ctx.loop_state.update(loop_state or {})
|
||||
if preloaded:
|
||||
ctx.node_results.update(preloaded)
|
||||
done |= set(preloaded)
|
||||
# Re-derive edge states from the ports each replayed node selected.
|
||||
for node_id in list(done):
|
||||
self._apply_ports(
|
||||
node_id,
|
||||
ctx.node_results.get(node_id, {}).get("metadata") or {},
|
||||
outgoing,
|
||||
edge_state,
|
||||
)
|
||||
|
||||
semaphore = asyncio.Semaphore(max(1, graph.settings.max_parallelism))
|
||||
steps = 0
|
||||
max_steps = max(1, graph.settings.max_steps)
|
||||
|
||||
while True:
|
||||
ctx.cancel.raise_if_cancelled()
|
||||
self._propagate_skips(nodes, incoming, outgoing, edge_state, done, skipped)
|
||||
ready = [node_id for node_id in nodes if node_id not in done and node_id not in skipped and self._readiness(node_id, incoming, edge_state, nodes) == "ready"]
|
||||
if not ready:
|
||||
break
|
||||
|
||||
steps += len(ready)
|
||||
if steps > max_steps:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_MAX_STEPS",
|
||||
f"运行步数超过上限 {max_steps}",
|
||||
details={"maxSteps": max_steps},
|
||||
)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(self._run_one(nodes[node_id], ctx, semaphore, attempt) for node_id in ready),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
paused: WorkflowPaused | None = None
|
||||
failure: BaseException | None = None
|
||||
for node_id, outcome in zip(ready, results, strict=True):
|
||||
if isinstance(outcome, WorkflowPaused):
|
||||
paused = paused or outcome
|
||||
continue
|
||||
if isinstance(outcome, BaseException):
|
||||
failure = failure or outcome
|
||||
continue
|
||||
ctx.node_results[node_id] = outcome
|
||||
done.add(node_id)
|
||||
metadata = outcome.get("metadata") or {}
|
||||
self._apply_ports(node_id, metadata, outgoing, edge_state)
|
||||
if nodes[node_id].type == "loop":
|
||||
await self._maybe_reset_loop_body(nodes[node_id], ctx, nodes, outgoing, edge_state, done, skipped)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
if paused is not None:
|
||||
raise paused
|
||||
|
||||
return EngineResult(
|
||||
output=self._collect_output(nodes, ctx),
|
||||
node_results=dict(ctx.node_results),
|
||||
loop_state=dict(ctx.loop_state),
|
||||
steps=steps,
|
||||
)
|
||||
|
||||
# ── scheduling helpers ──────────────────────────────────────────────
|
||||
def _readiness(
|
||||
self,
|
||||
node_id: str,
|
||||
incoming: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
nodes: dict[str, WorkflowNode],
|
||||
) -> str:
|
||||
edges = incoming.get(node_id) or []
|
||||
if not edges:
|
||||
return "ready"
|
||||
back = [e for e in edges if _is_back_edge(e, nodes)]
|
||||
if any(edge_state[e.id] == ACTIVE for e in back):
|
||||
return "ready"
|
||||
normal = [e for e in edges if not _is_back_edge(e, nodes)]
|
||||
if not normal:
|
||||
return "blocked"
|
||||
if any(edge_state[e.id] == PENDING for e in normal):
|
||||
return "blocked"
|
||||
if any(edge_state[e.id] == ACTIVE for e in normal):
|
||||
return "ready"
|
||||
return "skip"
|
||||
|
||||
def _apply_ports(
|
||||
self,
|
||||
node_id: str,
|
||||
metadata: dict[str, Any],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
) -> None:
|
||||
active_ports = metadata.get(ACTIVE_PORTS)
|
||||
allowed = {str(p) for p in active_ports} if isinstance(active_ports, list) else None
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
port = edge.source_port or ""
|
||||
if allowed is None or not port or port in allowed:
|
||||
edge_state[edge.id] = ACTIVE
|
||||
else:
|
||||
edge_state[edge.id] = PRUNED
|
||||
|
||||
def _propagate_skips(
|
||||
self,
|
||||
nodes: dict[str, WorkflowNode],
|
||||
incoming: dict[str, list[WorkflowEdge]],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
done: set[str],
|
||||
skipped: set[str],
|
||||
) -> None:
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node_id in nodes:
|
||||
if node_id in done or node_id in skipped:
|
||||
continue
|
||||
if self._readiness(node_id, incoming, edge_state, nodes) != "skip":
|
||||
continue
|
||||
skipped.add(node_id)
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
edge_state[edge.id] = PRUNED
|
||||
changed = True
|
||||
|
||||
async def _maybe_reset_loop_body(
|
||||
self,
|
||||
loop_node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
nodes: dict[str, WorkflowNode],
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
edge_state: dict[str, str],
|
||||
done: set[str],
|
||||
skipped: set[str],
|
||||
) -> None:
|
||||
back_edges = [e for e in outgoing.get(loop_node.id) or [] if _is_back_edge(e, nodes) and edge_state[e.id] == ACTIVE]
|
||||
if not back_edges:
|
||||
return
|
||||
body_entry = str(loop_node.config.get("bodyEntry") or loop_node.config.get("body_entry") or "")
|
||||
if not body_entry or body_entry not in nodes:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_LOOP_FAILED",
|
||||
"循环节点的 bodyEntry 不存在",
|
||||
node_id=loop_node.id,
|
||||
details={"bodyEntry": body_entry},
|
||||
)
|
||||
body = self._loop_body(body_entry, loop_node.id, outgoing, nodes)
|
||||
for node_id in body:
|
||||
done.discard(node_id)
|
||||
skipped.discard(node_id)
|
||||
ctx.node_results.pop(node_id, None)
|
||||
for edge in outgoing.get(node_id) or []:
|
||||
if not _is_back_edge(edge, nodes):
|
||||
edge_state[edge.id] = PENDING
|
||||
# The back edge stays ACTIVE so the body entry is reachable again; the
|
||||
# loop node prunes it itself once it selects the ``done`` port.
|
||||
if self._store is not None and body:
|
||||
try:
|
||||
await self._store.invalidate_node_runs(ctx.run_id, sorted(body))
|
||||
except Exception: # noqa: BLE001 - bookkeeping must not kill the run
|
||||
logger.warning("failed to invalidate node runs for loop reset", exc_info=True)
|
||||
await ctx.emit_event(
|
||||
"node.progress",
|
||||
data={
|
||||
"phase": "loop.iteration",
|
||||
"iteration": ctx.loop_state.get(loop_node.id, 0),
|
||||
"bodyEntry": body_entry,
|
||||
},
|
||||
node_id=loop_node.id,
|
||||
)
|
||||
|
||||
def _loop_body(
|
||||
self,
|
||||
body_entry: str,
|
||||
loop_id: str,
|
||||
outgoing: dict[str, list[WorkflowEdge]],
|
||||
nodes: dict[str, WorkflowNode],
|
||||
) -> set[str]:
|
||||
"""Nodes reachable from ``body_entry`` up to (and including) the loop node."""
|
||||
body: set[str] = set()
|
||||
stack = [body_entry]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if current in body:
|
||||
continue
|
||||
body.add(current)
|
||||
if current == loop_id:
|
||||
continue
|
||||
for edge in outgoing.get(current) or []:
|
||||
if not _is_back_edge(edge, nodes):
|
||||
stack.append(edge.target)
|
||||
return body
|
||||
|
||||
# ── node execution ──────────────────────────────────────────────────
|
||||
async def _run_one(
|
||||
self,
|
||||
node: WorkflowNode,
|
||||
ctx: RunContext,
|
||||
semaphore: asyncio.Semaphore,
|
||||
attempt: int,
|
||||
) -> dict[str, Any]:
|
||||
async with semaphore:
|
||||
ctx.cancel.raise_if_cancelled(node.id)
|
||||
retry_cfg = node.config.get("retry") if isinstance(node.config.get("retry"), dict) else {}
|
||||
max_attempts = max(1, min(5, int(retry_cfg.get("maxAttempts") or 1)))
|
||||
backoff = max(0.0, float(retry_cfg.get("backoffSeconds") or 0.5))
|
||||
on_error = str(node.config.get("onError") or "fail")
|
||||
timeout = self._node_timeout(node, ctx)
|
||||
|
||||
last_error: WorkflowError | None = None
|
||||
for node_attempt in range(1, max_attempts + 1):
|
||||
node_run = await self._begin_node_run(ctx, node, attempt, node_attempt)
|
||||
started = time.monotonic()
|
||||
await ctx.emit_event(
|
||||
"node.started",
|
||||
data=_node_started_event_data(node, node_attempt),
|
||||
node_id=node.id,
|
||||
)
|
||||
try:
|
||||
executor = self._registry.get(node.type)
|
||||
result: NodeResult = await asyncio.wait_for(executor.execute(node, ctx), timeout=timeout)
|
||||
except WorkflowPaused:
|
||||
await self._finish_node_run(ctx, node_run, status="queued", output=None)
|
||||
raise
|
||||
except TimeoutError as exc:
|
||||
last_error = WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"节点执行超时({timeout}s)",
|
||||
retryable=True,
|
||||
node_id=node.id,
|
||||
details={"timeoutSeconds": timeout},
|
||||
)
|
||||
await self._fail_node_run(ctx, node_run, last_error, started)
|
||||
if node_attempt < max_attempts:
|
||||
await asyncio.sleep(backoff * node_attempt)
|
||||
continue
|
||||
if on_error == "continue":
|
||||
return self._error_result(last_error)
|
||||
raise last_error from exc
|
||||
except WorkflowError as exc:
|
||||
last_error = exc
|
||||
exc.node_id = exc.node_id or node.id
|
||||
await self._fail_node_run(ctx, node_run, exc, started)
|
||||
if exc.retryable and node_attempt < max_attempts:
|
||||
await asyncio.sleep(backoff * node_attempt)
|
||||
continue
|
||||
if on_error == "continue":
|
||||
return self._error_result(exc)
|
||||
raise
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - unexpected node bug
|
||||
logger.exception("workflow node %s crashed", node.id)
|
||||
last_error = WorkflowError(
|
||||
"WORKFLOW_INTERNAL",
|
||||
"节点执行发生内部错误",
|
||||
node_id=node.id,
|
||||
details={"reason": type(exc).__name__},
|
||||
)
|
||||
await self._fail_node_run(ctx, node_run, last_error, started)
|
||||
if on_error == "continue":
|
||||
return self._error_result(last_error)
|
||||
raise last_error from exc
|
||||
|
||||
payload = result.model_dump(by_alias=False)
|
||||
duration = int((time.monotonic() - started) * 1000)
|
||||
await self._finish_node_run(ctx, node_run, status="completed", output=payload, duration_ms=duration)
|
||||
await ctx.emit_event(
|
||||
"node.completed",
|
||||
data={
|
||||
"nodeType": node.type,
|
||||
"durationMs": duration,
|
||||
"data": payload.get("data"),
|
||||
"warnings": payload.get("warnings") or [],
|
||||
},
|
||||
node_id=node.id,
|
||||
)
|
||||
return payload
|
||||
|
||||
raise last_error or WorkflowError("WORKFLOW_INTERNAL", "节点执行失败", node_id=node.id)
|
||||
|
||||
def _node_timeout(self, node: WorkflowNode, ctx: RunContext) -> int:
|
||||
configured = node.config.get("timeoutSeconds") or node.config.get("timeout_seconds")
|
||||
graph_limit = ctx.graph.settings.node_timeout_seconds
|
||||
try:
|
||||
value = int(configured) if configured else graph_limit
|
||||
except (TypeError, ValueError):
|
||||
value = graph_limit
|
||||
return max(1, min(value, graph_limit))
|
||||
|
||||
def _error_result(self, error: WorkflowError) -> dict[str, Any]:
|
||||
return {
|
||||
"data": {},
|
||||
"messages": [],
|
||||
"artifacts": [],
|
||||
"metadata": {"failed": True, "error": error.to_body().model_dump(by_alias=True)},
|
||||
"warnings": [error.message],
|
||||
}
|
||||
|
||||
async def _begin_node_run(self, ctx: RunContext, node: WorkflowNode, run_attempt: int, node_attempt: int) -> dict[str, Any] | None:
|
||||
if self._store is None:
|
||||
return None
|
||||
try:
|
||||
row = await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node.id,
|
||||
"node_type": node.type,
|
||||
"status": "running",
|
||||
"attempt": node_attempt,
|
||||
"iteration": int(ctx.loop_state.get(node.id, 0)),
|
||||
"started_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run start for %s", node.id, exc_info=True)
|
||||
return None
|
||||
ctx.node_run_ids[node.id] = str(row.get("id") or "")
|
||||
return row
|
||||
|
||||
async def _finish_node_run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
node_run: dict[str, Any] | None,
|
||||
*,
|
||||
status: str,
|
||||
output: dict[str, Any] | None,
|
||||
duration_ms: int | None = None,
|
||||
) -> None:
|
||||
if self._store is None or node_run is None:
|
||||
return
|
||||
try:
|
||||
await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node_run["node_id"],
|
||||
"node_type": node_run["node_type"],
|
||||
"attempt": node_run["attempt"],
|
||||
"iteration": node_run["iteration"],
|
||||
"status": status,
|
||||
"output": output,
|
||||
"duration_ms": duration_ms,
|
||||
"finished_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run finish", exc_info=True)
|
||||
|
||||
async def _fail_node_run(
|
||||
self,
|
||||
ctx: RunContext,
|
||||
node_run: dict[str, Any] | None,
|
||||
error: WorkflowError,
|
||||
started: float,
|
||||
) -> None:
|
||||
duration = int((time.monotonic() - started) * 1000)
|
||||
body = error.to_body().model_dump(by_alias=True)
|
||||
if self._store is not None and node_run is not None:
|
||||
try:
|
||||
await self._store.record_node_run(
|
||||
{
|
||||
"run_id": ctx.run_id,
|
||||
"node_id": node_run["node_id"],
|
||||
"node_type": node_run["node_type"],
|
||||
"attempt": node_run["attempt"],
|
||||
"iteration": node_run["iteration"],
|
||||
"status": "failed",
|
||||
"error": body,
|
||||
"duration_ms": duration,
|
||||
"finished_at": datetime.now(UTC),
|
||||
"is_current": True,
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning("failed to record node run failure", exc_info=True)
|
||||
await ctx.emit_event(
|
||||
"node.failed",
|
||||
data={"error": body, "durationMs": duration},
|
||||
node_id=error.node_id,
|
||||
)
|
||||
|
||||
def _collect_output(self, nodes: dict[str, WorkflowNode], ctx: RunContext) -> dict[str, Any]:
|
||||
output: dict[str, Any] = {}
|
||||
for node_id, node in nodes.items():
|
||||
if node.type != "output":
|
||||
continue
|
||||
result = ctx.node_results.get(node_id)
|
||||
if isinstance(result, dict) and isinstance(result.get("data"), dict):
|
||||
output.update(result["data"])
|
||||
return output
|
||||
|
||||
|
||||
__all__ = ["EngineResult", "WorkflowEngine"]
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Persist-then-publish event sink for workflow runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.workflows.events import WorkflowEventEnvelope, WorkflowEventType
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PublishFn = Callable[[dict[str, Any]], Awaitable[None]]
|
||||
|
||||
|
||||
class PersistingWorkflowEventSink:
|
||||
"""Assigns ``seq`` in the database first, then fans out to live listeners.
|
||||
|
||||
A persistence failure never aborts the run, but the event is **dropped
|
||||
rather than published**: SSE clients tail by cursor and rebuild from the
|
||||
durable log, so a live-only frame with a made-up seq would render once and
|
||||
then vanish on reconnect (a "ghost event"). Callers can see the drop via
|
||||
``degraded_count`` / the ``eventPersistenceDegraded`` marker on the returned
|
||||
envelope; the stream endpoint synthesises a closing frame from the run row
|
||||
when a terminal event could not be logged.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
store: WorkflowEventStore,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
publish: PublishFn | None = None,
|
||||
) -> None:
|
||||
self._store = store
|
||||
self._run_id = run_id
|
||||
self._workflow_id = workflow_id
|
||||
self._version_id = version_id
|
||||
self._publish = publish
|
||||
self.degraded_count = 0
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
event_type: WorkflowEventType,
|
||||
*,
|
||||
data: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> WorkflowEventEnvelope:
|
||||
payload = data or {}
|
||||
seq = 0
|
||||
degraded = False
|
||||
try:
|
||||
row = await self._store.append(
|
||||
run_id=self._run_id,
|
||||
workflow_id=self._workflow_id,
|
||||
version_id=self._version_id,
|
||||
event_type=event_type,
|
||||
payload=payload,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
)
|
||||
seq = int(row.get("seq") or 0)
|
||||
except Exception as exc: # noqa: BLE001 - the run must survive a log outage
|
||||
logger.warning(
|
||||
"workflow event not persisted (dropped, not published): run=%s event=%s node=%s error=%s:%s",
|
||||
self._run_id,
|
||||
event_type,
|
||||
node_id,
|
||||
type(exc).__name__,
|
||||
exc,
|
||||
)
|
||||
self.degraded_count += 1
|
||||
degraded = True
|
||||
|
||||
envelope = WorkflowEventEnvelope(
|
||||
runId=self._run_id,
|
||||
workflowId=self._workflow_id,
|
||||
versionId=self._version_id,
|
||||
seq=seq,
|
||||
event=event_type,
|
||||
nodeId=node_id,
|
||||
nodeRunId=node_run_id,
|
||||
timestamp=datetime.now(UTC),
|
||||
data={**payload, "eventPersistenceDegraded": True} if degraded else payload,
|
||||
)
|
||||
if degraded:
|
||||
return envelope
|
||||
if self._publish is not None:
|
||||
try:
|
||||
await self._publish(envelope.to_sse_dict())
|
||||
except Exception: # noqa: BLE001 - live fan-out is best effort
|
||||
logger.debug("workflow live publish failed", exc_info=True)
|
||||
return envelope
|
||||
|
||||
|
||||
def envelope_from_row(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Turn a stored event row into the SSE wire object (replay path)."""
|
||||
return {
|
||||
"schemaVersion": "1.0",
|
||||
"runId": row.get("run_id"),
|
||||
"workflowId": row.get("workflow_id"),
|
||||
"versionId": row.get("version_id"),
|
||||
"seq": int(row.get("seq") or 0),
|
||||
"event": row.get("event_type"),
|
||||
"nodeId": row.get("node_id"),
|
||||
"nodeRunId": row.get("node_run_id"),
|
||||
"timestamp": row.get("created_at"),
|
||||
"data": row.get("payload") or {},
|
||||
}
|
||||
|
||||
|
||||
__all__ = ["PersistingWorkflowEventSink", "envelope_from_row"]
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
"""Read-only query execution for the ``sql_read`` node.
|
||||
|
||||
Engines are cached per DSN with a tiny pool because a workflow typically hits
|
||||
the same reporting database many times in one run. Values are bound as
|
||||
parameters and every cell is truncated before it can reach an event payload.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_engines: dict[str, AsyncEngine] = {}
|
||||
_lock = asyncio.Lock()
|
||||
|
||||
|
||||
async def _get_engine(dsn: str) -> AsyncEngine:
|
||||
async with _lock:
|
||||
engine = _engines.get(dsn)
|
||||
if engine is None:
|
||||
engine = create_async_engine(
|
||||
dsn,
|
||||
pool_pre_ping=True,
|
||||
pool_size=2,
|
||||
max_overflow=2,
|
||||
pool_recycle=1800,
|
||||
)
|
||||
_engines[dsn] = engine
|
||||
return engine
|
||||
|
||||
|
||||
async def dispose_engines() -> None:
|
||||
async with _lock:
|
||||
for engine in _engines.values():
|
||||
await engine.dispose()
|
||||
_engines.clear()
|
||||
|
||||
|
||||
def _coerce(value: Any, max_chars: int) -> Any:
|
||||
if value is None or isinstance(value, (bool, int, float)):
|
||||
return value
|
||||
if isinstance(value, Decimal):
|
||||
return float(value)
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, (bytes, bytearray, memoryview)):
|
||||
return f"<binary {len(bytes(value))} bytes>"
|
||||
text_value = str(value)
|
||||
return text_value if len(text_value) <= max_chars else text_value[:max_chars] + "…"
|
||||
|
||||
|
||||
async def execute_read_query(
|
||||
*,
|
||||
dsn: str,
|
||||
statement: str,
|
||||
parameters: dict[str, Any],
|
||||
max_rows: int,
|
||||
timeout_seconds: int,
|
||||
max_cell_chars: int = 4000,
|
||||
) -> dict[str, Any]:
|
||||
engine = await _get_engine(dsn)
|
||||
|
||||
async def _run() -> dict[str, Any]:
|
||||
async with engine.connect() as conn:
|
||||
result = await conn.execute(text(statement), parameters)
|
||||
columns = list(result.keys())
|
||||
rows = result.fetchmany(max_rows)
|
||||
truncated = bool(result.fetchone())
|
||||
return {
|
||||
"columns": columns,
|
||||
"rows": [{col: _coerce(val, max_cell_chars) for col, val in zip(columns, row, strict=False)} for row in rows],
|
||||
"rowCount": len(rows),
|
||||
"truncated": truncated,
|
||||
}
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(_run(), timeout=timeout_seconds)
|
||||
except TimeoutError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_TIMEOUT",
|
||||
f"数据库查询超时({timeout_seconds}s)",
|
||||
retryable=True,
|
||||
details={"timeoutSeconds": timeout_seconds},
|
||||
) from exc
|
||||
except WorkflowError:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001 - never leak DSN/credentials upward
|
||||
logger.warning("workflow sql_read failed", exc_info=True)
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
"数据库查询执行失败",
|
||||
retryable=True,
|
||||
details={"reason": type(exc).__name__},
|
||||
) from exc
|
||||
|
||||
|
||||
__all__ = ["dispose_engines", "execute_read_query"]
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
"""Load frozen workflow sample graphs shipped with the harness."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from deerflow.workflows.schemas import WorkflowGraph
|
||||
|
||||
_SAMPLES_DIR = Path(__file__).resolve().parent
|
||||
_NODES_DIR = _SAMPLES_DIR / "nodes"
|
||||
|
||||
|
||||
def samples_dir() -> Path:
|
||||
return _SAMPLES_DIR
|
||||
|
||||
|
||||
def nodes_dir() -> Path:
|
||||
return _NODES_DIR
|
||||
|
||||
|
||||
@lru_cache(maxsize=32)
|
||||
def load_sample_graph(name: str = "report_generation") -> WorkflowGraph:
|
||||
"""Load and validate a sample graph by stem name (without ``.json``)."""
|
||||
path = _SAMPLES_DIR / f"{name}.json"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"workflow sample not found: {path}")
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
return WorkflowGraph.model_validate(raw)
|
||||
|
||||
|
||||
@lru_cache(maxsize=64)
|
||||
def load_node_sample(node_type: str) -> dict:
|
||||
"""Load a minimal single-node fixture (raw dict, not a full graph)."""
|
||||
path = _NODES_DIR / f"{node_type}.json"
|
||||
if not path.is_file():
|
||||
raise FileNotFoundError(f"node sample not found: {path}")
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def list_node_samples() -> list[str]:
|
||||
if not _NODES_DIR.is_dir():
|
||||
return []
|
||||
return sorted(p.stem for p in _NODES_DIR.glob("*.json"))
|
||||
|
||||
|
||||
__all__ = [
|
||||
"list_node_samples",
|
||||
"load_node_sample",
|
||||
"load_sample_graph",
|
||||
"nodes_dir",
|
||||
"samples_dir",
|
||||
]
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"runId": "run_01",
|
||||
"workflowId": "wf_report_generation",
|
||||
"versionId": "wv_03",
|
||||
"seq": 42,
|
||||
"event": "node.output.delta",
|
||||
"nodeId": "agent_write",
|
||||
"nodeRunId": "nr_09",
|
||||
"timestamp": "2026-08-26T10:00:00+08:00",
|
||||
"data": {
|
||||
"channel": "answer",
|
||||
"delta": "第一段内容"
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "agent_1",
|
||||
"type": "agent",
|
||||
"name": "智能体",
|
||||
"config": {
|
||||
"agentId": "report-writer",
|
||||
"promptTemplate": "根据材料撰写:{{ nodes.merge_1.data }}",
|
||||
"responseMode": "text",
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600,
|
||||
"skillNames": []
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"id": "code_1",
|
||||
"type": "code",
|
||||
"name": "代码(沙箱)",
|
||||
"config": {
|
||||
"language": "python",
|
||||
"source": "def main(inputs):\n rows = inputs.get(\"rows\") or []\n return {\"count\": len(rows), \"titles\": [r.get(\"title\") for r in rows[:10]]}\n",
|
||||
"timeoutSeconds": 60,
|
||||
"allowNetwork": false
|
||||
}
|
||||
}
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"id": "cond_1",
|
||||
"type": "condition",
|
||||
"name": "条件",
|
||||
"config": {
|
||||
"branches": [
|
||||
{
|
||||
"name": "passed",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == true"
|
||||
},
|
||||
{
|
||||
"name": "rejected",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == false"
|
||||
}
|
||||
],
|
||||
"defaultBranch": "rejected"
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"id": "http_1",
|
||||
"type": "http",
|
||||
"name": "HTTP",
|
||||
"config": {
|
||||
"method": "GET",
|
||||
"url": "https://api.example.internal/items/{{ inputs.taskId }}",
|
||||
"headers": { "Accept": "application/json" },
|
||||
"query": {},
|
||||
"body": null,
|
||||
"credentialRef": "cred_task_api",
|
||||
"timeoutSeconds": 30,
|
||||
"maxResponseBytes": 1048576,
|
||||
"allowRedirects": false
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "human_1",
|
||||
"type": "human_input",
|
||||
"name": "人工介入",
|
||||
"config": {
|
||||
"formSchema": {
|
||||
"type": "object",
|
||||
"required": ["decision"],
|
||||
"properties": {
|
||||
"decision": {
|
||||
"type": "string",
|
||||
"enum": ["approve", "revise"]
|
||||
},
|
||||
"comment": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"prompt": "请确认是否放行当前报告草稿。",
|
||||
"actions": ["approve", "revise"]
|
||||
}
|
||||
}
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"id": "loop_1",
|
||||
"type": "loop",
|
||||
"name": "受限循环",
|
||||
"config": {
|
||||
"maxIterations": 2,
|
||||
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
|
||||
"bodyEntry": "agent_write"
|
||||
}
|
||||
}
|
||||
+9
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"id": "merge_1",
|
||||
"type": "merge",
|
||||
"name": "合并",
|
||||
"config": {
|
||||
"strategy": "all",
|
||||
"sources": ["sql_1", "http_1", "agent_collect"]
|
||||
}
|
||||
}
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"id": "output_1",
|
||||
"type": "output",
|
||||
"name": "输出",
|
||||
"config": {
|
||||
"mapping": {
|
||||
"reportMarkdown": "{{ nodes.agent_write.data.answer }}"
|
||||
},
|
||||
"llmRepair": false
|
||||
}
|
||||
}
|
||||
+12
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"id": "skill_1",
|
||||
"type": "skill",
|
||||
"name": "技能(agent_skill)",
|
||||
"config": {
|
||||
"mode": "agent_skill",
|
||||
"agentId": "report-collector",
|
||||
"skillNames": ["web-research"],
|
||||
"promptTemplate": "使用指定技能完成:{{ inputs.topic }}",
|
||||
"timeoutSeconds": 300
|
||||
}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "sql_1",
|
||||
"type": "sql_read",
|
||||
"name": "SQL 只读",
|
||||
"config": {
|
||||
"dataSourceId": "ds_report_readonly",
|
||||
"statement": "SELECT id, title FROM report_source WHERE task_id = :task_id",
|
||||
"parameters": {
|
||||
"task_id": "{{ inputs.taskId }}"
|
||||
},
|
||||
"maxRows": 1000
|
||||
}
|
||||
}
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"id": "start_1",
|
||||
"type": "start",
|
||||
"name": "开始",
|
||||
"config": {}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"id": "sub_1",
|
||||
"type": "subworkflow",
|
||||
"name": "子工作流",
|
||||
"config": {
|
||||
"workflowId": "wf_collect_pack",
|
||||
"versionId": "wv_01",
|
||||
"inputMapping": {
|
||||
"topic": "{{ inputs.topic }}",
|
||||
"taskId": "{{ inputs.taskId }}"
|
||||
}
|
||||
}
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "transform_1",
|
||||
"type": "transform",
|
||||
"name": "模板/转换",
|
||||
"config": {
|
||||
"operations": [
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.sql_1.data.rows }}",
|
||||
"to": "rows"
|
||||
},
|
||||
{
|
||||
"op": "template",
|
||||
"template": "# {{ inputs.topic }}\n\n共 {{ nodes.sql_1.metadata.rowCount }} 行。",
|
||||
"to": "markdown"
|
||||
}
|
||||
],
|
||||
"template": null
|
||||
}
|
||||
}
|
||||
+260
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"id": "wf_report_generation",
|
||||
"name": "专题报告生成",
|
||||
"description": "阶段0验收基线:并行取数 → 合并 → 整理 → 撰写 → 审核 → 条件退回/输出",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"required": ["topic", "taskId"],
|
||||
"properties": {
|
||||
"topic": { "type": "string", "description": "报告主题" },
|
||||
"taskId": { "type": "string", "description": "业务任务 ID" },
|
||||
"dateRange": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"start": { "type": "string" },
|
||||
"end": { "type": "string" }
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"outputSchema": {
|
||||
"type": "object",
|
||||
"required": ["reportMarkdown", "passed"],
|
||||
"properties": {
|
||||
"reportMarkdown": { "type": "string" },
|
||||
"passed": { "type": "boolean" },
|
||||
"artifacts": { "type": "array" },
|
||||
"issues": { "type": "array" }
|
||||
}
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"id": "start_1",
|
||||
"type": "start",
|
||||
"name": "开始",
|
||||
"config": {}
|
||||
},
|
||||
{
|
||||
"id": "sql_1",
|
||||
"type": "sql_read",
|
||||
"name": "查询内部结构化数据",
|
||||
"config": {
|
||||
"dataSourceId": "ds_report_readonly",
|
||||
"statement": "SELECT id, title, content, updated_at FROM report_source WHERE task_id = :task_id ORDER BY updated_at DESC",
|
||||
"parameters": {
|
||||
"task_id": "{{ inputs.taskId }}"
|
||||
},
|
||||
"maxRows": 1000
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "http_1",
|
||||
"type": "http",
|
||||
"name": "调用只读业务接口",
|
||||
"config": {
|
||||
"method": "GET",
|
||||
"url": "https://api.example.internal/tasks/{{ inputs.taskId }}/summary",
|
||||
"headers": {
|
||||
"Accept": "application/json"
|
||||
},
|
||||
"credentialRef": "cred_task_api",
|
||||
"timeoutSeconds": 30,
|
||||
"maxResponseBytes": 524288,
|
||||
"allowRedirects": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_collect",
|
||||
"type": "agent",
|
||||
"name": "外部材料采集",
|
||||
"config": {
|
||||
"agentId": "report-collector",
|
||||
"promptTemplate": "围绕主题「{{ inputs.topic }}」搜集外部公开材料,任务 ID:{{ inputs.taskId }}。输出 JSON:{sources:[], notes:string}",
|
||||
"responseMode": "json",
|
||||
"responseSchema": {
|
||||
"type": "object",
|
||||
"required": ["sources", "notes"],
|
||||
"properties": {
|
||||
"sources": { "type": "array" },
|
||||
"notes": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600,
|
||||
"skillNames": ["web-research"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "merge_1",
|
||||
"type": "merge",
|
||||
"name": "合并材料并保留来源",
|
||||
"config": {
|
||||
"strategy": "all",
|
||||
"sources": ["sql_1", "http_1", "agent_collect"]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_clean",
|
||||
"type": "agent",
|
||||
"name": "数据整理与事实去重",
|
||||
"config": {
|
||||
"agentId": "report-cleaner",
|
||||
"promptTemplate": "整理并去重以下材料,输出严格 JSON:{{ nodes.merge_1.data }}",
|
||||
"responseMode": "json",
|
||||
"responseSchema": {
|
||||
"type": "object",
|
||||
"required": ["facts", "sources"],
|
||||
"properties": {
|
||||
"facts": { "type": "array" },
|
||||
"sources": { "type": "array" }
|
||||
}
|
||||
},
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_write",
|
||||
"type": "agent",
|
||||
"name": "撰写 Markdown 报告",
|
||||
"config": {
|
||||
"agentId": "report-writer",
|
||||
"promptTemplate": "根据整理结果撰写专题报告(Markdown)。主题:{{ inputs.topic }}\n材料:{{ nodes.agent_clean.data }}\n若存在审核意见:{{ nodes.agent_review.data.revisedAdvice }}",
|
||||
"responseMode": "text",
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 900
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "agent_review",
|
||||
"type": "agent",
|
||||
"name": "审核报告",
|
||||
"config": {
|
||||
"agentId": "report-reviewer",
|
||||
"promptTemplate": "审核以下报告,输出 JSON {passed:boolean, issues:array, revisedAdvice:string}:\n{{ nodes.agent_write.data.answer }}",
|
||||
"responseMode": "json",
|
||||
"responseSchema": {
|
||||
"type": "object",
|
||||
"required": ["passed", "issues", "revisedAdvice"],
|
||||
"properties": {
|
||||
"passed": { "type": "boolean" },
|
||||
"issues": { "type": "array" },
|
||||
"revisedAdvice": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"threadMode": "isolated_per_node",
|
||||
"timeoutSeconds": 600
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "cond_review",
|
||||
"type": "condition",
|
||||
"name": "审核结果分支",
|
||||
"config": {
|
||||
"branches": [
|
||||
{
|
||||
"name": "passed",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == true"
|
||||
},
|
||||
{
|
||||
"name": "rejected",
|
||||
"expression": "{{ nodes.agent_review.data.passed }} == false"
|
||||
}
|
||||
],
|
||||
"defaultBranch": "rejected"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "loop_revise",
|
||||
"type": "loop",
|
||||
"name": "退回修改(最多 2 次)",
|
||||
"config": {
|
||||
"maxIterations": 2,
|
||||
"continueWhen": "{{ nodes.agent_review.data.passed }} == false",
|
||||
"bodyEntry": "agent_write"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "transform_pack",
|
||||
"type": "transform",
|
||||
"name": "组装最终字段",
|
||||
"config": {
|
||||
"operations": [
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.agent_write.data.answer }}",
|
||||
"to": "reportMarkdown"
|
||||
},
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.agent_review.data.passed }}",
|
||||
"to": "passed"
|
||||
},
|
||||
{
|
||||
"op": "map",
|
||||
"from": "{{ nodes.agent_review.data.issues }}",
|
||||
"to": "issues"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "output_1",
|
||||
"type": "output",
|
||||
"name": "输出",
|
||||
"config": {
|
||||
"mapping": {
|
||||
"reportMarkdown": "{{ nodes.transform_pack.data.reportMarkdown }}",
|
||||
"passed": "{{ nodes.transform_pack.data.passed }}",
|
||||
"issues": "{{ nodes.transform_pack.data.issues }}"
|
||||
},
|
||||
"llmRepair": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{ "id": "e_start_sql", "source": "start_1", "target": "sql_1" },
|
||||
{ "id": "e_start_http", "source": "start_1", "target": "http_1" },
|
||||
{ "id": "e_start_collect", "source": "start_1", "target": "agent_collect" },
|
||||
{ "id": "e_sql_merge", "source": "sql_1", "target": "merge_1" },
|
||||
{ "id": "e_http_merge", "source": "http_1", "target": "merge_1" },
|
||||
{ "id": "e_collect_merge", "source": "agent_collect", "target": "merge_1" },
|
||||
{ "id": "e_merge_clean", "source": "merge_1", "target": "agent_clean" },
|
||||
{ "id": "e_clean_write", "source": "agent_clean", "target": "agent_write" },
|
||||
{ "id": "e_write_review", "source": "agent_write", "target": "agent_review" },
|
||||
{ "id": "e_review_cond", "source": "agent_review", "target": "cond_review" },
|
||||
{
|
||||
"id": "e_cond_pass",
|
||||
"source": "cond_review",
|
||||
"target": "transform_pack",
|
||||
"sourcePort": "passed"
|
||||
},
|
||||
{
|
||||
"id": "e_cond_reject",
|
||||
"source": "cond_review",
|
||||
"target": "loop_revise",
|
||||
"sourcePort": "rejected"
|
||||
},
|
||||
{
|
||||
"id": "e_loop_write",
|
||||
"source": "loop_revise",
|
||||
"target": "agent_write",
|
||||
"sourcePort": "continue"
|
||||
},
|
||||
{
|
||||
"id": "e_loop_fail",
|
||||
"source": "loop_revise",
|
||||
"target": "transform_pack",
|
||||
"sourcePort": "exhausted"
|
||||
},
|
||||
{ "id": "e_pack_out", "source": "transform_pack", "target": "output_1" }
|
||||
],
|
||||
"settings": {
|
||||
"runTimeoutSeconds": 1800,
|
||||
"nodeTimeoutSeconds": 300,
|
||||
"maxSteps": 200,
|
||||
"maxLoopIterations": 5,
|
||||
"maxParallelism": 4
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"id": "wf_start_output",
|
||||
"name": "空图:开始→输出",
|
||||
"description": "阶段2运行骨架用的最小可执行图",
|
||||
"inputSchema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"message": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"outputSchema": {
|
||||
"type": "object",
|
||||
"required": ["message"],
|
||||
"properties": {
|
||||
"message": { "type": "string" }
|
||||
}
|
||||
},
|
||||
"nodes": [
|
||||
{ "id": "start_1", "type": "start", "name": "开始", "config": {} },
|
||||
{
|
||||
"id": "output_1",
|
||||
"type": "output",
|
||||
"name": "输出",
|
||||
"config": {
|
||||
"mapping": { "message": "{{ inputs.message }}" },
|
||||
"llmRepair": false
|
||||
}
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{ "id": "e1", "source": "start_1", "target": "output_1" }
|
||||
],
|
||||
"settings": {
|
||||
"runTimeoutSeconds": 60,
|
||||
"nodeTimeoutSeconds": 30,
|
||||
"maxSteps": 10,
|
||||
"maxLoopIterations": 1,
|
||||
"maxParallelism": 1
|
||||
}
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
"""Full JSON Schema validation for workflow input/output values.
|
||||
|
||||
The engine used to check only ``required`` keys; everything else (wrong type,
|
||||
unknown enum value, nested ``properties``/``items`` violations) slipped through
|
||||
until downstream nodes exploded with confusing errors. This module runs the
|
||||
graph's ``inputSchema`` / ``outputSchema`` through ``Draft202012Validator`` and
|
||||
returns structured, frontend-friendly errors (path + message) instead of
|
||||
leaking Python tracebacks.
|
||||
|
||||
Kept as pure functions so both the node executors (``WorkflowError``) and the
|
||||
HTTP router (400 responses) can share the exact same rules.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
# Cap the reported errors so a wildly invalid payload cannot produce a
|
||||
# multi-megabyte error body.
|
||||
_MAX_ERRORS = 20
|
||||
|
||||
|
||||
def schema_issues(value: Any, schema: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
"""Validate ``value`` against ``schema``.
|
||||
|
||||
Returns ``[]`` when the schema is empty/absent (nothing declared = nothing
|
||||
enforced). Each issue is ``{"path": [...], "message": str}`` with ``path``
|
||||
pointing at the offending field, e.g. ``["options", 0, "name"]``.
|
||||
"""
|
||||
if not isinstance(schema, dict) or not schema:
|
||||
return []
|
||||
try:
|
||||
validator = Draft202012Validator(schema)
|
||||
# Construction alone does not check the schema — malformed keywords
|
||||
# (e.g. a non-string ``pattern``) only explode during validation.
|
||||
errors = sorted(validator.iter_errors(value), key=lambda err: list(err.absolute_path))
|
||||
except Exception: # noqa: BLE001 - a malformed user schema must not 500
|
||||
return [{"path": [], "message": "schema 定义本身无效,无法用于校验"}]
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[tuple[Any, ...]] = set()
|
||||
for err in errors:
|
||||
path = [str(p) if not isinstance(p, int) else p for p in err.absolute_path]
|
||||
key = tuple(path)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append({"path": path, "message": _humanize(err)})
|
||||
if len(out) >= _MAX_ERRORS:
|
||||
out.append({"path": [], "message": f"……其余问题已省略(共 {len(errors)} 处)"})
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def _humanize(err: Any) -> str:
|
||||
"""Map the common jsonschema phrasings to short Chinese messages."""
|
||||
message = str(err.message)
|
||||
validator = getattr(err, "validator", "")
|
||||
path = ".".join(str(p) for p in err.absolute_path) or "输入"
|
||||
if validator == "required":
|
||||
missing = message.split("'")[1] if "'" in message else "?"
|
||||
return f"{path} 缺少必填字段:{missing}"
|
||||
if validator == "enum":
|
||||
return f"{path} 的值不在允许范围内:{message}"
|
||||
if validator == "type":
|
||||
return f"{path} 类型不正确:{message}"
|
||||
if validator == "additionalProperties":
|
||||
return f"{path} 含有未声明的额外字段:{message}"
|
||||
return f"{path} 不符合 schema:{message}"
|
||||
|
||||
|
||||
__all__ = ["schema_issues"]
|
||||
@@ -0,0 +1,374 @@
|
||||
"""Frozen workflow graph schema v1.0 (internal DTO).
|
||||
|
||||
Coze canvas DTO is converted into this structure at the Gateway boundary.
|
||||
The runtime must never depend on Coze-private fields.
|
||||
|
||||
Wire JSON uses camelCase aliases; Python code uses snake_case.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||||
|
||||
SCHEMA_VERSION = "1.0"
|
||||
|
||||
NodeType = Literal[
|
||||
"start",
|
||||
"output",
|
||||
"agent",
|
||||
"skill",
|
||||
"http",
|
||||
"sql_read",
|
||||
"code",
|
||||
"transform",
|
||||
"condition",
|
||||
"merge",
|
||||
"human_input",
|
||||
"subworkflow",
|
||||
"loop",
|
||||
]
|
||||
|
||||
ALL_NODE_TYPES: tuple[NodeType, ...] = (
|
||||
"start",
|
||||
"output",
|
||||
"agent",
|
||||
"skill",
|
||||
"http",
|
||||
"sql_read",
|
||||
"code",
|
||||
"transform",
|
||||
"condition",
|
||||
"merge",
|
||||
"human_input",
|
||||
"subworkflow",
|
||||
"loop",
|
||||
)
|
||||
|
||||
RunStatus = Literal[
|
||||
"queued",
|
||||
"running",
|
||||
"awaiting_input",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancel_requested",
|
||||
"cancelled",
|
||||
]
|
||||
|
||||
ALL_RUN_STATUSES: tuple[RunStatus, ...] = (
|
||||
"queued",
|
||||
"running",
|
||||
"awaiting_input",
|
||||
"completed",
|
||||
"failed",
|
||||
"cancel_requested",
|
||||
"cancelled",
|
||||
)
|
||||
|
||||
TERMINAL_RUN_STATUSES: frozenset[RunStatus] = frozenset({"completed", "failed", "cancelled"})
|
||||
|
||||
NodeRunStatus = Literal[
|
||||
"queued",
|
||||
"running",
|
||||
"completed",
|
||||
"failed",
|
||||
"skipped",
|
||||
"cancelled",
|
||||
]
|
||||
|
||||
SkillMode = Literal["agent_skill", "callable_skill"]
|
||||
MergeStrategy = Literal["all", "first_success", "any_success"]
|
||||
HttpMethod = Literal["GET", "POST", "PUT", "PATCH", "DELETE"]
|
||||
AgentResponseMode = Literal["text", "json"]
|
||||
AgentThreadMode = Literal["isolated_per_node", "shared_run_thread"]
|
||||
ExecutionMode = Literal["normal"]
|
||||
|
||||
|
||||
class WorkflowGraphSettings(BaseModel):
|
||||
"""Per-graph resource limits (still capped by system limits at publish)."""
|
||||
|
||||
run_timeout_seconds: int = Field(default=1800, ge=1, alias="runTimeoutSeconds")
|
||||
node_timeout_seconds: int = Field(default=300, ge=1, alias="nodeTimeoutSeconds")
|
||||
max_steps: int = Field(default=200, ge=1, alias="maxSteps")
|
||||
max_loop_iterations: int = Field(default=5, ge=1, alias="maxLoopIterations")
|
||||
max_parallelism: int = Field(default=4, ge=1, alias="maxParallelism")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class WorkflowNode(BaseModel):
|
||||
"""A single node on the internal standard graph."""
|
||||
|
||||
id: str = Field(min_length=1)
|
||||
type: NodeType
|
||||
name: str = ""
|
||||
config: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
@field_validator("id")
|
||||
@classmethod
|
||||
def _id_not_blank(cls, value: str) -> str:
|
||||
stripped = value.strip()
|
||||
if not stripped:
|
||||
raise ValueError("node id must be non-empty")
|
||||
return stripped
|
||||
|
||||
|
||||
class WorkflowEdge(BaseModel):
|
||||
"""Directed edge. ``source_port`` names a condition / loop outlet when set."""
|
||||
|
||||
id: str = Field(min_length=1)
|
||||
source: str
|
||||
target: str
|
||||
source_port: str | None = Field(default=None, alias="sourcePort")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class WorkflowGraph(BaseModel):
|
||||
"""Internal standard workflow graph (publish / run snapshot)."""
|
||||
|
||||
schema_version: Literal["1.0"] = Field(default=SCHEMA_VERSION, alias="schemaVersion")
|
||||
id: str = Field(min_length=1)
|
||||
name: str = ""
|
||||
description: str = ""
|
||||
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
|
||||
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
|
||||
nodes: list[WorkflowNode] = Field(default_factory=list)
|
||||
edges: list[WorkflowEdge] = Field(default_factory=list)
|
||||
settings: WorkflowGraphSettings = Field(default_factory=WorkflowGraphSettings)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _unique_ids(self) -> WorkflowGraph:
|
||||
node_ids = [n.id for n in self.nodes]
|
||||
if len(node_ids) != len(set(node_ids)):
|
||||
raise ValueError("duplicate node id")
|
||||
edge_ids = [e.id for e in self.edges]
|
||||
if len(edge_ids) != len(set(edge_ids)):
|
||||
raise ValueError("duplicate edge id")
|
||||
known = set(node_ids)
|
||||
for edge in self.edges:
|
||||
if edge.source not in known:
|
||||
raise ValueError(f"edge {edge.id} source not found: {edge.source}")
|
||||
if edge.target not in known:
|
||||
raise ValueError(f"edge {edge.id} target not found: {edge.target}")
|
||||
return self
|
||||
|
||||
def node_map(self) -> dict[str, WorkflowNode]:
|
||||
return {node.id: node for node in self.nodes}
|
||||
|
||||
def canonical_json(self) -> str:
|
||||
"""Deterministic JSON used for ``graph_hash`` (sorted keys, no whitespace)."""
|
||||
return self.model_dump_json(by_alias=True, exclude_none=True)
|
||||
|
||||
def graph_hash(self) -> str:
|
||||
return sha256(self.canonical_json().encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class ArtifactRef(BaseModel):
|
||||
"""Reference to a large/binary result; full bytes live outside events."""
|
||||
|
||||
artifact_id: str = Field(alias="artifactId")
|
||||
name: str
|
||||
mime_type: str = Field(default="application/octet-stream", alias="mimeType")
|
||||
path: str = ""
|
||||
size_bytes: int | None = Field(default=None, alias="sizeBytes")
|
||||
preview: str | None = None
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class NodeResult(BaseModel):
|
||||
"""Uniform result returned by every node executor."""
|
||||
|
||||
data: dict[str, Any] = Field(default_factory=dict)
|
||||
messages: list[dict[str, Any]] = Field(default_factory=list)
|
||||
artifacts: list[ArtifactRef] = Field(default_factory=list)
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
warnings: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class StartRunRequest(BaseModel):
|
||||
"""Body for ``POST /api/workflows/{workflow_id}/runs``."""
|
||||
|
||||
version_id: str = Field(alias="versionId")
|
||||
inputs: dict[str, Any] = Field(default_factory=dict)
|
||||
idempotency_key: str | None = Field(default=None, alias="idempotencyKey")
|
||||
execution_mode: ExecutionMode = Field(default="normal", alias="executionMode")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class ResumeRunRequest(BaseModel):
|
||||
"""Body for ``POST /api/workflows/runs/{run_id}/resume``."""
|
||||
|
||||
resume_token: str = Field(alias="resumeToken")
|
||||
action: str
|
||||
payload: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
# ── Typed node config helpers (optional; validators may use later) ───────────
|
||||
|
||||
|
||||
class AgentNodeConfig(BaseModel):
|
||||
agent_id: str = Field(alias="agentId")
|
||||
prompt_template: str = Field(default="", alias="promptTemplate")
|
||||
response_mode: AgentResponseMode = Field(default="text", alias="responseMode")
|
||||
response_schema: dict[str, Any] = Field(default_factory=dict, alias="responseSchema")
|
||||
thread_mode: AgentThreadMode = Field(default="isolated_per_node", alias="threadMode")
|
||||
timeout_seconds: int = Field(default=600, ge=1, alias="timeoutSeconds")
|
||||
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class SkillNodeConfig(BaseModel):
|
||||
mode: SkillMode = "agent_skill"
|
||||
agent_id: str | None = Field(default=None, alias="agentId")
|
||||
skill_names: list[str] = Field(default_factory=list, alias="skillNames")
|
||||
prompt_template: str = Field(default="", alias="promptTemplate")
|
||||
# callable_skill fields (phase 4+; accepted in schema, not executed in phase 0)
|
||||
entry: str | None = None
|
||||
input_schema: dict[str, Any] = Field(default_factory=dict, alias="inputSchema")
|
||||
output_schema: dict[str, Any] = Field(default_factory=dict, alias="outputSchema")
|
||||
timeout_seconds: int = Field(default=300, ge=1, alias="timeoutSeconds")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class HttpNodeConfig(BaseModel):
|
||||
method: HttpMethod = "GET"
|
||||
url: str
|
||||
headers: dict[str, str] = Field(default_factory=dict)
|
||||
query: dict[str, Any] = Field(default_factory=dict)
|
||||
body: Any = None
|
||||
credential_ref: str | None = Field(default=None, alias="credentialRef")
|
||||
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
|
||||
max_response_bytes: int = Field(default=1_048_576, ge=1, alias="maxResponseBytes")
|
||||
allow_redirects: bool = Field(default=False, alias="allowRedirects")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class SqlReadNodeConfig(BaseModel):
|
||||
data_source_id: str = Field(alias="dataSourceId")
|
||||
statement: str
|
||||
parameters: dict[str, Any] = Field(default_factory=dict)
|
||||
max_rows: int = Field(default=1000, ge=1, alias="maxRows")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class CodeNodeConfig(BaseModel):
|
||||
language: Literal["python"] = "python"
|
||||
source: str
|
||||
timeout_seconds: int = Field(default=60, ge=1, alias="timeoutSeconds")
|
||||
allow_network: bool = Field(default=False, alias="allowNetwork")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class TransformNodeConfig(BaseModel):
|
||||
operations: list[dict[str, Any]] = Field(default_factory=list)
|
||||
template: str | None = None
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class ConditionBranch(BaseModel):
|
||||
name: str
|
||||
expression: str
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class ConditionNodeConfig(BaseModel):
|
||||
branches: list[ConditionBranch] = Field(default_factory=list)
|
||||
default_branch: str | None = Field(default=None, alias="defaultBranch")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class MergeNodeConfig(BaseModel):
|
||||
strategy: MergeStrategy = "all"
|
||||
sources: list[str] = Field(default_factory=list)
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class HumanInputNodeConfig(BaseModel):
|
||||
form_schema: dict[str, Any] = Field(default_factory=dict, alias="formSchema")
|
||||
prompt: str = ""
|
||||
actions: list[str] = Field(default_factory=lambda: ["submit"])
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class SubworkflowNodeConfig(BaseModel):
|
||||
workflow_id: str = Field(alias="workflowId")
|
||||
version_id: str = Field(alias="versionId")
|
||||
input_mapping: dict[str, str] = Field(default_factory=dict, alias="inputMapping")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class LoopNodeConfig(BaseModel):
|
||||
max_iterations: int = Field(default=5, ge=1, alias="maxIterations")
|
||||
continue_when: str = Field(default="", alias="continueWhen")
|
||||
body_entry: str = Field(alias="bodyEntry")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
class OutputNodeConfig(BaseModel):
|
||||
mapping: dict[str, str] = Field(default_factory=dict)
|
||||
llm_repair: bool = Field(default=False, alias="llmRepair")
|
||||
|
||||
model_config = {"extra": "forbid", "populate_by_name": True}
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SCHEMA_VERSION",
|
||||
"ALL_NODE_TYPES",
|
||||
"ALL_RUN_STATUSES",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"AgentNodeConfig",
|
||||
"AgentResponseMode",
|
||||
"AgentThreadMode",
|
||||
"ArtifactRef",
|
||||
"CodeNodeConfig",
|
||||
"ConditionBranch",
|
||||
"ConditionNodeConfig",
|
||||
"ExecutionMode",
|
||||
"HttpMethod",
|
||||
"HttpNodeConfig",
|
||||
"HumanInputNodeConfig",
|
||||
"LoopNodeConfig",
|
||||
"MergeNodeConfig",
|
||||
"MergeStrategy",
|
||||
"NodeResult",
|
||||
"NodeRunStatus",
|
||||
"NodeType",
|
||||
"OutputNodeConfig",
|
||||
"ResumeRunRequest",
|
||||
"RunStatus",
|
||||
"SkillMode",
|
||||
"SkillNodeConfig",
|
||||
"SqlReadNodeConfig",
|
||||
"StartRunRequest",
|
||||
"SubworkflowNodeConfig",
|
||||
"TransformNodeConfig",
|
||||
"WorkflowEdge",
|
||||
"WorkflowGraph",
|
||||
"WorkflowGraphSettings",
|
||||
"WorkflowNode",
|
||||
]
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
"""Security policies for external-effect workflow nodes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.workflows.security.http_policy import enforce_http_target, enforce_redirect
|
||||
from deerflow.workflows.security.sql_policy import (
|
||||
assert_parameters_bound,
|
||||
assert_read_only,
|
||||
enforce_limit,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"assert_parameters_bound",
|
||||
"assert_read_only",
|
||||
"enforce_http_target",
|
||||
"enforce_limit",
|
||||
"enforce_redirect",
|
||||
]
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
"""Egress policy for the ``http`` node (SSRF defence).
|
||||
|
||||
Checks, in order: scheme → host allow/deny lists → port → every resolved IP
|
||||
address. Resolving the host and validating *all* A/AAAA answers is what stops
|
||||
`http://internal.example.com` from pointing at `169.254.169.254` or `127.0.0.1`.
|
||||
|
||||
Known limitation: we validate the resolved addresses and then let httpx dial the
|
||||
hostname again, so a DNS answer that changes between the two lookups is not
|
||||
covered (classic DNS-rebinding TOCTOU). Closing that fully requires dialing the
|
||||
pinned IP with SNI/Host overrides; for now the deployment guidance is to keep
|
||||
``allow_private_networks`` false and prefer an explicit ``allowed_hosts`` list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse, urlsplit
|
||||
|
||||
from deerflow.config.workflow_config import WorkflowHttpPolicyConfig
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
_ALLOWED_SCHEMES = ("http", "https")
|
||||
_DEFAULT_PORTS = {"http": 80, "https": 443}
|
||||
# Cloud metadata endpoints get their own explicit deny so a permissive
|
||||
# intranet configuration cannot accidentally expose instance credentials.
|
||||
_METADATA_HOSTS = frozenset({"169.254.169.254", "metadata.google.internal", "metadata"})
|
||||
|
||||
|
||||
def _denied(message: str, **details: object) -> WorkflowError:
|
||||
return WorkflowError("WORKFLOW_HTTP_POLICY_DENIED", message, details=dict(details))
|
||||
|
||||
|
||||
def _host_matches(host: str, patterns: list[str]) -> bool:
|
||||
target = host.lower().rstrip(".")
|
||||
for raw in patterns:
|
||||
pattern = str(raw).strip().lower().rstrip(".")
|
||||
if not pattern:
|
||||
continue
|
||||
if pattern.startswith("."):
|
||||
if target == pattern[1:] or target.endswith(pattern):
|
||||
return True
|
||||
elif pattern.startswith("*."):
|
||||
if target == pattern[2:] or target.endswith(pattern[1:]):
|
||||
return True
|
||||
elif target == pattern:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _is_private(address: str) -> bool:
|
||||
try:
|
||||
ip = ipaddress.ip_address(address)
|
||||
except ValueError:
|
||||
return False
|
||||
return ip.is_private or ip.is_loopback or ip.is_link_local or ip.is_reserved or ip.is_multicast or ip.is_unspecified
|
||||
|
||||
|
||||
async def _resolve(host: str, port: int) -> list[str]:
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
infos = await loop.getaddrinfo(host, port, type=socket.SOCK_STREAM)
|
||||
except OSError as exc:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_HTTP_FAILED",
|
||||
"无法解析目标域名",
|
||||
retryable=True,
|
||||
details={"host": host, "reason": type(exc).__name__},
|
||||
) from exc
|
||||
return sorted({str(info[4][0]) for info in infos})
|
||||
|
||||
|
||||
async def enforce_http_target(url: str, policy: WorkflowHttpPolicyConfig) -> dict[str, object]:
|
||||
"""Validate a target URL. Returns the resolved metadata or raises."""
|
||||
if not policy.enabled:
|
||||
raise _denied("HTTP 节点已被系统禁用")
|
||||
parsed = urlparse(url)
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in _ALLOWED_SCHEMES:
|
||||
raise _denied("仅允许 http/https 协议", scheme=scheme or "")
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host:
|
||||
raise _denied("URL 缺少主机名")
|
||||
port = parsed.port or _DEFAULT_PORTS[scheme]
|
||||
if host in _METADATA_HOSTS:
|
||||
raise _denied("目标为云元数据地址,已拒绝", host=host)
|
||||
if _host_matches(host, policy.blocked_hosts):
|
||||
raise _denied("目标主机在禁止名单中", host=host)
|
||||
if policy.allowed_hosts and not _host_matches(host, policy.allowed_hosts):
|
||||
raise _denied("目标主机不在允许名单中", host=host)
|
||||
if policy.allowed_ports and port not in policy.allowed_ports:
|
||||
raise _denied("目标端口不被允许", port=port)
|
||||
|
||||
addresses = await _resolve(host, port)
|
||||
if not policy.allow_private_networks:
|
||||
private = [addr for addr in addresses if _is_private(addr)]
|
||||
if private:
|
||||
raise _denied("目标解析到内网/环回地址,已拒绝", host=host)
|
||||
if any(addr in _METADATA_HOSTS for addr in addresses):
|
||||
raise _denied("目标解析到云元数据地址,已拒绝", host=host)
|
||||
return {"host": host, "port": port, "scheme": scheme, "addresses": addresses}
|
||||
|
||||
|
||||
async def enforce_redirect(location: str, base_url: str, policy: WorkflowHttpPolicyConfig) -> str:
|
||||
"""Validate a redirect hop, resolving relative locations against the base."""
|
||||
if location.startswith("/"):
|
||||
base = urlsplit(base_url)
|
||||
location = f"{base.scheme}://{base.netloc}{location}"
|
||||
await enforce_http_target(location, policy)
|
||||
return location
|
||||
|
||||
|
||||
__all__ = ["enforce_http_target", "enforce_redirect"]
|
||||
+97
@@ -0,0 +1,97 @@
|
||||
"""Envelope encryption for workflow secrets (data-source DSNs, credentials).
|
||||
|
||||
The key comes from ``WORKFLOW_SECRET_KEY`` (or ``DEERFLOW_SECRET_KEY``) and is
|
||||
stretched with SHA-256 into a Fernet key, so operators can supply any
|
||||
sufficiently long passphrase.
|
||||
|
||||
When no key is configured we refuse to encrypt rather than silently storing
|
||||
plaintext: the caller decides whether to reject the write or fall back. That
|
||||
choice is deliberate — a silent plaintext fallback is how credentials end up in
|
||||
database backups without anyone noticing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_ENV_KEYS = ("WORKFLOW_SECRET_KEY", "DEERFLOW_SECRET_KEY")
|
||||
_PREFIX = "enc:v1:"
|
||||
|
||||
|
||||
class SecretKeyMissingError(RuntimeError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__("未配置 WORKFLOW_SECRET_KEY,无法加密保存敏感连接信息")
|
||||
|
||||
|
||||
def _fernet():
|
||||
key_material = ""
|
||||
for name in _ENV_KEYS:
|
||||
value = os.environ.get(name)
|
||||
if value and value.strip():
|
||||
key_material = value.strip()
|
||||
break
|
||||
if not key_material:
|
||||
raise SecretKeyMissingError()
|
||||
try:
|
||||
from cryptography.fernet import Fernet
|
||||
except ImportError as exc: # pragma: no cover - dependency is declared
|
||||
raise SecretKeyMissingError() from exc
|
||||
digest = hashlib.sha256(key_material.encode("utf-8")).digest()
|
||||
return Fernet(base64.urlsafe_b64encode(digest))
|
||||
|
||||
|
||||
def secrets_available() -> bool:
|
||||
try:
|
||||
_fernet()
|
||||
except Exception: # noqa: BLE001 - any failure means "cannot encrypt"
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def encrypt_secret(plaintext: str) -> str:
|
||||
token = _fernet().encrypt(plaintext.encode("utf-8")).decode("ascii")
|
||||
return f"{_PREFIX}{token}"
|
||||
|
||||
|
||||
def decrypt_secret(stored: str) -> str:
|
||||
if not stored:
|
||||
return ""
|
||||
if not stored.startswith(_PREFIX):
|
||||
# Legacy / externally seeded plaintext row.
|
||||
return stored
|
||||
return _fernet().decrypt(stored[len(_PREFIX) :].encode("ascii")).decode("utf-8")
|
||||
|
||||
|
||||
def is_encrypted(stored: str) -> bool:
|
||||
return bool(stored) and stored.startswith(_PREFIX)
|
||||
|
||||
|
||||
def mask_dsn(dsn: str) -> str:
|
||||
"""Human-readable summary that never reveals user/password/host details."""
|
||||
if not dsn:
|
||||
return ""
|
||||
scheme, _, rest = dsn.partition("://")
|
||||
if not rest:
|
||||
return f"{scheme[:24]}://***"
|
||||
host_part = rest.split("@")[-1]
|
||||
host = host_part.split("/")[0]
|
||||
database = host_part.split("/", 1)[1].split("?")[0] if "/" in host_part else ""
|
||||
masked_host = host.split(":")[0]
|
||||
if len(masked_host) > 4:
|
||||
masked_host = f"{masked_host[:2]}***{masked_host[-2:]}"
|
||||
return f"{scheme}://***@{masked_host}/{database}"
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SecretKeyMissingError",
|
||||
"decrypt_secret",
|
||||
"encrypt_secret",
|
||||
"is_encrypted",
|
||||
"mask_dsn",
|
||||
"secrets_available",
|
||||
]
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
"""Read-only statement policy for the ``sql_read`` node.
|
||||
|
||||
The node is *only* allowed to run a single read query. We reject anything that
|
||||
is not a `SELECT`/`WITH`, anything containing a statement separator, and any
|
||||
write/DDL/administrative keyword — after stripping comments, because
|
||||
``SELECT 1 -- ; DROP TABLE t`` and ``/*!*/`` style tricks are exactly how these
|
||||
filters get bypassed.
|
||||
|
||||
Parameters are always bound (never interpolated), so this policy is a second
|
||||
line of defence rather than the only one. Production deployments should also
|
||||
point the data source at a read-only database account.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError
|
||||
|
||||
_FORBIDDEN_KEYWORDS = (
|
||||
"insert",
|
||||
"update",
|
||||
"delete",
|
||||
"drop",
|
||||
"alter",
|
||||
"create",
|
||||
"truncate",
|
||||
"replace into",
|
||||
"grant",
|
||||
"revoke",
|
||||
"merge",
|
||||
"call",
|
||||
"execute",
|
||||
"exec",
|
||||
"copy",
|
||||
"load_file",
|
||||
"outfile",
|
||||
"dumpfile",
|
||||
"into outfile",
|
||||
"pg_read_file",
|
||||
"pg_sleep",
|
||||
"sleep",
|
||||
"benchmark",
|
||||
"shutdown",
|
||||
"vacuum",
|
||||
"attach",
|
||||
"detach",
|
||||
"pragma",
|
||||
"set ",
|
||||
"lock",
|
||||
"do ",
|
||||
)
|
||||
|
||||
_BLOCK_COMMENT = re.compile(r"/\*.*?\*/", re.DOTALL)
|
||||
_LINE_COMMENT = re.compile(r"(--|#)[^\n]*")
|
||||
_PARAM_RE = re.compile(r":([A-Za-z_][A-Za-z0-9_]*)")
|
||||
|
||||
|
||||
def _denied(message: str, **details: object) -> WorkflowError:
|
||||
return WorkflowError("WORKFLOW_SQL_POLICY_DENIED", message, details=dict(details))
|
||||
|
||||
|
||||
def strip_comments(statement: str) -> str:
|
||||
return _LINE_COMMENT.sub(" ", _BLOCK_COMMENT.sub(" ", statement))
|
||||
|
||||
|
||||
def assert_read_only(statement: str) -> str:
|
||||
"""Validate and return the normalised single-statement query."""
|
||||
raw = (statement or "").strip()
|
||||
if not raw:
|
||||
raise _denied("SQL 语句为空")
|
||||
cleaned = strip_comments(raw).strip()
|
||||
if not cleaned:
|
||||
raise _denied("SQL 语句仅包含注释")
|
||||
body = cleaned.rstrip().rstrip(";").strip()
|
||||
if ";" in body:
|
||||
raise _denied("只允许执行单条语句")
|
||||
normalized = re.sub(r"\s+", " ", body.lower()).strip()
|
||||
if not (normalized.startswith("select") or normalized.startswith("with")):
|
||||
raise _denied("只允许 SELECT / WITH 查询")
|
||||
for keyword in _FORBIDDEN_KEYWORDS:
|
||||
token = keyword.strip()
|
||||
if re.search(rf"\b{re.escape(token)}\b", normalized):
|
||||
raise _denied("语句包含被禁止的关键字", keyword=token)
|
||||
return body
|
||||
|
||||
|
||||
def enforce_limit(statement: str, max_rows: int) -> str:
|
||||
"""Append a LIMIT when the query does not already bound its result size."""
|
||||
lowered = statement.lower()
|
||||
if re.search(r"\blimit\b", lowered) or re.search(r"\bfetch\s+first\b", lowered):
|
||||
return statement
|
||||
return f"{statement} LIMIT {int(max_rows)}"
|
||||
|
||||
|
||||
_FROM_JOIN_RE = re.compile(r"\b(?:from|join)\s+([A-Za-z_][\w.]*)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _table_name(raw: str) -> str:
|
||||
return raw.split(".")[-1].strip("`\"[]").lower()
|
||||
|
||||
|
||||
def assert_tables_allowed(statement: str, allowed_tables: list[str]) -> None:
|
||||
"""When the data source has a table allowlist, every FROM/JOIN target must
|
||||
appear on it. An empty allowlist means no extra restriction."""
|
||||
if not allowed_tables:
|
||||
return
|
||||
allowed = {item.lower() for item in allowed_tables} | {_table_name(item) for item in allowed_tables}
|
||||
for name in _FROM_JOIN_RE.findall(strip_comments(statement)):
|
||||
short = _table_name(name)
|
||||
if short not in allowed and name.lower() not in allowed:
|
||||
raise _denied("查询引用了未授权的表", table=name, allowedTables=list(allowed_tables))
|
||||
|
||||
|
||||
def declared_parameters(statement: str) -> set[str]:
|
||||
return set(_PARAM_RE.findall(strip_comments(statement)))
|
||||
|
||||
|
||||
def assert_parameters_bound(statement: str, parameters: dict[str, object]) -> None:
|
||||
missing = sorted(declared_parameters(statement) - set(parameters))
|
||||
if missing:
|
||||
raise WorkflowError(
|
||||
"WORKFLOW_SQL_FAILED",
|
||||
f"缺少查询参数:{'、'.join(missing)}",
|
||||
details={"missing": missing},
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"assert_parameters_bound",
|
||||
"assert_read_only",
|
||||
"assert_tables_allowed",
|
||||
"declared_parameters",
|
||||
"enforce_limit",
|
||||
"strip_comments",
|
||||
]
|
||||
@@ -0,0 +1,335 @@
|
||||
"""Publish-time structural validator for workflow graphs (phase 1).
|
||||
|
||||
Full resource-permission and SQL EXPLAIN checks land in later phases.
|
||||
This module never imports ``app.*``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict, deque
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from deerflow.workflows.errors import WorkflowError, WorkflowErrorBody
|
||||
from deerflow.workflows.expressions import collect_referenced_nodes
|
||||
from deerflow.workflows.schemas import ALL_NODE_TYPES, WorkflowGraph
|
||||
|
||||
ResourceChecker = Callable[[str, dict[str, Any]], "str | dict[str, Any] | None"]
|
||||
"""Return an error message when a node resource is missing/forbidden.
|
||||
|
||||
May instead return ``{"message": ..., "field": "config.agentId"}`` so API
|
||||
responses can point the canvas form at the exact offending field. ``None``
|
||||
means the node's resources check out."""
|
||||
|
||||
|
||||
class ValidationIssue(WorkflowErrorBody):
|
||||
"""Alias kept for readability in API responses."""
|
||||
|
||||
|
||||
def _collect_reachable(start_ids: set[str], adjacency: dict[str, list[str]]) -> set[str]:
|
||||
seen: set[str] = set()
|
||||
queue: deque[str] = deque(start_ids)
|
||||
while queue:
|
||||
node_id = queue.popleft()
|
||||
if node_id in seen:
|
||||
continue
|
||||
seen.add(node_id)
|
||||
for nxt in adjacency.get(node_id, []):
|
||||
if nxt not in seen:
|
||||
queue.append(nxt)
|
||||
return seen
|
||||
|
||||
|
||||
def _has_uncontrolled_cycle(graph: WorkflowGraph) -> bool:
|
||||
"""Detect cycles that are not entered via an explicit ``loop`` node.
|
||||
|
||||
Phase-1 heuristic: if the condensation of the graph (ignoring edges whose
|
||||
source is a ``loop`` node) still has a back-edge, reject.
|
||||
"""
|
||||
loop_ids = {n.id for n in graph.nodes if n.type == "loop"}
|
||||
adjacency: dict[str, list[str]] = defaultdict(list)
|
||||
for edge in graph.edges:
|
||||
if edge.source in loop_ids:
|
||||
continue
|
||||
adjacency[edge.source].append(edge.target)
|
||||
|
||||
visiting: set[str] = set()
|
||||
visited: set[str] = set()
|
||||
|
||||
def dfs(node_id: str) -> bool:
|
||||
visiting.add(node_id)
|
||||
for nxt in adjacency.get(node_id, []):
|
||||
if nxt in visiting:
|
||||
return True
|
||||
if nxt not in visited and dfs(nxt):
|
||||
return True
|
||||
visiting.discard(node_id)
|
||||
visited.add(node_id)
|
||||
return False
|
||||
|
||||
for node in graph.nodes:
|
||||
if node.id not in visited and dfs(node.id):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def validate_workflow_graph(
|
||||
raw: dict[str, Any] | WorkflowGraph,
|
||||
*,
|
||||
system_max_steps: int = 500,
|
||||
system_max_loop_iterations: int = 20,
|
||||
system_max_parallelism: int = 16,
|
||||
resource_checker: ResourceChecker | None = None,
|
||||
) -> list[ValidationIssue]:
|
||||
"""Validate a graph. Empty list means publishable."""
|
||||
|
||||
issues: list[ValidationIssue] = []
|
||||
|
||||
try:
|
||||
graph = raw if isinstance(raw, WorkflowGraph) else WorkflowGraph.model_validate(raw)
|
||||
except ValidationError as exc:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message="图结构不符合 schema v1.0",
|
||||
details={"errors": exc.errors()},
|
||||
)
|
||||
)
|
||||
return issues
|
||||
|
||||
starts = [n for n in graph.nodes if n.type == "start"]
|
||||
outputs = [n for n in graph.nodes if n.type == "output"]
|
||||
if len(starts) != 1:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message=f"工作流必须恰好有 1 个开始节点,当前为 {len(starts)}",
|
||||
details={"count": len(starts)},
|
||||
)
|
||||
)
|
||||
if len(outputs) < 1:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message="工作流至少需要 1 个输出节点",
|
||||
)
|
||||
)
|
||||
|
||||
unknown = [n for n in graph.nodes if n.type not in ALL_NODE_TYPES]
|
||||
for node in unknown:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message=f"未知节点类型: {node.type}",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
|
||||
if _has_uncontrolled_cycle(graph):
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_GRAPH_CYCLE",
|
||||
message="检测到未受控环;循环必须通过显式 loop 节点",
|
||||
)
|
||||
)
|
||||
|
||||
if starts:
|
||||
adjacency: dict[str, list[str]] = defaultdict(list)
|
||||
for edge in graph.edges:
|
||||
adjacency[edge.source].append(edge.target)
|
||||
reachable = _collect_reachable({starts[0].id}, adjacency)
|
||||
for node in graph.nodes:
|
||||
if node.id not in reachable:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_NODE_NOT_FOUND",
|
||||
message=f"节点不可从开始节点到达: {node.id}",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
for out in outputs:
|
||||
if out.id not in reachable:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message=f"输出节点不可达: {out.id}",
|
||||
node_id=out.id,
|
||||
)
|
||||
)
|
||||
|
||||
settings = graph.settings
|
||||
if settings.max_steps > system_max_steps:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_LIMIT_EXCEEDED",
|
||||
message=f"maxSteps 超过系统上限 {system_max_steps}",
|
||||
details={"maxSteps": settings.max_steps, "limit": system_max_steps},
|
||||
)
|
||||
)
|
||||
if settings.max_loop_iterations > system_max_loop_iterations:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_LIMIT_EXCEEDED",
|
||||
message=f"maxLoopIterations 超过系统上限 {system_max_loop_iterations}",
|
||||
details={
|
||||
"maxLoopIterations": settings.max_loop_iterations,
|
||||
"limit": system_max_loop_iterations,
|
||||
},
|
||||
)
|
||||
)
|
||||
if settings.max_parallelism > system_max_parallelism:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_LIMIT_EXCEEDED",
|
||||
message=f"maxParallelism 超过系统上限 {system_max_parallelism}",
|
||||
details={"maxParallelism": settings.max_parallelism, "limit": system_max_parallelism},
|
||||
)
|
||||
)
|
||||
|
||||
for node in graph.nodes:
|
||||
if node.type == "condition":
|
||||
branches = node.config.get("branches") or []
|
||||
ports = {str(b.get("name")) for b in branches if isinstance(b, dict) and b.get("name")}
|
||||
default_branch = node.config.get("defaultBranch") or node.config.get("default_branch")
|
||||
if default_branch:
|
||||
ports.add(str(default_branch))
|
||||
edge_ports = {e.source_port for e in graph.edges if e.source == node.id and e.source_port}
|
||||
missing = edge_ports - ports
|
||||
for port in missing:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_EDGE_INVALID",
|
||||
message=f"条件节点出口未声明: {port}",
|
||||
node_id=node.id,
|
||||
details={"sourcePort": port},
|
||||
)
|
||||
)
|
||||
if node.type == "loop":
|
||||
max_iter = node.config.get("maxIterations") or node.config.get("max_iterations")
|
||||
if max_iter is None:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message="loop 节点必须配置 maxIterations",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
body_entry = node.config.get("bodyEntry") or node.config.get("body_entry")
|
||||
if not body_entry:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_SCHEMA_INVALID",
|
||||
message="loop 节点必须配置 bodyEntry",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
elif body_entry not in graph.node_map():
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_NODE_NOT_FOUND",
|
||||
message=f"loop bodyEntry 不存在: {body_entry}",
|
||||
node_id=node.id,
|
||||
)
|
||||
)
|
||||
|
||||
if resource_checker is not None:
|
||||
verdict = resource_checker(node.type, node.config or {})
|
||||
if verdict:
|
||||
message = verdict if isinstance(verdict, str) else str(verdict.get("message") or "节点资源配置无效")
|
||||
field = verdict.get("field") if isinstance(verdict, dict) else None
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_RESOURCE_MISSING",
|
||||
message=message,
|
||||
node_id=node.id,
|
||||
details={"field": field} if field else None,
|
||||
)
|
||||
)
|
||||
|
||||
issues.extend(_check_upstream_references(graph))
|
||||
return issues
|
||||
|
||||
|
||||
def _predecessors(graph: WorkflowGraph) -> dict[str, set[str]]:
|
||||
preds: dict[str, set[str]] = defaultdict(set)
|
||||
for edge in graph.edges:
|
||||
preds[edge.target].add(edge.source)
|
||||
return preds
|
||||
|
||||
|
||||
def _ancestors(node_id: str, preds: dict[str, set[str]]) -> set[str]:
|
||||
seen: set[str] = set()
|
||||
stack = list(preds.get(node_id) or ())
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
stack.extend(preds.get(current) or ())
|
||||
return seen
|
||||
|
||||
|
||||
def _check_upstream_references(graph: WorkflowGraph) -> list[ValidationIssue]:
|
||||
"""§8.1: node input references must come from upstream nodes."""
|
||||
issues: list[ValidationIssue] = []
|
||||
known = graph.node_map()
|
||||
preds = _predecessors(graph)
|
||||
for node in graph.nodes:
|
||||
refs = collect_referenced_nodes(node.config)
|
||||
ancestors = _ancestors(node.id, preds)
|
||||
for ref in sorted(refs):
|
||||
if ref not in known:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_NODE_NOT_FOUND",
|
||||
message=f"表达式引用了不存在的节点: {ref}",
|
||||
node_id=node.id,
|
||||
details={"referencedNodeId": ref},
|
||||
)
|
||||
)
|
||||
continue
|
||||
if ref == node.id and node.type != "loop":
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_EXPRESSION_INVALID",
|
||||
message="节点不能引用自身输出",
|
||||
node_id=node.id,
|
||||
details={"referencedNodeId": ref},
|
||||
)
|
||||
)
|
||||
continue
|
||||
if ref != node.id and ref not in ancestors:
|
||||
issues.append(
|
||||
ValidationIssue(
|
||||
code="WORKFLOW_EXPRESSION_INVALID",
|
||||
message=f"节点引用必须来自上游,{ref} 不在 {node.id} 的上游",
|
||||
node_id=node.id,
|
||||
details={"referencedNodeId": ref},
|
||||
)
|
||||
)
|
||||
return issues
|
||||
|
||||
|
||||
def assert_publishable(raw: dict[str, Any] | WorkflowGraph, **kwargs: Any) -> WorkflowGraph:
|
||||
"""Validate and return the typed graph, or raise ``WorkflowError``."""
|
||||
|
||||
issues = validate_workflow_graph(raw, **kwargs)
|
||||
if issues:
|
||||
first = issues[0]
|
||||
raise WorkflowError(
|
||||
first.code,
|
||||
first.message,
|
||||
node_id=first.node_id,
|
||||
details={"issues": [i.model_dump(by_alias=True) for i in issues]},
|
||||
)
|
||||
return raw if isinstance(raw, WorkflowGraph) else WorkflowGraph.model_validate(raw)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ResourceChecker",
|
||||
"ValidationIssue",
|
||||
"assert_publishable",
|
||||
"validate_workflow_graph",
|
||||
]
|
||||
Reference in new issue
Block a user