代码提交

This commit is contained in:
董宏凯 committed 2026-08-30 01:25:20 +08:00
1 parent 4ac7bc21cd
commit 17cf849361
110 files changed
+17007 -518

No files matched your search

@@ -0,0 +1,64 @@
"""Create workflow_definitions / workflow_versions tables.
Revision ID: 20260828_01
Revises: 20260825_01
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_01"
down_revision = "20260825_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
op.create_table(
"workflow_definitions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=255), nullable=False),
sa.Column("description", sa.Text(), nullable=False, server_default=""),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
sa.Column("draft_graph_json", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_workflow_definitions_owner_id", "workflow_definitions", ["owner_id"])
op.create_index("ix_workflow_definitions_status", "workflow_definitions", ["status"])
if not inspector.has_table("workflow_versions"):
op.create_table(
"workflow_versions",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_number", sa.Integer(), nullable=False),
sa.Column("graph_json", sa.Text(), nullable=False),
sa.Column("graph_hash", sa.String(length=64), nullable=False),
sa.Column("input_schema", sa.Text(), nullable=True),
sa.Column("output_schema", sa.Text(), nullable=True),
sa.Column("published_by", sa.String(length=64), nullable=True),
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
)
op.create_index("ix_workflow_versions_workflow_id", "workflow_versions", ["workflow_id"])
op.create_index("ix_workflow_versions_graph_hash", "workflow_versions", ["graph_hash"])
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_versions"):
op.drop_table("workflow_versions")
if inspector.has_table("workflow_definitions"):
op.drop_table("workflow_definitions")
@@ -0,0 +1,127 @@
"""Create workflow run / node-run / event / artifact tables.
Revision ID: 20260828_02
Revises: 20260828_01
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_02"
down_revision = "20260828_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_runs"):
op.create_table(
"workflow_runs",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("workflow_version_id", sa.String(length=64), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
sa.Column("owner_id", sa.String(length=64), nullable=False),
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
sa.Column("input_json", sa.Text(), nullable=True),
sa.Column("output_json", sa.Text(), nullable=True),
sa.Column("error_json", sa.Text(), nullable=True),
sa.Column("context_json", sa.Text(), nullable=True),
sa.Column("pending_input_json", sa.Text(), nullable=True),
sa.Column("resume_token", sa.String(length=64), nullable=True),
sa.Column("lease_owner", sa.String(length=64), nullable=True),
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("attempt", sa.Integer(), nullable=False, server_default="0"),
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
sa.Column("version", sa.Integer(), nullable=False, server_default="0"),
sa.Column("retry_of_run_id", sa.String(length=64), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),
)
op.create_index("ix_workflow_runs_workflow_id", "workflow_runs", ["workflow_id"])
op.create_index("ix_workflow_runs_workflow_version_id", "workflow_runs", ["workflow_version_id"])
op.create_index("ix_workflow_runs_owner_id", "workflow_runs", ["owner_id"])
op.create_index("ix_workflow_runs_status", "workflow_runs", ["status"])
op.create_index("ix_workflow_runs_resume_token", "workflow_runs", ["resume_token"])
if not inspector.has_table("workflow_node_runs"):
op.create_table(
"workflow_node_runs",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=False),
sa.Column("node_type", sa.String(length=32), nullable=False),
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
sa.Column("iteration", sa.Integer(), nullable=False, server_default="0"),
sa.Column("is_current", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("input_snapshot", sa.Text(), nullable=True),
sa.Column("output_snapshot", sa.Text(), nullable=True),
sa.Column("error_json", sa.Text(), nullable=True),
sa.Column("parent_node_run_id", sa.String(length=64), nullable=True),
sa.Column("duration_ms", sa.Integer(), nullable=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint(
"run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"
),
)
op.create_index("ix_workflow_node_runs_run_id", "workflow_node_runs", ["run_id"])
op.create_index("ix_workflow_node_runs_status", "workflow_node_runs", ["status"])
if not inspector.has_table("workflow_run_events"):
op.create_table(
"workflow_run_events",
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("workflow_id", sa.String(length=64), nullable=False),
sa.Column("version_id", sa.String(length=64), nullable=False),
sa.Column("seq", sa.Integer(), nullable=False),
sa.Column("event_type", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=True),
sa.Column("node_run_id", sa.String(length=64), nullable=True),
sa.Column("payload_json", sa.Text(), nullable=False),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
)
op.create_index("ix_workflow_run_events_run_id", "workflow_run_events", ["run_id"])
op.create_index("ix_workflow_run_events_run_seq", "workflow_run_events", ["run_id", "seq"])
if not inspector.has_table("workflow_run_artifacts"):
op.create_table(
"workflow_run_artifacts",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("run_id", sa.String(length=64), nullable=False),
sa.Column("node_id", sa.String(length=128), nullable=False),
sa.Column("name", sa.String(length=512), nullable=False),
sa.Column("mime_type", sa.String(length=191), nullable=False),
sa.Column("path", sa.Text(), nullable=False),
sa.Column("size_bytes", sa.Integer(), nullable=True),
sa.Column("preview", sa.Text(), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
)
op.create_index("ix_workflow_run_artifacts_run_id", "workflow_run_artifacts", ["run_id"])
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
for table in (
"workflow_run_artifacts",
"workflow_run_events",
"workflow_node_runs",
"workflow_runs",
):
if inspector.has_table(table):
op.drop_table(table)
@@ -0,0 +1,49 @@
"""Create workflow_data_sources table.
Revision ID: 20260828_03
Revises: 20260828_02
Create Date: 2026-08-28
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_03"
down_revision = "20260828_02"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_data_sources"):
return
op.create_table(
"workflow_data_sources",
sa.Column("id", sa.String(length=64), primary_key=True),
sa.Column("name", sa.String(length=191), nullable=False),
sa.Column("description", sa.String(length=512), nullable=False, server_default=""),
sa.Column("kind", sa.String(length=32), nullable=False, server_default="sql"),
sa.Column("driver", sa.String(length=64), nullable=False, server_default=""),
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
sa.Column("secret_encrypted", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("masked_target", sa.String(length=512), nullable=False, server_default=""),
sa.Column("max_rows", sa.Integer(), nullable=False, server_default="1000"),
sa.Column("allowed_tables", sa.Text(), nullable=True),
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")),
sa.Column("owner_id", sa.String(length=64), nullable=True),
sa.Column("created_by", sa.String(length=64), nullable=True),
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
sa.UniqueConstraint("name", name="uq_workflow_data_sources_name"),
)
op.create_index("ix_workflow_data_sources_owner_id", "workflow_data_sources", ["owner_id"])
def downgrade() -> None:
bind = op.get_bind()
if sa.inspect(bind).has_table("workflow_data_sources"):
op.drop_table("workflow_data_sources")
@@ -0,0 +1,66 @@
"""Add workflow counter columns: version allocator + event seq counter.
Revision ID: 20260828_04
Revises: 20260828_03
Create Date: 2026-08-28
Functional-robustness remediation (audit items P1-02 / P1-03):
* ``workflow_definitions.next_version_number`` — publish allocates the version
number by CAS-bumping this counter instead of ``MAX(version_number) + 1``,
so two concurrent publishes get adjacent distinct numbers instead of a
unique-constraint 500.
* ``workflow_runs.next_event_seq`` — the event log draws each event's ``seq``
from this counter in the insert's own transaction instead of ``MAX(seq)+1``.
Both columns carry a ``server_default`` so existing databases get them via the
startup ``_ensure_orm_columns_sync`` sweep even without running Alembic; rows
keep the default (1 / 0) and the first write self-heals from the existing
maximum on unique-conflict.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260828_04"
down_revision = "20260828_03"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_definitions"):
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
if "next_version_number" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("next_version_number", sa.Integer(), nullable=False, server_default="1"),
)
if inspector.has_table("workflow_runs"):
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
if "next_event_seq" not in existing:
op.add_column(
"workflow_runs",
sa.Column("next_event_seq", sa.Integer(), nullable=False, server_default="0"),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if inspector.has_table("workflow_definitions"):
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
if "next_version_number" in existing:
op.drop_column("workflow_definitions", "next_version_number")
if inspector.has_table("workflow_runs"):
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
if "next_event_seq" in existing:
op.drop_column("workflow_runs", "next_event_seq")
@@ -0,0 +1,50 @@
"""Add non-secret HTTP metadata to workflow data sources.
Revision ID: 20260829_01
Revises: 20260828_04
Create Date: 2026-08-29
HTTP request headers remain in the encrypted secret blob. The endpoint and
method allowlist are intentionally separate non-secret fields so the canvas
can show a reusable interface resource without exposing credentials.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260829_01"
down_revision = "20260828_04"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_data_sources"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
if "base_url" not in existing:
op.add_column(
"workflow_data_sources",
sa.Column("base_url", sa.String(length=2048), nullable=False, server_default=""),
)
if "allowed_methods" not in existing:
op.add_column(
"workflow_data_sources",
sa.Column("allowed_methods", sa.Text(), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_data_sources"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
if "allowed_methods" in existing:
op.drop_column("workflow_data_sources", "allowed_methods")
if "base_url" in existing:
op.drop_column("workflow_data_sources", "base_url")
@@ -0,0 +1,51 @@
"""Add Coze canvas draft column to workflow_definitions.
Revision ID: 20260829_02
Revises: 20260829_01
Create Date: 2026-08-29
The canvas document (positions, Coze form internals) is stored separately from
the executable WorkflowGraph so neither representation can impersonate the
other. ``published_canvas_schema_hash`` records which canvas the current
published version was compiled from, for audit only.
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = "20260829_02"
down_revision = "20260829_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "draft_canvas_schema_json" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
)
if "published_canvas_schema_hash" not in existing:
op.add_column(
"workflow_definitions",
sa.Column("published_canvas_schema_hash", sa.String(length=64), nullable=True),
)
def downgrade() -> None:
bind = op.get_bind()
inspector = sa.inspect(bind)
if not inspector.has_table("workflow_definitions"):
return
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
if "published_canvas_schema_hash" in existing:
op.drop_column("workflow_definitions", "published_canvas_schema_hash")
if "draft_canvas_schema_json" in existing:
op.drop_column("workflow_definitions", "draft_canvas_schema_json")
@@ -50,6 +50,14 @@ from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
from deerflow.persistence.light_apps.model import LightAppRow
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
@@ -128,6 +136,13 @@ __all__ = [
"KnowledgeSourceRow",
"KnowledgeTagRow",
"LightAppRow",
"WorkflowDataSourceRow",
"WorkflowDefinitionRow",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunEventRow",
"WorkflowRunRow",
"WorkflowVersionRow",
"LlmCallMetricRow",
"LlmWikiKnowledgeBaseRow",
"LlmWikiConversationDepositRow",
@@ -0,0 +1,19 @@
"""Workflow data-source persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.persistence.workflow_data_sources.store import (
MemoryWorkflowDataSourceStore,
SqlWorkflowDataSourceStore,
WorkflowDataSourceStore,
make_workflow_data_source_store,
)
__all__ = [
"MemoryWorkflowDataSourceStore",
"SqlWorkflowDataSourceStore",
"WorkflowDataSourceRow",
"WorkflowDataSourceStore",
"make_workflow_data_source_store",
]
@@ -0,0 +1,51 @@
"""ORM model for workflow SQL data sources and HTTP credentials."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowDataSourceRow(Base):
"""A read-only database connection usable by ``sql_read`` nodes.
``secret_ciphertext`` holds the DSN encrypted with the workflow secret key;
it is never returned by any API. ``owner_id`` NULL means the data source is
shared (admin-managed)."""
__tablename__ = "workflow_data_sources"
__table_args__ = (UniqueConstraint("name", name="uq_workflow_data_sources_name"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(191), nullable=False)
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="sql")
driver: Mapped[str] = mapped_column(String(64), nullable=False, default="")
base_url: Mapped[str] = mapped_column(
String(2048),
nullable=False,
default="",
server_default=sa_text("''"),
)
# Nullable so the startup column synchronizer can safely patch existing databases.
allowed_methods: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
secret_ciphertext: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
secret_encrypted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
masked_target: Mapped[str] = mapped_column(String(512), nullable=False, default="")
max_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=1000, server_default=sa_text("1000"))
allowed_tables: Mapped[list] = mapped_column(PortableJSON(), default=list)
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
owner_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,293 @@
"""Workflow data-source store (SQL + in-memory).
Secrets are encrypted before they reach the database and only ever decrypted
inside :meth:`resolve_dsn`, which the run executor calls. No read path returns
the ciphertext.
"""
from __future__ import annotations
import copy
import uuid
from abc import ABC, abstractmethod
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import select
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
from deerflow.workflows.security.secret_box import decrypt_secret, encrypt_secret, mask_dsn
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _public_dict(row: WorkflowDataSourceRow) -> dict[str, Any]:
return {
"id": row.id,
"name": row.name,
"description": row.description or "",
"kind": row.kind,
"driver": row.driver or "",
"base_url": row.base_url or "",
"allowed_methods": list(row.allowed_methods or []),
"masked_target": row.masked_target or "",
"secret_encrypted": bool(row.secret_encrypted),
"max_rows": row.max_rows,
"allowed_tables": list(row.allowed_tables or []),
"enabled": bool(row.enabled),
"owner_id": row.owner_id,
"created_by": row.created_by,
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
class WorkflowDataSourceStore(ABC):
@abstractmethod
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_source(self, source_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def delete_source(self, source_id: str) -> bool:
raise NotImplementedError
@abstractmethod
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
"""Decrypted connection info for the executor. Never expose via HTTP."""
raise NotImplementedError
class SqlWorkflowDataSourceStore(WorkflowDataSourceStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
stmt = select(WorkflowDataSourceRow).order_by(WorkflowDataSourceRow.name.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
visible = []
for row in rows:
if row.owner_id is None and include_shared:
visible.append(row)
elif owner_id is not None and row.owner_id == owner_id:
visible.append(row)
return [_public_dict(row) for row in visible]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
return _public_dict(row) if row else None
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
dsn = str(data.get("dsn") or "")
kind = str(data.get("kind") or "sql")
base_url = str(data.get("base_url") or "")
row = WorkflowDataSourceRow(
id=str(data.get("id") or uuid.uuid4()),
name=str(data["name"])[:191],
description=str(data.get("description") or "")[:512],
kind=kind,
driver=str(
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
)[:64],
base_url=base_url[:2048],
allowed_methods=list(data.get("allowed_methods") or []),
secret_ciphertext=encrypt_secret(dsn) if dsn else "",
secret_encrypted=bool(dsn),
masked_target=(base_url if kind == "http" else mask_dsn(dsn))[:512],
max_rows=int(data.get("max_rows") or 1000),
allowed_tables=list(data.get("allowed_tables") or []),
enabled=bool(data.get("enabled", True)),
owner_id=data.get("owner_id"),
created_by=data.get("created_by"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = _public_dict(row)
await session.commit()
return out
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None:
return None
if data.get("name"):
row.name = str(data["name"])[:191]
if "description" in data:
row.description = str(data.get("description") or "")[:512]
if "base_url" in data:
row.base_url = str(data.get("base_url") or "")[:2048]
if row.kind == "http":
row.masked_target = row.base_url[:512]
if "allowed_methods" in data:
row.allowed_methods = list(data.get("allowed_methods") or [])
if data.get("dsn"):
dsn = str(data["dsn"])
row.secret_ciphertext = encrypt_secret(dsn)
row.secret_encrypted = True
if row.kind == "http":
row.driver = "HTTP API"
else:
row.masked_target = mask_dsn(dsn)[:512]
row.driver = dsn.split("://", 1)[0][:64]
if data.get("max_rows") is not None:
row.max_rows = int(data["max_rows"])
if "allowed_tables" in data:
row.allowed_tables = list(data.get("allowed_tables") or [])
if "enabled" in data:
row.enabled = bool(data["enabled"])
row.updated_at = datetime.now(UTC)
await session.flush()
out = _public_dict(row)
await session.commit()
return out
async def delete_source(self, source_id: str) -> bool:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None:
return False
await session.delete(row)
await session.commit()
return True
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDataSourceRow, source_id)
if row is None or not row.enabled or not row.secret_ciphertext:
return None
return {
"id": row.id,
"dsn": decrypt_secret(row.secret_ciphertext),
"kind": row.kind,
"base_url": row.base_url or "",
"allowed_methods": list(row.allowed_methods or []),
"max_rows": row.max_rows,
"allowed_tables": list(row.allowed_tables or []),
"owner_id": row.owner_id,
}
class MemoryWorkflowDataSourceStore(WorkflowDataSourceStore):
def __init__(self) -> None:
self._rows: dict[str, dict[str, Any]] = {}
self._secrets: dict[str, str] = {}
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
out = []
for row in self._rows.values():
if row["owner_id"] is None and include_shared:
out.append(row)
elif owner_id is not None and row["owner_id"] == owner_id:
out.append(row)
return [copy.deepcopy(r) for r in sorted(out, key=lambda r: r["name"])]
async def get_source(self, source_id: str) -> dict[str, Any] | None:
row = self._rows.get(source_id)
return copy.deepcopy(row) if row else None
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
source_id = str(data.get("id") or uuid.uuid4())
dsn = str(data.get("dsn") or "")
kind = str(data.get("kind") or "sql")
base_url = str(data.get("base_url") or "")
now = _iso(datetime.now(UTC))
row = {
"id": source_id,
"name": str(data["name"]),
"description": str(data.get("description") or ""),
"kind": kind,
"driver": str(
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
),
"base_url": base_url,
"allowed_methods": list(data.get("allowed_methods") or []),
"masked_target": base_url if kind == "http" else mask_dsn(dsn),
"secret_encrypted": bool(dsn),
"max_rows": int(data.get("max_rows") or 1000),
"allowed_tables": list(data.get("allowed_tables") or []),
"enabled": bool(data.get("enabled", True)),
"owner_id": data.get("owner_id"),
"created_by": data.get("created_by"),
"created_at": now,
"updated_at": now,
}
self._rows[source_id] = row
self._secrets[source_id] = dsn
return copy.deepcopy(row)
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._rows.get(source_id)
if row is None:
return None
for key in (
"name",
"description",
"base_url",
"allowed_methods",
"max_rows",
"allowed_tables",
"enabled",
):
if key in data and data[key] is not None:
row[key] = data[key]
if "base_url" in data and row["kind"] == "http":
row["masked_target"] = str(row["base_url"])
if data.get("dsn"):
self._secrets[source_id] = str(data["dsn"])
if row["kind"] == "http":
row["driver"] = "HTTP API"
else:
row["masked_target"] = mask_dsn(str(data["dsn"]))
row["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(row)
async def delete_source(self, source_id: str) -> bool:
self._secrets.pop(source_id, None)
return self._rows.pop(source_id, None) is not None
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
row = self._rows.get(source_id)
if row is None or not row["enabled"]:
return None
return {
"id": source_id,
"dsn": self._secrets.get(source_id, ""),
"kind": row["kind"],
"base_url": row.get("base_url", ""),
"allowed_methods": list(row.get("allowed_methods") or []),
"max_rows": row["max_rows"],
"allowed_tables": row["allowed_tables"],
"owner_id": row["owner_id"],
}
def make_workflow_data_source_store(session_factory):
if session_factory is None:
return MemoryWorkflowDataSourceStore()
return SqlWorkflowDataSourceStore(session_factory)
__all__ = [
"MemoryWorkflowDataSourceStore",
"SqlWorkflowDataSourceStore",
"WorkflowDataSourceStore",
"make_workflow_data_source_store",
]
@@ -0,0 +1,22 @@
"""Workflow run event-log persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
__all__ = [
"MemoryWorkflowEventStore",
"SqlWorkflowEventStore",
"WorkflowEventStore",
"WorkflowRunEventRow",
"make_workflow_event_store",
]
def make_workflow_event_store(session_factory, run_store=None):
if session_factory is None:
return MemoryWorkflowEventStore(run_store=run_store)
return SqlWorkflowEventStore(session_factory)
@@ -0,0 +1,42 @@
"""Abstract workflow run event store (durable, replayable)."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowEventStore(ABC):
@abstractmethod
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
"""Persist one event, assigning the next per-run ``seq``."""
raise NotImplementedError
@abstractmethod
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def max_seq(self, run_id: str) -> int:
raise NotImplementedError
@abstractmethod
async def purge_before(self, *, older_than_days: int) -> int:
"""Retention purge by event age; returns deleted row count."""
raise NotImplementedError
@abstractmethod
async def purge_for_runs(self, run_ids: list[str]) -> int:
"""Delete every event belonging to the given runs (used when the runs
themselves are retained away). Returns deleted row count."""
raise NotImplementedError
@@ -0,0 +1,103 @@
"""In-memory workflow event log (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
class MemoryWorkflowEventStore(WorkflowEventStore):
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
self._events: dict[str, list[dict[str, Any]]] = {}
self._max_per_run = max_per_run
self._run_store = run_store
self._lock = asyncio.Lock()
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
async with self._lock:
rows = self._events.setdefault(run_id, [])
event = {
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
"run_id": run_id,
"workflow_id": workflow_id,
"version_id": version_id,
"event_type": event_type,
"node_id": node_id,
"node_run_id": node_run_id,
"payload": payload or {},
"created_at": datetime.now(UTC).isoformat(),
}
rows.append(event)
if len(rows) > self._max_per_run:
del rows[: len(rows) - self._max_per_run]
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
# where the counter bump and the insert share one transaction).
try:
await self._run_store.note_event_seq(run_id, event["seq"])
except Exception: # noqa: BLE001 - counter sync is best effort
pass
return copy.deepcopy(event)
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
return [copy.deepcopy(r) for r in rows[:limit]]
async def max_seq(self, run_id: str) -> int:
rows = self._events.get(run_id, [])
return int(rows[-1]["seq"]) if rows else 0
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted = 0
for run_id, rows in list(self._events.items()):
kept: list[dict[str, Any]] = []
for row in rows:
created = _parse_dt(row.get("created_at"))
if created is not None and created < cutoff:
deleted += 1
else:
kept.append(row)
if kept:
self._events[run_id] = kept
else:
del self._events[run_id]
return deleted
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._lock:
deleted = 0
for run_id in run_ids:
rows = self._events.pop(run_id, None)
if rows:
deleted += len(rows)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,34 @@
"""ORM model for the durable workflow run event log."""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class WorkflowRunEventRow(Base):
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
__tablename__ = "workflow_run_events"
__table_args__ = (
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,157 @@
"""SQLAlchemy-backed workflow event log."""
from __future__ import annotations
import json
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
_MAX_PAYLOAD_CHARS = 64_000
_APPEND_ATTEMPTS = 5
def _dump_payload(payload: dict[str, Any] | None) -> str:
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
if len(text) > _MAX_PAYLOAD_CHARS:
return json.dumps(
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
ensure_ascii=False,
separators=(",", ":"),
)
return text
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
try:
payload = json.loads(row.payload_json or "{}")
except json.JSONDecodeError:
payload = {}
return {
"seq": row.seq,
"run_id": row.run_id,
"workflow_id": row.workflow_id,
"version_id": row.version_id,
"event_type": row.event_type,
"node_id": row.node_id,
"node_run_id": row.node_run_id,
"payload": payload if isinstance(payload, dict) else {},
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
}
class SqlWorkflowEventStore(WorkflowEventStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
max_retries: int = _APPEND_ATTEMPTS,
) -> dict[str, Any]:
"""Persist one event, taking its ``seq`` from the run's own counter.
``MAX(seq) + 1`` let two appenders draw the same number and burn a
unique-constraint retry; here the counter increment and the insert share
one transaction, and the row write-lock held by the UPDATE serialises
concurrent appenders until commit — so seqs are strictly increasing,
unique and gap-free per run, in commit order.
"""
text = _dump_payload(payload)
for _ in range(max_retries):
async with self._sf() as session:
seq = await self._take_seq(session, run_id)
row = WorkflowRunEventRow(
run_id=run_id,
workflow_id=workflow_id,
version_id=version_id,
seq=seq,
event_type=event_type,
node_id=node_id,
node_run_id=node_run_id,
payload_json=text,
)
session.add(row)
try:
await session.flush()
except IntegrityError:
# Only reachable for legacy rows whose counter was backfilled
# to 0 while events already existed. Re-sync the counter to
# MAX(seq) in its own committed transaction, then retry.
await session.rollback()
await self._resync_counter(run_id)
continue
out = event_to_dict(row)
await session.commit()
return out
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
async def _take_seq(self, session: Any, run_id: str) -> int:
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
missing (event store used standalone in tests)."""
result = await session.execute(
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
)
if result.rowcount:
# Our own uncommitted increment; the row lock guards it until commit.
current = (
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
).scalar()
return int(current or 1)
current = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar()
return int(current or 0) + 1
async def _resync_counter(self, run_id: str) -> None:
"""Point a legacy/stale run counter back at MAX(seq)."""
async with self._sf() as session:
max_seq = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar() or 0
await session.execute(
update(WorkflowRunRow)
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
.values(next_event_seq=int(max_seq))
)
await session.commit()
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [event_to_dict(r) for r in rows]
async def max_seq(self, run_id: str) -> int:
async with self._sf() as session:
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
return int(value or 0)
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
await session.commit()
return int(result.rowcount or 0)
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
await session.commit()
return int(result.rowcount or 0)
@@ -0,0 +1,38 @@
"""Workflow run / node-run / artifact persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunConflictError,
WorkflowRunNotFoundError,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
__all__ = [
"CANCELLABLE_RUN_STATUSES",
"MemoryWorkflowRunStore",
"SqlWorkflowRunStore",
"TERMINAL_RUN_STATUSES",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunConflictError",
"WorkflowRunNotFoundError",
"WorkflowRunRow",
"WorkflowRunStore",
"make_workflow_run_store",
]
def make_workflow_run_store(session_factory):
if session_factory is None:
return MemoryWorkflowRunStore()
return SqlWorkflowRunStore(session_factory)
@@ -0,0 +1,161 @@
"""Abstract workflow run / node-run / artifact store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
class WorkflowRunNotFoundError(Exception):
def __init__(self, run_id: str) -> None:
super().__init__(f"workflow run not found: {run_id}")
self.run_id = run_id
class WorkflowRunConflictError(Exception):
"""Raised when a state transition is not legal for the current status."""
def __init__(self, run_id: str, status: str) -> None:
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
self.run_id = run_id
self.status = status
class WorkflowRunStore(ABC):
# ── runs ────────────────────────────────────────────────────────────
@abstractmethod
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
"""Create a run. Returns ``(run, created)``; an existing
``idempotency_key`` yields the stored run with ``created=False``."""
raise NotImplementedError
@abstractmethod
async def get_run(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
raise NotImplementedError
@abstractmethod
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
raise NotImplementedError
@abstractmethod
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
"""CAS update. When ``lease_owner`` is given the row must still be leased
by that owner. Terminal statuses clear the lease."""
raise NotImplementedError
@abstractmethod
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Pause atomically: status + token + pending descriptor (+ optional run
context) in one conditional update, so a concurrent resume can never be
overwritten by a follow-up context write."""
raise NotImplementedError
@abstractmethod
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
``queued``. Returns None when the token is wrong/already used."""
raise NotImplementedError
@abstractmethod
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic human-input resume in a single conditional update.
Validates the token, stores ``payload`` as ``context.resume_payload``
(replacing the whole run context with ``context`` when given), clears the
token/pending descriptor and flips the run to ``queued`` — all under
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
the resumed run on success, ``None`` when the token is wrong, already
used, or the run is no longer awaiting input (caller maps that to 409).
"""
raise NotImplementedError
# ── node runs ───────────────────────────────────────────────────────
@abstractmethod
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
raise NotImplementedError
@abstractmethod
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
"""Mark node runs stale (used when a loop re-enters its body)."""
raise NotImplementedError
# ── artifacts ───────────────────────────────────────────────────────
@abstractmethod
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
"""Fetch one artifact row scoped to its run (authorization key)."""
raise NotImplementedError
# ── retention ───────────────────────────────────────────────────────
@abstractmethod
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
"""Delete finished runs older than the cutoff, with their node runs and
artifact rows. Only terminal runs are eligible. Returns deleted run ids
(bounded per call so a sweeper can loop)."""
raise NotImplementedError
@@ -0,0 +1,388 @@
"""In-memory workflow run store (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
class MemoryWorkflowRunStore(WorkflowRunStore):
def __init__(self) -> None:
self._runs: dict[str, dict[str, Any]] = {}
self._node_runs: dict[str, list[dict[str, Any]]] = {}
self._artifacts: dict[str, list[dict[str, Any]]] = {}
self._lock = asyncio.Lock()
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
async with self._lock:
key = data.get("idempotency_key")
if key:
for run in self._runs.values():
if run.get("idempotency_key") == str(key):
return copy.deepcopy(run), False
now = datetime.now(UTC)
run_id = str(data.get("id") or uuid.uuid4())
run = {
"id": run_id,
"workflow_id": str(data["workflow_id"]),
"workflow_version_id": str(data["workflow_version_id"]),
"status": str(data.get("status") or "queued"),
"owner_id": str(data["owner_id"]),
"idempotency_key": str(key) if key else None,
"input": data.get("input") or {},
"output": None,
"error": None,
"context": data.get("context") or {},
"pending_input": None,
"resume_token": None,
"lease_owner": None,
"lease_until": None,
"heartbeat_at": None,
"attempt": 0,
"max_attempts": int(data.get("max_attempts") or 3),
"version": 0,
"retry_of_run_id": data.get("retry_of_run_id"),
"next_event_seq": 0,
"started_at": None,
"finished_at": None,
"created_at": _iso(now),
"updated_at": _iso(now),
"_created": now,
}
self._runs[run_id] = run
return copy.deepcopy(run), True
async def get_run(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
return copy.deepcopy(run) if run else None
async def note_event_seq(self, run_id: str, seq: int) -> None:
"""Mirror the SQL counter semantics for no-DB mode: the memory event
store reports each handed-out seq so run views can expose lastSeq."""
async with self._lock:
run = self._runs.get(run_id)
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
run["next_event_seq"] = int(seq)
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
if workflow_id:
rows = [r for r in rows if r["workflow_id"] == workflow_id]
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
out: list[str] = []
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
if run["status"] == "queued":
out.append(run["id"])
elif run["status"] == "running" and run["lease_until"]:
if datetime.fromisoformat(run["lease_until"]) < now:
out.append(run["id"])
if len(out) >= limit:
break
return out
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
now = datetime.now(UTC)
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
if not claimable:
return None
run.update(
status="running",
lease_owner=lease_owner,
lease_until=_iso(lease_until),
heartbeat_at=_iso(now),
attempt=run["attempt"] + 1,
version=run["version"] + 1,
started_at=run["started_at"] or _iso(now),
updated_at=_iso(now),
)
return copy.deepcopy(run)
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
run = self._runs.get(run_id)
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
return False
run["lease_until"] = _iso(lease_until)
run["heartbeat_at"] = _iso(datetime.now(UTC))
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
mapping = {
"output_json": "output",
"error_json": "error",
"context_json": "context",
"pending_input_json": "pending_input",
}
for key, value in fields.items():
run[mapping.get(key, key)] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
run["lease_owner"] = None
run["lease_until"] = None
run["resume_token"] = None
run["pending_input"] = None
run["finished_at"] = run.get("finished_at") or _iso(now)
for key in ("started_at", "finished_at"):
if isinstance(run.get(key), datetime):
run[key] = _iso(run[key])
run["version"] += 1
run["updated_at"] = _iso(now)
return copy.deepcopy(run)
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None:
return None
if run["status"] in ("cancel_requested", "cancelled"):
return copy.deepcopy(run)
if run["status"] not in CANCELLABLE_RUN_STATUSES:
return None
run["status"] = "cancel_requested"
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None or run["status"] != "cancel_requested":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input=None,
finished_at=_iso(now),
updated_at=_iso(now),
)
run["version"] += 1
return copy.deepcopy(run)
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "running":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="awaiting_input",
pending_input=copy.deepcopy(pending_input),
resume_token=resume_token,
lease_owner=None,
lease_until=None,
updated_at=_iso(now),
)
if context is not None:
run["context"] = copy.deepcopy(context)
run["version"] += 1
return copy.deepcopy(run)
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
run.update(status="queued", resume_token=None, pending_input=None)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
base_context = context if context is not None else run.get("context") or {}
run.update(
context={**base_context, "resume_payload": copy.deepcopy(payload)},
status="queued",
resume_token=None,
pending_input=None,
)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
rows = self._node_runs.setdefault(run_id, [])
existing = next(
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
None,
)
if existing is None:
existing = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": run_id,
"node_id": node_id,
"node_type": str(data.get("node_type") or "unknown"),
"status": "queued",
"attempt": attempt,
"iteration": iteration,
"is_current": True,
"input": None,
"output": None,
"error": None,
"parent_node_run_id": None,
"duration_ms": None,
"started_at": None,
"finished_at": None,
"created_at": _iso(datetime.now(UTC)),
}
rows.append(existing)
for key in (
"status",
"node_type",
"input",
"output",
"error",
"parent_node_run_id",
"duration_ms",
"started_at",
"finished_at",
):
if key in data and data[key] is not None:
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
if "is_current" in data:
existing["is_current"] = bool(data["is_current"])
return copy.deepcopy(existing)
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
rows = self._node_runs.get(run_id, [])
if current_only:
rows = [r for r in rows if r["is_current"]]
return [copy.deepcopy(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
count = 0
for row in self._node_runs.get(run_id, []):
if row["node_id"] in node_ids and row["is_current"]:
row["is_current"] = False
count += 1
return count
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": str(data["run_id"]),
"node_id": str(data.get("node_id") or ""),
"name": str(data.get("name") or ""),
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
"path": str(data.get("path") or ""),
"size_bytes": data.get("size_bytes"),
"preview": data.get("preview"),
"created_at": _iso(datetime.now(UTC)),
}
self._artifacts.setdefault(row["run_id"], []).append(row)
return copy.deepcopy(row)
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
for row in self._artifacts.get(run_id, []):
if row["id"] == artifact_id:
return copy.deepcopy(row)
return None
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted: list[str] = []
for run_id, run in list(self._runs.items()):
if run["status"] not in TERMINAL_RUN_STATUSES:
continue
finished = _parse_dt(run.get("finished_at"))
if finished is None or finished >= cutoff:
continue
deleted.append(run_id)
for run_id in deleted:
self._runs.pop(run_id, None)
self._node_runs.pop(run_id, None)
self._artifacts.pop(run_id, None)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,96 @@
"""ORM models for workflow runs and per-node runs."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowRunRow(Base):
"""One workflow execution bound to an immutable published version."""
__tablename__ = "workflow_runs"
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
# Non-sensitive run context (never credentials).
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
# Pending human-input descriptor when status == awaiting_input.
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
# Last seq handed out by the event log for this run (0 = none yet). The
# event store increments it in the same transaction that inserts the event
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
# first append self-heals from MAX(seq) on conflict.
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowNodeRunRow(Base):
"""Per-node execution record (also the resume checkpoint for a paused run)."""
__tablename__ = "workflow_node_runs"
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
# queued | running | completed | failed | skipped | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class WorkflowRunArtifactRow(Base):
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
__tablename__ = "workflow_run_artifacts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
name: Mapped[str] = mapped_column(String(512), nullable=False)
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,566 @@
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
_RUN_UPDATABLE = {
"status",
"output_json",
"error_json",
"context_json",
"pending_input_json",
"resume_token",
"started_at",
"finished_at",
}
# Retention deletes in bounded batches so a long-neglected table cannot produce
# one enormous transaction.
_PURGE_BATCH = 500
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _load_text_json(raw: str | None) -> Any:
if not raw:
return None
try:
return json.loads(raw)
except (TypeError, json.JSONDecodeError):
return None
def _dump_text_json(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
return {
"id": row.id,
"workflow_id": row.workflow_id,
"workflow_version_id": row.workflow_version_id,
"status": row.status,
"owner_id": row.owner_id,
"idempotency_key": row.idempotency_key,
"input": row.input_json or {},
"output": row.output_json,
"error": row.error_json,
"context": row.context_json or {},
"pending_input": row.pending_input_json,
"resume_token": row.resume_token,
"lease_owner": row.lease_owner,
"lease_until": _iso(row.lease_until),
"heartbeat_at": _iso(row.heartbeat_at),
"attempt": row.attempt,
"max_attempts": row.max_attempts,
"version": row.version,
"retry_of_run_id": row.retry_of_run_id,
# Last seq handed out by the event log (0 = no events yet).
"next_event_seq": int(row.next_event_seq or 0),
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"node_type": row.node_type,
"status": row.status,
"attempt": row.attempt,
"iteration": row.iteration,
"is_current": bool(row.is_current),
"input": _load_text_json(row.input_snapshot),
"output": _load_text_json(row.output_snapshot),
"error": row.error_json,
"parent_node_run_id": row.parent_node_run_id,
"duration_ms": row.duration_ms,
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
}
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"name": row.name,
"mime_type": row.mime_type,
"path": row.path,
"size_bytes": row.size_bytes,
"preview": row.preview,
"created_at": _iso(row.created_at),
}
class SqlWorkflowRunStore(WorkflowRunStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
key = data.get("idempotency_key")
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
row = WorkflowRunRow(
id=str(data.get("id") or uuid.uuid4()),
workflow_id=str(data["workflow_id"]),
workflow_version_id=str(data["workflow_version_id"]),
status=str(data.get("status") or "queued"),
owner_id=str(data["owner_id"]),
idempotency_key=str(key) if key else None,
input_json=data.get("input") or {},
context_json=data.get("context") or {},
max_attempts=int(data.get("max_attempts") or 3),
retry_of_run_id=data.get("retry_of_run_id"),
)
async with self._sf() as session:
session.add(row)
try:
await session.flush()
except IntegrityError:
await session.rollback()
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
raise
out = run_to_dict(row)
await session.commit()
return out, True
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
return run_to_dict(row) if row else None
async def get_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
stmt = select(WorkflowRunRow)
if workflow_id:
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
if owner_id:
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowRunRow.status == status)
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [run_to_dict(r) for r in rows]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
stmt = (
select(WorkflowRunRow.id)
.where(
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
)
)
.order_by(WorkflowRunRow.created_at.asc())
.limit(limit)
)
async with self._sf() as session:
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
),
)
.values(
status="running",
lease_owner=lease_owner,
lease_until=lease_until,
heartbeat_at=now,
attempt=WorkflowRunRow.attempt + 1,
version=WorkflowRunRow.version + 1,
started_at=func.coalesce(WorkflowRunRow.started_at, now),
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.lease_owner == lease_owner,
WorkflowRunRow.status == "running",
)
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
return False
await session.commit()
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
for key, value in fields.items():
if key in _RUN_UPDATABLE:
values[key] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
values["lease_owner"] = None
values["lease_until"] = None
values["resume_token"] = None
values["pending_input_json"] = None
values.setdefault("finished_at", now)
where = [WorkflowRunRow.id == run_id]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
)
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(*where)
.values(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input_json=None,
finished_at=now,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
values: dict[str, Any] = {
"status": "awaiting_input",
"pending_input_json": pending_input,
"resume_token": resume_token,
"lease_owner": None,
"lease_until": None,
"version": WorkflowRunRow.version + 1,
"updated_at": now,
}
if context is not None:
values["context_json"] = context
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
if context is None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
if row is None:
return None
context = dict(row.context_json or {})
new_context = {**context, "resume_payload": payload}
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
context_json=new_context,
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
async with self._sf() as session:
row = (
await session.execute(
select(WorkflowNodeRunRow).where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id == node_id,
WorkflowNodeRunRow.attempt == attempt,
WorkflowNodeRunRow.iteration == iteration,
)
)
).scalar_one_or_none()
if row is None:
row = WorkflowNodeRunRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=run_id,
node_id=node_id,
node_type=str(data.get("node_type") or "unknown"),
attempt=attempt,
iteration=iteration,
)
session.add(row)
row.status = str(data.get("status") or row.status)
row.node_type = str(data.get("node_type") or row.node_type)
row.is_current = bool(data.get("is_current", True))
if "input" in data:
row.input_snapshot = _dump_text_json(data.get("input"))
if "output" in data:
row.output_snapshot = _dump_text_json(data.get("output"))
if "error" in data:
row.error_json = data.get("error")
if data.get("parent_node_run_id"):
row.parent_node_run_id = str(data["parent_node_run_id"])
if data.get("duration_ms") is not None:
row.duration_ms = int(data["duration_ms"])
if data.get("started_at") is not None:
row.started_at = data["started_at"]
if data.get("finished_at") is not None:
row.finished_at = data["finished_at"]
await session.flush()
out = node_run_to_dict(row)
await session.commit()
return out
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
if current_only:
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [node_run_to_dict(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
if not node_ids:
return 0
async with self._sf() as session:
result = await session.execute(
update(WorkflowNodeRunRow)
.where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id.in_(node_ids),
WorkflowNodeRunRow.is_current.is_(True),
)
.values(is_current=False)
)
await session.commit()
return int(result.rowcount or 0)
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = WorkflowRunArtifactRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=str(data["run_id"]),
node_id=str(data.get("node_id") or ""),
name=str(data.get("name") or "")[:512],
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
path=str(data.get("path") or ""),
size_bytes=data.get("size_bytes"),
preview=data.get("preview"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = artifact_to_dict(row)
await session.commit()
return out
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [artifact_to_dict(r) for r in rows]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
stmt = select(WorkflowRunArtifactRow).where(
WorkflowRunArtifactRow.id == artifact_id,
WorkflowRunArtifactRow.run_id == run_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return artifact_to_dict(row) if row is not None else None
# ── retention ───────────────────────────────────────────────────────
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
run_ids = (
(
await session.execute(
select(WorkflowRunRow.id)
.where(
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
WorkflowRunRow.finished_at.is_not(None),
WorkflowRunRow.finished_at < cutoff,
)
.limit(_PURGE_BATCH)
)
)
.scalars()
.all()
)
if not run_ids:
return []
ids = list(run_ids)
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
await session.commit()
return ids
@@ -0,0 +1,31 @@
"""Workflow definition / version persistence."""
from __future__ import annotations
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
WorkflowVersionNotFoundError,
)
from deerflow.persistence.workflows.memory import MemoryWorkflowStore
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
from deerflow.persistence.workflows.sql import WorkflowRepository
__all__ = [
"MemoryWorkflowStore",
"WorkflowDefinitionRow",
"WorkflowDraftConflictError",
"WorkflowNotFoundError",
"WorkflowRepository",
"WorkflowStore",
"WorkflowVersionNotFoundError",
"WorkflowVersionRow",
"make_workflow_store",
]
def make_workflow_store(session_factory):
if session_factory is None:
return MemoryWorkflowStore()
return WorkflowRepository(session_factory)
@@ -0,0 +1,112 @@
"""Abstract workflow definition / version store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowDraftConflictError(Exception):
"""Raised when ``expected_revision`` does not match the stored draft revision."""
def __init__(self, workflow_id: str, current_revision: int) -> None:
super().__init__(f"draft conflict for {workflow_id}: current={current_revision}")
self.workflow_id = workflow_id
self.current_revision = current_revision
class WorkflowNotFoundError(Exception):
def __init__(self, workflow_id: str) -> None:
super().__init__(f"workflow not found: {workflow_id}")
self.workflow_id = workflow_id
class WorkflowVersionNotFoundError(Exception):
def __init__(self, version_id: str) -> None:
super().__init__(f"workflow version not found: {version_id}")
self.version_id = version_id
class WorkflowStore(ABC):
@abstractmethod
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
"""CAS update of ``draft_graph_json``. Raises ``WorkflowDraftConflictError``."""
raise NotImplementedError
@abstractmethod
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
"""Atomically CAS-update BOTH the execution graph and the canvas document.
A single conditional UPDATE bumps ``draft_revision`` and writes
``draft_graph_json`` + ``draft_canvas_schema_json`` together, so the two
representations can never diverge across revisions. Raises
``WorkflowDraftConflictError`` on a stale ``expected_revision``.
"""
raise NotImplementedError
@abstractmethod
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_version(self, version_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
"""Lightweight list for subworkflow resource catalog."""
raise NotImplementedError
@@ -0,0 +1,216 @@
"""In-memory workflow store for tests / no-DB mode."""
from __future__ import annotations
import copy
import uuid
from datetime import UTC, datetime
from typing import Any
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
def _now() -> str:
return datetime.now(UTC).isoformat()
class MemoryWorkflowStore(WorkflowStore):
def __init__(self) -> None:
self._definitions: dict[str, dict[str, Any]] = {}
self._versions: dict[str, dict[str, Any]] = {}
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
rows = list(self._definitions.values())
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
elif not include_archived:
rows = [r for r in rows if r["status"] != "archived"]
rows.sort(key=lambda r: r.get("updated_at") or "", reverse=True)
return [
{k: v for k, v in r.items() if k not in ("draft_graph", "draft_canvas_schema", "next_version_number")}
for r in rows
]
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
row = self._definitions.get(workflow_id)
if row is None:
return None
out = copy.deepcopy(row)
if not include_draft:
out.pop("draft_graph", None)
return out
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
graph = copy.deepcopy(data.get("draft_graph") or data.get("graph") or {
"schemaVersion": "1.0",
"id": workflow_id,
"name": data.get("name") or "未命名工作流",
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
"edges": [],
"inputSchema": {"type": "object", "properties": {}},
"outputSchema": {"type": "object", "properties": {}},
"settings": {},
})
graph.setdefault("id", workflow_id)
now = _now()
row = {
"id": workflow_id,
"name": data.get("name") or graph.get("name") or "未命名工作流",
"description": data.get("description") or "",
"owner_id": data["owner_id"],
"status": data.get("status") or "draft",
"draft_revision": 0,
"next_version_number": 1,
"draft_graph": graph,
"draft_canvas_schema": str(data.get("draft_canvas_schema") or "{}"),
"created_at": now,
"updated_at": now,
}
self._definitions[workflow_id] = row
return copy.deepcopy(row)
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
row = self._definitions.get(workflow_id)
if row is None:
return None
if "name" in data and data["name"] is not None:
row["name"] = data["name"]
if "description" in data and data["description"] is not None:
row["description"] = data["description"]
if "status" in data and data["status"] is not None:
row["status"] = data["status"]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
return await self.update_definition(workflow_id, {"status": "archived"})
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by
row = self._definitions.get(workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row["draft_revision"] != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
payload = copy.deepcopy(graph)
payload.setdefault("id", workflow_id)
row["draft_graph"] = payload
row["draft_revision"] = expected_revision + 1
if payload.get("name"):
row["name"] = str(payload["name"])[:255]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by
row = self._definitions.get(workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row["draft_revision"] != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
payload = copy.deepcopy(graph)
payload.setdefault("id", workflow_id)
row["draft_graph"] = payload
row["draft_canvas_schema"] = canvas_schema or "{}"
row["draft_revision"] = expected_revision + 1
if payload.get("name"):
row["name"] = str(payload["name"])[:255]
row["updated_at"] = _now()
return copy.deepcopy(row)
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
definition = self._definitions.get(workflow_id)
if definition is None:
raise WorkflowNotFoundError(workflow_id)
existing = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
# Allocate from the definition's own counter (mirrors the SQL store) so
# the number never goes backwards even if events are purged/replayed.
counter = int(definition.get("next_version_number") or 1)
version_number = max(counter, max((v["version_number"] for v in existing), default=0) + 1)
definition["next_version_number"] = version_number + 1
version_id = f"wv_{uuid.uuid4().hex[:12]}"
version = {
"id": version_id,
"workflow_id": workflow_id,
"version_number": version_number,
"graph": copy.deepcopy(graph),
"graph_hash": graph_hash,
"input_schema": graph.get("inputSchema") or graph.get("input_schema") or {},
"output_schema": graph.get("outputSchema") or graph.get("output_schema") or {},
"published_by": published_by,
"published_at": _now(),
"change_note": change_note or "",
}
self._versions[version_id] = version
definition["status"] = "active"
if canvas_schema_hash:
definition["published_canvas_schema_hash"] = str(canvas_schema_hash)[:64]
definition["updated_at"] = _now()
return copy.deepcopy(version)
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
rows = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
rows.sort(key=lambda r: r["version_number"], reverse=True)
return [{k: v for k, v in r.items() if k != "graph"} for r in rows]
async def get_version(self, version_id: str) -> dict[str, Any] | None:
row = self._versions.get(version_id)
return copy.deepcopy(row) if row is not None else None
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
by_wf: dict[str, dict[str, Any]] = {}
for version in self._versions.values():
wf_id = version["workflow_id"]
if exclude_workflow_id and wf_id == exclude_workflow_id:
continue
definition = self._definitions.get(wf_id)
if definition is None or definition.get("status") == "archived":
continue
current = by_wf.get(wf_id)
if current is None or version["version_number"] > current["version_number"]:
by_wf[wf_id] = {
"workflow_id": wf_id,
"name": definition["name"],
"version_id": version["id"],
"version_number": version["version_number"],
"graph_hash": version["graph_hash"],
"published_at": version["published_at"],
}
return sorted(by_wf.values(), key=lambda r: r.get("published_at") or "", reverse=True)
@@ -0,0 +1,69 @@
"""ORM models for workflow definitions and immutable published versions."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowDefinitionRow(Base):
"""Mutable workflow metadata + current draft canvas."""
__tablename__ = "workflow_definitions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str] = mapped_column(Text, default="", nullable=False)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# draft | active | archived
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
draft_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
# Monotonic version allocator. "1" = no version published yet; publish takes
# the value then CAS-bumps it, replacing the old MAX(version_number)+1 race.
next_version_number: Mapped[int] = mapped_column(
Integer, nullable=False, default=1, server_default=sa_text("1")
)
draft_graph_json: Mapped[str] = mapped_column(PortableLongText(), default="{}", nullable=False)
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
# the executable WorkflowGraph lives in draft_graph_json above.
draft_canvas_schema_json: Mapped[str] = mapped_column(
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
)
# sha256 of the canvas document the current published version was compiled
# from (audit trail only; publish always freezes the execution graph).
published_canvas_schema_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowVersionRow(Base):
"""Immutable published graph snapshot. Runs bind to version id."""
__tablename__ = "workflow_versions"
__table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(
String(64),
ForeignKey("workflow_definitions.id", ondelete="CASCADE"),
nullable=False,
index=True,
)
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
graph_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
graph_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
input_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
published_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
change_note: Mapped[str] = mapped_column(Text, default="", nullable=False)
@@ -0,0 +1,385 @@
"""SQLAlchemy-backed workflow definition / version store."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.workflows.base import (
WorkflowDraftConflictError,
WorkflowNotFoundError,
WorkflowStore,
)
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
# Concurrent publishers CAS-retry against the workflow's own counter; a handful
# of attempts is far more than any realistic contention window needs.
_PUBLISH_ATTEMPTS = 6
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _parse_graph(raw: str | dict[str, Any] | None) -> dict[str, Any]:
if raw is None:
return {}
if isinstance(raw, dict):
return raw
text = str(raw).strip()
if not text:
return {}
try:
data = json.loads(text)
except json.JSONDecodeError:
return {}
return data if isinstance(data, dict) else {}
def _dump_graph(graph: dict[str, Any]) -> str:
return json.dumps(graph, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
def _definition_dict(row: WorkflowDefinitionRow, *, include_draft: bool) -> dict[str, Any]:
data = {
"id": row.id,
"name": row.name,
"description": row.description or "",
"owner_id": row.owner_id,
"status": row.status,
"draft_revision": row.draft_revision,
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
if include_draft:
data["draft_graph"] = _parse_graph(row.draft_graph_json)
# Canvas document as stored (opaque JSON text; "{}" when never saved).
data["draft_canvas_schema"] = row.draft_canvas_schema_json or "{}"
return data
def _version_dict(row: WorkflowVersionRow, *, include_graph: bool = True) -> dict[str, Any]:
data = {
"id": row.id,
"workflow_id": row.workflow_id,
"version_number": row.version_number,
"graph_hash": row.graph_hash,
"input_schema": row.input_schema or {},
"output_schema": row.output_schema or {},
"published_by": row.published_by,
"published_at": _iso(row.published_at),
"change_note": row.change_note or "",
}
if include_graph:
data["graph"] = _parse_graph(row.graph_json)
return data
class WorkflowRepository(WorkflowStore):
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
async def list_definitions(
self,
*,
owner_id: str | None = None,
status: str | None = None,
include_archived: bool = False,
) -> list[dict[str, Any]]:
stmt = select(WorkflowDefinitionRow)
if owner_id:
stmt = stmt.where(WorkflowDefinitionRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowDefinitionRow.status == status)
elif not include_archived:
stmt = stmt.where(WorkflowDefinitionRow.status != "archived")
stmt = stmt.order_by(WorkflowDefinitionRow.updated_at.desc())
async with self._sf() as session:
result = await session.execute(stmt)
return [_definition_dict(row, include_draft=False) for row in result.scalars()]
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
return _definition_dict(row, include_draft=include_draft) if row is not None else None
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
graph = data.get("draft_graph") or data.get("graph") or {
"schemaVersion": "1.0",
"id": workflow_id,
"name": data.get("name") or "未命名工作流",
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
"edges": [],
"inputSchema": {"type": "object", "properties": {}},
"outputSchema": {"type": "object", "properties": {}},
"settings": {},
}
if isinstance(graph, dict) and not graph.get("id"):
graph = {**graph, "id": workflow_id}
if isinstance(graph, dict) and data.get("name") and not graph.get("name"):
graph = {**graph, "name": data["name"]}
row = WorkflowDefinitionRow(
id=workflow_id,
name=data.get("name") or graph.get("name") or "未命名工作流",
description=data.get("description") or "",
owner_id=data["owner_id"],
status=data.get("status") or "draft",
draft_revision=0,
draft_graph_json=_dump_graph(graph),
draft_canvas_schema_json=str(data.get("draft_canvas_schema") or "{}"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = _definition_dict(row, include_draft=True)
await session.commit()
return out
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
return None
if "name" in data and data["name"] is not None:
row.name = data["name"]
if "description" in data and data["description"] is not None:
row.description = data["description"]
if "status" in data and data["status"] is not None:
row.status = data["status"]
row.updated_at = datetime.now(UTC)
await session.flush()
out = _definition_dict(row, include_draft=True)
await session.commit()
return out
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
return await self.update_definition(workflow_id, {"status": "archived"})
async def save_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by # audit reserved for later
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row.draft_revision != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
payload = dict(graph)
payload.setdefault("id", workflow_id)
if row.name and not payload.get("name"):
payload["name"] = row.name
# Conditional UPDATE on the revision itself: two sessions that both
# read the same revision cannot both win — the loser's WHERE clause
# matches zero rows and surfaces a deterministic conflict instead of
# silently overwriting the other editor's draft.
values: dict[str, Any] = {
"draft_graph_json": _dump_graph(payload),
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
"updated_at": datetime.now(UTC),
}
if payload.get("name"):
values["name"] = str(payload["name"])[:255]
result = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.draft_revision == expected_revision,
)
.values(**values)
)
if result.rowcount == 0:
await session.rollback()
current = await session.get(WorkflowDefinitionRow, workflow_id)
if current is None:
raise WorkflowNotFoundError(workflow_id)
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
await session.commit()
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
assert fresh is not None
return _definition_dict(fresh, include_draft=True)
async def save_studio_draft(
self,
workflow_id: str,
*,
expected_revision: int,
graph: dict[str, Any],
canvas_schema: str = "{}",
updated_by: str | None = None,
) -> dict[str, Any]:
del updated_by # audit reserved for later
async with self._sf() as session:
row = await session.get(WorkflowDefinitionRow, workflow_id)
if row is None:
raise WorkflowNotFoundError(workflow_id)
if row.draft_revision != expected_revision:
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
payload = dict(graph)
payload.setdefault("id", workflow_id)
if row.name and not payload.get("name"):
payload["name"] = row.name
# One conditional UPDATE writes graph + canvas + revision together:
# the two representations are always from the same save, and a
# concurrent editor loses deterministically instead of interleaving
# a new canvas onto someone else's graph.
values: dict[str, Any] = {
"draft_graph_json": _dump_graph(payload),
"draft_canvas_schema_json": canvas_schema or "{}",
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
"updated_at": datetime.now(UTC),
}
if payload.get("name"):
values["name"] = str(payload["name"])[:255]
result = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.draft_revision == expected_revision,
)
.values(**values)
)
if result.rowcount == 0:
await session.rollback()
current = await session.get(WorkflowDefinitionRow, workflow_id)
if current is None:
raise WorkflowNotFoundError(workflow_id)
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
await session.commit()
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
assert fresh is not None
return _definition_dict(fresh, include_draft=True)
async def publish_version(
self,
workflow_id: str,
*,
graph: dict[str, Any],
graph_hash: str,
published_by: str | None = None,
change_note: str = "",
canvas_schema_hash: str | None = None,
) -> dict[str, Any]:
"""Allocate the version number from the workflow's own counter.
``MAX(version_number) + 1`` races between concurrent publishes; here the
number is claimed by CAS-bumping ``next_version_number`` inside the same
transaction that inserts the version row, and the unique
``(workflow_id, version_number)`` constraint is the final safety net.
Concurrent publishers get adjacent, distinct numbers.
"""
last_error: Exception | None = None
for _ in range(_PUBLISH_ATTEMPTS):
try:
async with self._sf() as session:
definition = await session.get(WorkflowDefinitionRow, workflow_id)
if definition is None:
raise WorkflowNotFoundError(workflow_id)
counter_seen = int(definition.next_version_number or 1)
# Legacy rows got the column auto-added with server_default 1
# while versions already exist — never allocate below the max.
latest = (
await session.execute(
select(func.max(WorkflowVersionRow.version_number)).where(
WorkflowVersionRow.workflow_id == workflow_id
)
)
).scalar() or 0
number = max(counter_seen, int(latest) + 1)
claimed = await session.execute(
update(WorkflowDefinitionRow)
.where(
WorkflowDefinitionRow.id == workflow_id,
WorkflowDefinitionRow.next_version_number == counter_seen,
)
.values(next_version_number=number + 1)
)
if claimed.rowcount == 0:
# Another publish landed first — release nothing (the
# counter bump never happened) and retry with fresh state.
await session.rollback()
last_error = None
continue
version = WorkflowVersionRow(
id=f"wv_{uuid.uuid4().hex[:12]}",
workflow_id=workflow_id,
version_number=number,
graph_json=_dump_graph(graph),
graph_hash=graph_hash,
input_schema=graph.get("inputSchema") or graph.get("input_schema") or {},
output_schema=graph.get("outputSchema") or graph.get("output_schema") or {},
published_by=published_by,
change_note=change_note or "",
)
definition.status = "active"
definition.updated_at = datetime.now(UTC)
if canvas_schema_hash:
definition.published_canvas_schema_hash = canvas_schema_hash[:64]
session.add(version)
await session.flush()
out = _version_dict(version, include_graph=True)
await session.commit()
return out
except IntegrityError as exc: # pragma: no cover - CAS makes this near-impossible
last_error = exc
continue
if last_error is not None:
raise last_error
raise RuntimeError(f"failed to allocate a workflow version number for {workflow_id}")
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
stmt = (
select(WorkflowVersionRow)
.where(WorkflowVersionRow.workflow_id == workflow_id)
.order_by(WorkflowVersionRow.version_number.desc())
)
async with self._sf() as session:
result = await session.execute(stmt)
return [_version_dict(row, include_graph=False) for row in result.scalars()]
async def get_version(self, version_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowVersionRow, version_id)
return _version_dict(row, include_graph=True) if row is not None else None
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
stmt = (
select(WorkflowVersionRow, WorkflowDefinitionRow)
.join(WorkflowDefinitionRow, WorkflowDefinitionRow.id == WorkflowVersionRow.workflow_id)
.where(WorkflowDefinitionRow.status != "archived")
.order_by(WorkflowVersionRow.published_at.desc())
)
if exclude_workflow_id:
stmt = stmt.where(WorkflowVersionRow.workflow_id != exclude_workflow_id)
async with self._sf() as session:
result = await session.execute(stmt)
out: list[dict[str, Any]] = []
seen: set[str] = set()
for version, definition in result.all():
# Latest version per workflow only.
if version.workflow_id in seen:
continue
seen.add(version.workflow_id)
out.append(
{
"workflow_id": definition.id,
"name": definition.name,
"version_id": version.id,
"version_number": version.version_number,
"graph_hash": version.graph_hash,
"published_at": _iso(version.published_at),
}
)
return out