代码提交
This commit is contained in:
1 parent
4ac7bc21cd
commit
17cf849361
110 files changed
+17007
-518
No files matched your search
+64
@@ -0,0 +1,64 @@
|
||||
"""Create workflow_definitions / workflow_versions tables.
|
||||
|
||||
Revision ID: 20260828_01
|
||||
Revises: 20260825_01
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_01"
|
||||
down_revision = "20260825_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
op.create_table(
|
||||
"workflow_definitions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("name", sa.String(length=255), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=False, server_default=""),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="draft"),
|
||||
sa.Column("draft_revision", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("draft_graph_json", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_workflow_definitions_owner_id", "workflow_definitions", ["owner_id"])
|
||||
op.create_index("ix_workflow_definitions_status", "workflow_definitions", ["status"])
|
||||
|
||||
if not inspector.has_table("workflow_versions"):
|
||||
op.create_table(
|
||||
"workflow_versions",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_number", sa.Integer(), nullable=False),
|
||||
sa.Column("graph_json", sa.Text(), nullable=False),
|
||||
sa.Column("graph_hash", sa.String(length=64), nullable=False),
|
||||
sa.Column("input_schema", sa.Text(), nullable=True),
|
||||
sa.Column("output_schema", sa.Text(), nullable=True),
|
||||
sa.Column("published_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("published_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("change_note", sa.Text(), nullable=False, server_default=""),
|
||||
sa.ForeignKeyConstraint(["workflow_id"], ["workflow_definitions.id"], ondelete="CASCADE"),
|
||||
sa.UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),
|
||||
)
|
||||
op.create_index("ix_workflow_versions_workflow_id", "workflow_versions", ["workflow_id"])
|
||||
op.create_index("ix_workflow_versions_graph_hash", "workflow_versions", ["graph_hash"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("workflow_versions"):
|
||||
op.drop_table("workflow_versions")
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
op.drop_table("workflow_definitions")
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
"""Create workflow run / node-run / event / artifact tables.
|
||||
|
||||
Revision ID: 20260828_02
|
||||
Revises: 20260828_01
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_02"
|
||||
down_revision = "20260828_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if not inspector.has_table("workflow_runs"):
|
||||
op.create_table(
|
||||
"workflow_runs",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("workflow_version_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("idempotency_key", sa.String(length=128), nullable=True),
|
||||
sa.Column("input_json", sa.Text(), nullable=True),
|
||||
sa.Column("output_json", sa.Text(), nullable=True),
|
||||
sa.Column("error_json", sa.Text(), nullable=True),
|
||||
sa.Column("context_json", sa.Text(), nullable=True),
|
||||
sa.Column("pending_input_json", sa.Text(), nullable=True),
|
||||
sa.Column("resume_token", sa.String(length=64), nullable=True),
|
||||
sa.Column("lease_owner", sa.String(length=64), nullable=True),
|
||||
sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("heartbeat_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
|
||||
sa.Column("version", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("retry_of_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),
|
||||
)
|
||||
op.create_index("ix_workflow_runs_workflow_id", "workflow_runs", ["workflow_id"])
|
||||
op.create_index("ix_workflow_runs_workflow_version_id", "workflow_runs", ["workflow_version_id"])
|
||||
op.create_index("ix_workflow_runs_owner_id", "workflow_runs", ["owner_id"])
|
||||
op.create_index("ix_workflow_runs_status", "workflow_runs", ["status"])
|
||||
op.create_index("ix_workflow_runs_resume_token", "workflow_runs", ["resume_token"])
|
||||
|
||||
if not inspector.has_table("workflow_node_runs"):
|
||||
op.create_table(
|
||||
"workflow_node_runs",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("node_type", sa.String(length=32), nullable=False),
|
||||
sa.Column("status", sa.String(length=32), nullable=False, server_default="queued"),
|
||||
sa.Column("attempt", sa.Integer(), nullable=False, server_default="1"),
|
||||
sa.Column("iteration", sa.Integer(), nullable=False, server_default="0"),
|
||||
sa.Column("is_current", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("input_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("output_snapshot", sa.Text(), nullable=True),
|
||||
sa.Column("error_json", sa.Text(), nullable=True),
|
||||
sa.Column("parent_node_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("duration_ms", sa.Integer(), nullable=True),
|
||||
sa.Column("started_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint(
|
||||
"run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"
|
||||
),
|
||||
)
|
||||
op.create_index("ix_workflow_node_runs_run_id", "workflow_node_runs", ["run_id"])
|
||||
op.create_index("ix_workflow_node_runs_status", "workflow_node_runs", ["status"])
|
||||
|
||||
if not inspector.has_table("workflow_run_events"):
|
||||
op.create_table(
|
||||
"workflow_run_events",
|
||||
sa.Column("id", sa.Integer(), primary_key=True, autoincrement=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("workflow_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("version_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("seq", sa.Integer(), nullable=False),
|
||||
sa.Column("event_type", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("node_run_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("payload_json", sa.Text(), nullable=False),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
)
|
||||
op.create_index("ix_workflow_run_events_run_id", "workflow_run_events", ["run_id"])
|
||||
op.create_index("ix_workflow_run_events_run_seq", "workflow_run_events", ["run_id", "seq"])
|
||||
|
||||
if not inspector.has_table("workflow_run_artifacts"):
|
||||
op.create_table(
|
||||
"workflow_run_artifacts",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("run_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("node_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("name", sa.String(length=512), nullable=False),
|
||||
sa.Column("mime_type", sa.String(length=191), nullable=False),
|
||||
sa.Column("path", sa.Text(), nullable=False),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=True),
|
||||
sa.Column("preview", sa.Text(), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
)
|
||||
op.create_index("ix_workflow_run_artifacts_run_id", "workflow_run_artifacts", ["run_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
for table in (
|
||||
"workflow_run_artifacts",
|
||||
"workflow_run_events",
|
||||
"workflow_node_runs",
|
||||
"workflow_runs",
|
||||
):
|
||||
if inspector.has_table(table):
|
||||
op.drop_table(table)
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
"""Create workflow_data_sources table.
|
||||
|
||||
Revision ID: 20260828_03
|
||||
Revises: 20260828_02
|
||||
Create Date: 2026-08-28
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_03"
|
||||
down_revision = "20260828_02"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
op.create_table(
|
||||
"workflow_data_sources",
|
||||
sa.Column("id", sa.String(length=64), primary_key=True),
|
||||
sa.Column("name", sa.String(length=191), nullable=False),
|
||||
sa.Column("description", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("kind", sa.String(length=32), nullable=False, server_default="sql"),
|
||||
sa.Column("driver", sa.String(length=64), nullable=False, server_default=""),
|
||||
sa.Column("secret_ciphertext", sa.Text(), nullable=False),
|
||||
sa.Column("secret_encrypted", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("masked_target", sa.String(length=512), nullable=False, server_default=""),
|
||||
sa.Column("max_rows", sa.Integer(), nullable=False, server_default="1000"),
|
||||
sa.Column("allowed_tables", sa.Text(), nullable=True),
|
||||
sa.Column("enabled", sa.Boolean(), nullable=False, server_default=sa.text("1")),
|
||||
sa.Column("owner_id", sa.String(length=64), nullable=True),
|
||||
sa.Column("created_by", sa.String(length=64), nullable=True),
|
||||
sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("name", name="uq_workflow_data_sources_name"),
|
||||
)
|
||||
op.create_index("ix_workflow_data_sources_owner_id", "workflow_data_sources", ["owner_id"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
if sa.inspect(bind).has_table("workflow_data_sources"):
|
||||
op.drop_table("workflow_data_sources")
|
||||
+66
@@ -0,0 +1,66 @@
|
||||
"""Add workflow counter columns: version allocator + event seq counter.
|
||||
|
||||
Revision ID: 20260828_04
|
||||
Revises: 20260828_03
|
||||
Create Date: 2026-08-28
|
||||
|
||||
Functional-robustness remediation (audit items P1-02 / P1-03):
|
||||
|
||||
* ``workflow_definitions.next_version_number`` — publish allocates the version
|
||||
number by CAS-bumping this counter instead of ``MAX(version_number) + 1``,
|
||||
so two concurrent publishes get adjacent distinct numbers instead of a
|
||||
unique-constraint 500.
|
||||
* ``workflow_runs.next_event_seq`` — the event log draws each event's ``seq``
|
||||
from this counter in the insert's own transaction instead of ``MAX(seq)+1``.
|
||||
|
||||
Both columns carry a ``server_default`` so existing databases get them via the
|
||||
startup ``_ensure_orm_columns_sync`` sweep even without running Alembic; rows
|
||||
keep the default (1 / 0) and the first write self-heals from the existing
|
||||
maximum on unique-conflict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260828_04"
|
||||
down_revision = "20260828_03"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
|
||||
if "next_version_number" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("next_version_number", sa.Integer(), nullable=False, server_default="1"),
|
||||
)
|
||||
|
||||
if inspector.has_table("workflow_runs"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
|
||||
if "next_event_seq" not in existing:
|
||||
op.add_column(
|
||||
"workflow_runs",
|
||||
sa.Column("next_event_seq", sa.Integer(), nullable=False, server_default="0"),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
|
||||
if inspector.has_table("workflow_definitions"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_definitions")}
|
||||
if "next_version_number" in existing:
|
||||
op.drop_column("workflow_definitions", "next_version_number")
|
||||
|
||||
if inspector.has_table("workflow_runs"):
|
||||
existing = {c["name"] for c in inspector.get_columns("workflow_runs")}
|
||||
if "next_event_seq" in existing:
|
||||
op.drop_column("workflow_runs", "next_event_seq")
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
"""Add non-secret HTTP metadata to workflow data sources.
|
||||
|
||||
Revision ID: 20260829_01
|
||||
Revises: 20260828_04
|
||||
Create Date: 2026-08-29
|
||||
|
||||
HTTP request headers remain in the encrypted secret blob. The endpoint and
|
||||
method allowlist are intentionally separate non-secret fields so the canvas
|
||||
can show a reusable interface resource without exposing credentials.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260829_01"
|
||||
down_revision = "20260828_04"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
|
||||
if "base_url" not in existing:
|
||||
op.add_column(
|
||||
"workflow_data_sources",
|
||||
sa.Column("base_url", sa.String(length=2048), nullable=False, server_default=""),
|
||||
)
|
||||
if "allowed_methods" not in existing:
|
||||
op.add_column(
|
||||
"workflow_data_sources",
|
||||
sa.Column("allowed_methods", sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_data_sources"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_data_sources")}
|
||||
if "allowed_methods" in existing:
|
||||
op.drop_column("workflow_data_sources", "allowed_methods")
|
||||
if "base_url" in existing:
|
||||
op.drop_column("workflow_data_sources", "base_url")
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""Add Coze canvas draft column to workflow_definitions.
|
||||
|
||||
Revision ID: 20260829_02
|
||||
Revises: 20260829_01
|
||||
Create Date: 2026-08-29
|
||||
|
||||
The canvas document (positions, Coze form internals) is stored separately from
|
||||
the executable WorkflowGraph so neither representation can impersonate the
|
||||
other. ``published_canvas_schema_hash`` records which canvas the current
|
||||
published version was compiled from, for audit only.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = "20260829_02"
|
||||
down_revision = "20260829_01"
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
|
||||
if "draft_canvas_schema_json" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("draft_canvas_schema_json", sa.Text(), nullable=False, server_default="'{}'"),
|
||||
)
|
||||
if "published_canvas_schema_hash" not in existing:
|
||||
op.add_column(
|
||||
"workflow_definitions",
|
||||
sa.Column("published_canvas_schema_hash", sa.String(length=64), nullable=True),
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
bind = op.get_bind()
|
||||
inspector = sa.inspect(bind)
|
||||
if not inspector.has_table("workflow_definitions"):
|
||||
return
|
||||
existing = {column["name"] for column in inspector.get_columns("workflow_definitions")}
|
||||
if "published_canvas_schema_hash" in existing:
|
||||
op.drop_column("workflow_definitions", "published_canvas_schema_hash")
|
||||
if "draft_canvas_schema_json" in existing:
|
||||
op.drop_column("workflow_definitions", "draft_canvas_schema_json")
|
||||
+15
@@ -50,6 +50,14 @@ from deerflow.persistence.feedback.model import FeedbackRow
|
||||
from deerflow.persistence.fixed_questions.model import FixedQuestionRow
|
||||
from deerflow.persistence.intent_turns.model import IntentTurnRow
|
||||
from deerflow.persistence.light_apps.model import LightAppRow
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
from deerflow.persistence.llm_metrics.model import LlmCallMetricRow
|
||||
from deerflow.persistence.llmwiki.model import LlmWikiConversationDepositRow, LlmWikiKnowledgeBaseRow
|
||||
from deerflow.persistence.llmwiki_index.model import LlmWikiWikiPageRow, LlmWikiWikiSyncStateRow, LlmWikiWikiVectorRow
|
||||
@@ -128,6 +136,13 @@ __all__ = [
|
||||
"KnowledgeSourceRow",
|
||||
"KnowledgeTagRow",
|
||||
"LightAppRow",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDefinitionRow",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunEventRow",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowVersionRow",
|
||||
"LlmCallMetricRow",
|
||||
"LlmWikiKnowledgeBaseRow",
|
||||
"LlmWikiConversationDepositRow",
|
||||
|
||||
+19
@@ -0,0 +1,19 @@
|
||||
"""Workflow data-source persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.persistence.workflow_data_sources.store import (
|
||||
MemoryWorkflowDataSourceStore,
|
||||
SqlWorkflowDataSourceStore,
|
||||
WorkflowDataSourceStore,
|
||||
make_workflow_data_source_store,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowDataSourceStore",
|
||||
"SqlWorkflowDataSourceStore",
|
||||
"WorkflowDataSourceRow",
|
||||
"WorkflowDataSourceStore",
|
||||
"make_workflow_data_source_store",
|
||||
]
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
"""ORM model for workflow SQL data sources and HTTP credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowDataSourceRow(Base):
|
||||
"""A read-only database connection usable by ``sql_read`` nodes.
|
||||
|
||||
``secret_ciphertext`` holds the DSN encrypted with the workflow secret key;
|
||||
it is never returned by any API. ``owner_id`` NULL means the data source is
|
||||
shared (admin-managed)."""
|
||||
|
||||
__tablename__ = "workflow_data_sources"
|
||||
__table_args__ = (UniqueConstraint("name", name="uq_workflow_data_sources_name"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(191), nullable=False)
|
||||
description: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
kind: Mapped[str] = mapped_column(String(32), nullable=False, default="sql")
|
||||
driver: Mapped[str] = mapped_column(String(64), nullable=False, default="")
|
||||
base_url: Mapped[str] = mapped_column(
|
||||
String(2048),
|
||||
nullable=False,
|
||||
default="",
|
||||
server_default=sa_text("''"),
|
||||
)
|
||||
# Nullable so the startup column synchronizer can safely patch existing databases.
|
||||
allowed_methods: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
|
||||
secret_ciphertext: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="")
|
||||
secret_encrypted: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
masked_target: Mapped[str] = mapped_column(String(512), nullable=False, default="")
|
||||
max_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=1000, server_default=sa_text("1000"))
|
||||
allowed_tables: Mapped[list] = mapped_column(PortableJSON(), default=list)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
owner_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
created_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
"""Workflow data-source store (SQL + in-memory).
|
||||
|
||||
Secrets are encrypted before they reach the database and only ever decrypted
|
||||
inside :meth:`resolve_dsn`, which the run executor calls. No read path returns
|
||||
the ciphertext.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_data_sources.model import WorkflowDataSourceRow
|
||||
from deerflow.workflows.security.secret_box import decrypt_secret, encrypt_secret, mask_dsn
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _public_dict(row: WorkflowDataSourceRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"description": row.description or "",
|
||||
"kind": row.kind,
|
||||
"driver": row.driver or "",
|
||||
"base_url": row.base_url or "",
|
||||
"allowed_methods": list(row.allowed_methods or []),
|
||||
"masked_target": row.masked_target or "",
|
||||
"secret_encrypted": bool(row.secret_encrypted),
|
||||
"max_rows": row.max_rows,
|
||||
"allowed_tables": list(row.allowed_tables or []),
|
||||
"enabled": bool(row.enabled),
|
||||
"owner_id": row.owner_id,
|
||||
"created_by": row.created_by,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
class WorkflowDataSourceStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
"""Decrypted connection info for the executor. Never expose via HTTP."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class SqlWorkflowDataSourceStore(WorkflowDataSourceStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowDataSourceRow).order_by(WorkflowDataSourceRow.name.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
visible = []
|
||||
for row in rows:
|
||||
if row.owner_id is None and include_shared:
|
||||
visible.append(row)
|
||||
elif owner_id is not None and row.owner_id == owner_id:
|
||||
visible.append(row)
|
||||
return [_public_dict(row) for row in visible]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
return _public_dict(row) if row else None
|
||||
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
dsn = str(data.get("dsn") or "")
|
||||
kind = str(data.get("kind") or "sql")
|
||||
base_url = str(data.get("base_url") or "")
|
||||
row = WorkflowDataSourceRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
name=str(data["name"])[:191],
|
||||
description=str(data.get("description") or "")[:512],
|
||||
kind=kind,
|
||||
driver=str(
|
||||
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
|
||||
)[:64],
|
||||
base_url=base_url[:2048],
|
||||
allowed_methods=list(data.get("allowed_methods") or []),
|
||||
secret_ciphertext=encrypt_secret(dsn) if dsn else "",
|
||||
secret_encrypted=bool(dsn),
|
||||
masked_target=(base_url if kind == "http" else mask_dsn(dsn))[:512],
|
||||
max_rows=int(data.get("max_rows") or 1000),
|
||||
allowed_tables=list(data.get("allowed_tables") or []),
|
||||
enabled=bool(data.get("enabled", True)),
|
||||
owner_id=data.get("owner_id"),
|
||||
created_by=data.get("created_by"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = _public_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None:
|
||||
return None
|
||||
if data.get("name"):
|
||||
row.name = str(data["name"])[:191]
|
||||
if "description" in data:
|
||||
row.description = str(data.get("description") or "")[:512]
|
||||
if "base_url" in data:
|
||||
row.base_url = str(data.get("base_url") or "")[:2048]
|
||||
if row.kind == "http":
|
||||
row.masked_target = row.base_url[:512]
|
||||
if "allowed_methods" in data:
|
||||
row.allowed_methods = list(data.get("allowed_methods") or [])
|
||||
if data.get("dsn"):
|
||||
dsn = str(data["dsn"])
|
||||
row.secret_ciphertext = encrypt_secret(dsn)
|
||||
row.secret_encrypted = True
|
||||
if row.kind == "http":
|
||||
row.driver = "HTTP API"
|
||||
else:
|
||||
row.masked_target = mask_dsn(dsn)[:512]
|
||||
row.driver = dsn.split("://", 1)[0][:64]
|
||||
if data.get("max_rows") is not None:
|
||||
row.max_rows = int(data["max_rows"])
|
||||
if "allowed_tables" in data:
|
||||
row.allowed_tables = list(data.get("allowed_tables") or [])
|
||||
if "enabled" in data:
|
||||
row.enabled = bool(data["enabled"])
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
out = _public_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None:
|
||||
return False
|
||||
await session.delete(row)
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDataSourceRow, source_id)
|
||||
if row is None or not row.enabled or not row.secret_ciphertext:
|
||||
return None
|
||||
return {
|
||||
"id": row.id,
|
||||
"dsn": decrypt_secret(row.secret_ciphertext),
|
||||
"kind": row.kind,
|
||||
"base_url": row.base_url or "",
|
||||
"allowed_methods": list(row.allowed_methods or []),
|
||||
"max_rows": row.max_rows,
|
||||
"allowed_tables": list(row.allowed_tables or []),
|
||||
"owner_id": row.owner_id,
|
||||
}
|
||||
|
||||
|
||||
class MemoryWorkflowDataSourceStore(WorkflowDataSourceStore):
|
||||
def __init__(self) -> None:
|
||||
self._rows: dict[str, dict[str, Any]] = {}
|
||||
self._secrets: dict[str, str] = {}
|
||||
|
||||
async def list_sources(self, *, owner_id: str | None, include_shared: bool = True) -> list[dict[str, Any]]:
|
||||
out = []
|
||||
for row in self._rows.values():
|
||||
if row["owner_id"] is None and include_shared:
|
||||
out.append(row)
|
||||
elif owner_id is not None and row["owner_id"] == owner_id:
|
||||
out.append(row)
|
||||
return [copy.deepcopy(r) for r in sorted(out, key=lambda r: r["name"])]
|
||||
|
||||
async def get_source(self, source_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
return copy.deepcopy(row) if row else None
|
||||
|
||||
async def create_source(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
source_id = str(data.get("id") or uuid.uuid4())
|
||||
dsn = str(data.get("dsn") or "")
|
||||
kind = str(data.get("kind") or "sql")
|
||||
base_url = str(data.get("base_url") or "")
|
||||
now = _iso(datetime.now(UTC))
|
||||
row = {
|
||||
"id": source_id,
|
||||
"name": str(data["name"]),
|
||||
"description": str(data.get("description") or ""),
|
||||
"kind": kind,
|
||||
"driver": str(
|
||||
data.get("driver") or ("HTTP API" if kind == "http" else dsn.split("://", 1)[0])
|
||||
),
|
||||
"base_url": base_url,
|
||||
"allowed_methods": list(data.get("allowed_methods") or []),
|
||||
"masked_target": base_url if kind == "http" else mask_dsn(dsn),
|
||||
"secret_encrypted": bool(dsn),
|
||||
"max_rows": int(data.get("max_rows") or 1000),
|
||||
"allowed_tables": list(data.get("allowed_tables") or []),
|
||||
"enabled": bool(data.get("enabled", True)),
|
||||
"owner_id": data.get("owner_id"),
|
||||
"created_by": data.get("created_by"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._rows[source_id] = row
|
||||
self._secrets[source_id] = dsn
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def update_source(self, source_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
if row is None:
|
||||
return None
|
||||
for key in (
|
||||
"name",
|
||||
"description",
|
||||
"base_url",
|
||||
"allowed_methods",
|
||||
"max_rows",
|
||||
"allowed_tables",
|
||||
"enabled",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
row[key] = data[key]
|
||||
if "base_url" in data and row["kind"] == "http":
|
||||
row["masked_target"] = str(row["base_url"])
|
||||
if data.get("dsn"):
|
||||
self._secrets[source_id] = str(data["dsn"])
|
||||
if row["kind"] == "http":
|
||||
row["driver"] = "HTTP API"
|
||||
else:
|
||||
row["masked_target"] = mask_dsn(str(data["dsn"]))
|
||||
row["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def delete_source(self, source_id: str) -> bool:
|
||||
self._secrets.pop(source_id, None)
|
||||
return self._rows.pop(source_id, None) is not None
|
||||
|
||||
async def resolve_dsn(self, source_id: str) -> dict[str, Any] | None:
|
||||
row = self._rows.get(source_id)
|
||||
if row is None or not row["enabled"]:
|
||||
return None
|
||||
return {
|
||||
"id": source_id,
|
||||
"dsn": self._secrets.get(source_id, ""),
|
||||
"kind": row["kind"],
|
||||
"base_url": row.get("base_url", ""),
|
||||
"allowed_methods": list(row.get("allowed_methods") or []),
|
||||
"max_rows": row["max_rows"],
|
||||
"allowed_tables": row["allowed_tables"],
|
||||
"owner_id": row["owner_id"],
|
||||
}
|
||||
|
||||
|
||||
def make_workflow_data_source_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowDataSourceStore()
|
||||
return SqlWorkflowDataSourceStore(session_factory)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowDataSourceStore",
|
||||
"SqlWorkflowDataSourceStore",
|
||||
"WorkflowDataSourceStore",
|
||||
"make_workflow_data_source_store",
|
||||
]
|
||||
+22
@@ -0,0 +1,22 @@
|
||||
"""Workflow run event-log persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowEventStore",
|
||||
"SqlWorkflowEventStore",
|
||||
"WorkflowEventStore",
|
||||
"WorkflowRunEventRow",
|
||||
"make_workflow_event_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_event_store(session_factory, run_store=None):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowEventStore(run_store=run_store)
|
||||
return SqlWorkflowEventStore(session_factory)
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Abstract workflow run event store (durable, replayable)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowEventStore(ABC):
|
||||
@abstractmethod
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, assigning the next per-run ``seq``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
"""Retention purge by event age; returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
"""Delete every event belonging to the given runs (used when the runs
|
||||
themselves are retained away). Returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""In-memory workflow event log (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
|
||||
|
||||
class MemoryWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
|
||||
self._events: dict[str, list[dict[str, Any]]] = {}
|
||||
self._max_per_run = max_per_run
|
||||
self._run_store = run_store
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
rows = self._events.setdefault(run_id, [])
|
||||
event = {
|
||||
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
|
||||
"run_id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_id": version_id,
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"node_run_id": node_run_id,
|
||||
"payload": payload or {},
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
rows.append(event)
|
||||
if len(rows) > self._max_per_run:
|
||||
del rows[: len(rows) - self._max_per_run]
|
||||
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
|
||||
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
|
||||
# where the counter bump and the insert share one transaction).
|
||||
try:
|
||||
await self._run_store.note_event_seq(run_id, event["seq"])
|
||||
except Exception: # noqa: BLE001 - counter sync is best effort
|
||||
pass
|
||||
return copy.deepcopy(event)
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
|
||||
return [copy.deepcopy(r) for r in rows[:limit]]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
rows = self._events.get(run_id, [])
|
||||
return int(rows[-1]["seq"]) if rows else 0
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id, rows in list(self._events.items()):
|
||||
kept: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
created = _parse_dt(row.get("created_at"))
|
||||
if created is not None and created < cutoff:
|
||||
deleted += 1
|
||||
else:
|
||||
kept.append(row)
|
||||
if kept:
|
||||
self._events[run_id] = kept
|
||||
else:
|
||||
del self._events[run_id]
|
||||
return deleted
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id in run_ids:
|
||||
rows = self._events.pop(run_id, None)
|
||||
if rows:
|
||||
deleted += len(rows)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""ORM model for the durable workflow run event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunEventRow(Base):
|
||||
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
|
||||
|
||||
__tablename__ = "workflow_run_events"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
|
||||
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+157
@@ -0,0 +1,157 @@
|
||||
"""SQLAlchemy-backed workflow event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
|
||||
|
||||
_MAX_PAYLOAD_CHARS = 64_000
|
||||
_APPEND_ATTEMPTS = 5
|
||||
|
||||
|
||||
def _dump_payload(payload: dict[str, Any] | None) -> str:
|
||||
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
if len(text) > _MAX_PAYLOAD_CHARS:
|
||||
return json.dumps(
|
||||
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
)
|
||||
return text
|
||||
|
||||
|
||||
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(row.payload_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
return {
|
||||
"seq": row.seq,
|
||||
"run_id": row.run_id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_id": row.version_id,
|
||||
"event_type": row.event_type,
|
||||
"node_id": row.node_id,
|
||||
"node_run_id": row.node_run_id,
|
||||
"payload": payload if isinstance(payload, dict) else {},
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
max_retries: int = _APPEND_ATTEMPTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, taking its ``seq`` from the run's own counter.
|
||||
|
||||
``MAX(seq) + 1`` let two appenders draw the same number and burn a
|
||||
unique-constraint retry; here the counter increment and the insert share
|
||||
one transaction, and the row write-lock held by the UPDATE serialises
|
||||
concurrent appenders until commit — so seqs are strictly increasing,
|
||||
unique and gap-free per run, in commit order.
|
||||
"""
|
||||
text = _dump_payload(payload)
|
||||
for _ in range(max_retries):
|
||||
async with self._sf() as session:
|
||||
seq = await self._take_seq(session, run_id)
|
||||
row = WorkflowRunEventRow(
|
||||
run_id=run_id,
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
seq=seq,
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
payload_json=text,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
# Only reachable for legacy rows whose counter was backfilled
|
||||
# to 0 while events already existed. Re-sync the counter to
|
||||
# MAX(seq) in its own committed transaction, then retry.
|
||||
await session.rollback()
|
||||
await self._resync_counter(run_id)
|
||||
continue
|
||||
out = event_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
|
||||
|
||||
async def _take_seq(self, session: Any, run_id: str) -> int:
|
||||
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
|
||||
missing (event store used standalone in tests)."""
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
|
||||
)
|
||||
if result.rowcount:
|
||||
# Our own uncommitted increment; the row lock guards it until commit.
|
||||
current = (
|
||||
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
|
||||
).scalar()
|
||||
return int(current or 1)
|
||||
current = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar()
|
||||
return int(current or 0) + 1
|
||||
|
||||
async def _resync_counter(self, run_id: str) -> None:
|
||||
"""Point a legacy/stale run counter back at MAX(seq)."""
|
||||
async with self._sf() as session:
|
||||
max_seq = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar() or 0
|
||||
await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
|
||||
.values(next_event_seq=int(max_seq))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [event_to_dict(r) for r in rows]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
|
||||
return int(value or 0)
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
"""Workflow run / node-run / artifact persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunConflictError,
|
||||
WorkflowRunNotFoundError,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
|
||||
|
||||
__all__ = [
|
||||
"CANCELLABLE_RUN_STATUSES",
|
||||
"MemoryWorkflowRunStore",
|
||||
"SqlWorkflowRunStore",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunConflictError",
|
||||
"WorkflowRunNotFoundError",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowRunStore",
|
||||
"make_workflow_run_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_run_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowRunStore()
|
||||
return SqlWorkflowRunStore(session_factory)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Abstract workflow run / node-run / artifact store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
|
||||
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
|
||||
|
||||
|
||||
class WorkflowRunNotFoundError(Exception):
|
||||
def __init__(self, run_id: str) -> None:
|
||||
super().__init__(f"workflow run not found: {run_id}")
|
||||
self.run_id = run_id
|
||||
|
||||
|
||||
class WorkflowRunConflictError(Exception):
|
||||
"""Raised when a state transition is not legal for the current status."""
|
||||
|
||||
def __init__(self, run_id: str, status: str) -> None:
|
||||
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
|
||||
self.run_id = run_id
|
||||
self.status = status
|
||||
|
||||
|
||||
class WorkflowRunStore(ABC):
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Create a run. Returns ``(run, created)``; an existing
|
||||
``idempotency_key`` yields the stored run with ``created=False``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""CAS update. When ``lease_owner`` is given the row must still be leased
|
||||
by that owner. Terminal statuses clear the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pause atomically: status + token + pending descriptor (+ optional run
|
||||
context) in one conditional update, so a concurrent resume can never be
|
||||
overwritten by a follow-up context write."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
|
||||
``queued``. Returns None when the token is wrong/already used."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomic human-input resume in a single conditional update.
|
||||
|
||||
Validates the token, stores ``payload`` as ``context.resume_payload``
|
||||
(replacing the whole run context with ``context`` when given), clears the
|
||||
token/pending descriptor and flips the run to ``queued`` — all under
|
||||
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
|
||||
the resumed run on success, ``None`` when the token is wrong, already
|
||||
used, or the run is no longer awaiting input (caller maps that to 409).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
"""Mark node runs stale (used when a loop re-enters its body)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch one artifact row scoped to its run (authorization key)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
"""Delete finished runs older than the cutoff, with their node runs and
|
||||
artifact rows. Only terminal runs are eligible. Returns deleted run ids
|
||||
(bounded per call so a sweeper can loop)."""
|
||||
raise NotImplementedError
|
||||
+388
@@ -0,0 +1,388 @@
|
||||
"""In-memory workflow run store (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
class MemoryWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self) -> None:
|
||||
self._runs: dict[str, dict[str, Any]] = {}
|
||||
self._node_runs: dict[str, list[dict[str, Any]]] = {}
|
||||
self._artifacts: dict[str, list[dict[str, Any]]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
async with self._lock:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
for run in self._runs.values():
|
||||
if run.get("idempotency_key") == str(key):
|
||||
return copy.deepcopy(run), False
|
||||
now = datetime.now(UTC)
|
||||
run_id = str(data.get("id") or uuid.uuid4())
|
||||
run = {
|
||||
"id": run_id,
|
||||
"workflow_id": str(data["workflow_id"]),
|
||||
"workflow_version_id": str(data["workflow_version_id"]),
|
||||
"status": str(data.get("status") or "queued"),
|
||||
"owner_id": str(data["owner_id"]),
|
||||
"idempotency_key": str(key) if key else None,
|
||||
"input": data.get("input") or {},
|
||||
"output": None,
|
||||
"error": None,
|
||||
"context": data.get("context") or {},
|
||||
"pending_input": None,
|
||||
"resume_token": None,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"heartbeat_at": None,
|
||||
"attempt": 0,
|
||||
"max_attempts": int(data.get("max_attempts") or 3),
|
||||
"version": 0,
|
||||
"retry_of_run_id": data.get("retry_of_run_id"),
|
||||
"next_event_seq": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"_created": now,
|
||||
}
|
||||
self._runs[run_id] = run
|
||||
return copy.deepcopy(run), True
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
return copy.deepcopy(run) if run else None
|
||||
|
||||
async def note_event_seq(self, run_id: str, seq: int) -> None:
|
||||
"""Mirror the SQL counter semantics for no-DB mode: the memory event
|
||||
store reports each handed-out seq so run views can expose lastSeq."""
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
|
||||
run["next_event_seq"] = int(seq)
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
|
||||
if workflow_id:
|
||||
rows = [r for r in rows if r["workflow_id"] == workflow_id]
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
out: list[str] = []
|
||||
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
|
||||
if run["status"] == "queued":
|
||||
out.append(run["id"])
|
||||
elif run["status"] == "running" and run["lease_until"]:
|
||||
if datetime.fromisoformat(run["lease_until"]) < now:
|
||||
out.append(run["id"])
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
|
||||
if not claimable:
|
||||
return None
|
||||
run.update(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=_iso(lease_until),
|
||||
heartbeat_at=_iso(now),
|
||||
attempt=run["attempt"] + 1,
|
||||
version=run["version"] + 1,
|
||||
started_at=run["started_at"] or _iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
|
||||
return False
|
||||
run["lease_until"] = _iso(lease_until)
|
||||
run["heartbeat_at"] = _iso(datetime.now(UTC))
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
mapping = {
|
||||
"output_json": "output",
|
||||
"error_json": "error",
|
||||
"context_json": "context",
|
||||
"pending_input_json": "pending_input",
|
||||
}
|
||||
for key, value in fields.items():
|
||||
run[mapping.get(key, key)] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
run["lease_owner"] = None
|
||||
run["lease_until"] = None
|
||||
run["resume_token"] = None
|
||||
run["pending_input"] = None
|
||||
run["finished_at"] = run.get("finished_at") or _iso(now)
|
||||
for key in ("started_at", "finished_at"):
|
||||
if isinstance(run.get(key), datetime):
|
||||
run[key] = _iso(run[key])
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(now)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if run["status"] in ("cancel_requested", "cancelled"):
|
||||
return copy.deepcopy(run)
|
||||
if run["status"] not in CANCELLABLE_RUN_STATUSES:
|
||||
return None
|
||||
run["status"] = "cancel_requested"
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "cancel_requested":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
finished_at=_iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "running":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="awaiting_input",
|
||||
pending_input=copy.deepcopy(pending_input),
|
||||
resume_token=resume_token,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
if context is not None:
|
||||
run["context"] = copy.deepcopy(context)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
run.update(status="queued", resume_token=None, pending_input=None)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
base_context = context if context is not None else run.get("context") or {}
|
||||
run.update(
|
||||
context={**base_context, "resume_payload": copy.deepcopy(payload)},
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
rows = self._node_runs.setdefault(run_id, [])
|
||||
existing = next(
|
||||
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
|
||||
None,
|
||||
)
|
||||
if existing is None:
|
||||
existing = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": run_id,
|
||||
"node_id": node_id,
|
||||
"node_type": str(data.get("node_type") or "unknown"),
|
||||
"status": "queued",
|
||||
"attempt": attempt,
|
||||
"iteration": iteration,
|
||||
"is_current": True,
|
||||
"input": None,
|
||||
"output": None,
|
||||
"error": None,
|
||||
"parent_node_run_id": None,
|
||||
"duration_ms": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
rows.append(existing)
|
||||
for key in (
|
||||
"status",
|
||||
"node_type",
|
||||
"input",
|
||||
"output",
|
||||
"error",
|
||||
"parent_node_run_id",
|
||||
"duration_ms",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
|
||||
if "is_current" in data:
|
||||
existing["is_current"] = bool(data["is_current"])
|
||||
return copy.deepcopy(existing)
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
rows = self._node_runs.get(run_id, [])
|
||||
if current_only:
|
||||
rows = [r for r in rows if r["is_current"]]
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
count = 0
|
||||
for row in self._node_runs.get(run_id, []):
|
||||
if row["node_id"] in node_ids and row["is_current"]:
|
||||
row["is_current"] = False
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": str(data["run_id"]),
|
||||
"node_id": str(data.get("node_id") or ""),
|
||||
"name": str(data.get("name") or ""),
|
||||
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
|
||||
"path": str(data.get("path") or ""),
|
||||
"size_bytes": data.get("size_bytes"),
|
||||
"preview": data.get("preview"),
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
self._artifacts.setdefault(row["run_id"], []).append(row)
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
for row in self._artifacts.get(run_id, []):
|
||||
if row["id"] == artifact_id:
|
||||
return copy.deepcopy(row)
|
||||
return None
|
||||
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted: list[str] = []
|
||||
for run_id, run in list(self._runs.items()):
|
||||
if run["status"] not in TERMINAL_RUN_STATUSES:
|
||||
continue
|
||||
finished = _parse_dt(run.get("finished_at"))
|
||||
if finished is None or finished >= cutoff:
|
||||
continue
|
||||
deleted.append(run_id)
|
||||
for run_id in deleted:
|
||||
self._runs.pop(run_id, None)
|
||||
self._node_runs.pop(run_id, None)
|
||||
self._artifacts.pop(run_id, None)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""ORM models for workflow runs and per-node runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunRow(Base):
|
||||
"""One workflow execution bound to an immutable published version."""
|
||||
|
||||
__tablename__ = "workflow_runs"
|
||||
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
# Non-sensitive run context (never credentials).
|
||||
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
# Pending human-input descriptor when status == awaiting_input.
|
||||
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
# Last seq handed out by the event log for this run (0 = none yet). The
|
||||
# event store increments it in the same transaction that inserts the event
|
||||
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
|
||||
# first append self-heals from MAX(seq) on conflict.
|
||||
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowNodeRunRow(Base):
|
||||
"""Per-node execution record (also the resume checkpoint for a paused run)."""
|
||||
|
||||
__tablename__ = "workflow_node_runs"
|
||||
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# queued | running | completed | failed | skipped | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
|
||||
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class WorkflowRunArtifactRow(Base):
|
||||
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
|
||||
|
||||
__tablename__ = "workflow_run_artifacts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
|
||||
_RUN_UPDATABLE = {
|
||||
"status",
|
||||
"output_json",
|
||||
"error_json",
|
||||
"context_json",
|
||||
"pending_input_json",
|
||||
"resume_token",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
}
|
||||
|
||||
# Retention deletes in bounded batches so a long-neglected table cannot produce
|
||||
# one enormous transaction.
|
||||
_PURGE_BATCH = 500
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _load_text_json(raw: str | None) -> Any:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _dump_text_json(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"workflow_version_id": row.workflow_version_id,
|
||||
"status": row.status,
|
||||
"owner_id": row.owner_id,
|
||||
"idempotency_key": row.idempotency_key,
|
||||
"input": row.input_json or {},
|
||||
"output": row.output_json,
|
||||
"error": row.error_json,
|
||||
"context": row.context_json or {},
|
||||
"pending_input": row.pending_input_json,
|
||||
"resume_token": row.resume_token,
|
||||
"lease_owner": row.lease_owner,
|
||||
"lease_until": _iso(row.lease_until),
|
||||
"heartbeat_at": _iso(row.heartbeat_at),
|
||||
"attempt": row.attempt,
|
||||
"max_attempts": row.max_attempts,
|
||||
"version": row.version,
|
||||
"retry_of_run_id": row.retry_of_run_id,
|
||||
# Last seq handed out by the event log (0 = no events yet).
|
||||
"next_event_seq": int(row.next_event_seq or 0),
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"node_type": row.node_type,
|
||||
"status": row.status,
|
||||
"attempt": row.attempt,
|
||||
"iteration": row.iteration,
|
||||
"is_current": bool(row.is_current),
|
||||
"input": _load_text_json(row.input_snapshot),
|
||||
"output": _load_text_json(row.output_snapshot),
|
||||
"error": row.error_json,
|
||||
"parent_node_run_id": row.parent_node_run_id,
|
||||
"duration_ms": row.duration_ms,
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"name": row.name,
|
||||
"mime_type": row.mime_type,
|
||||
"path": row.path,
|
||||
"size_bytes": row.size_bytes,
|
||||
"preview": row.preview,
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
row = WorkflowRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
workflow_id=str(data["workflow_id"]),
|
||||
workflow_version_id=str(data["workflow_version_id"]),
|
||||
status=str(data.get("status") or "queued"),
|
||||
owner_id=str(data["owner_id"]),
|
||||
idempotency_key=str(key) if key else None,
|
||||
input_json=data.get("input") or {},
|
||||
context_json=data.get("context") or {},
|
||||
max_attempts=int(data.get("max_attempts") or 3),
|
||||
retry_of_run_id=data.get("retry_of_run_id"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
raise
|
||||
out = run_to_dict(row)
|
||||
await session.commit()
|
||||
return out, True
|
||||
|
||||
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunRow)
|
||||
if workflow_id:
|
||||
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowRunRow.status == status)
|
||||
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [run_to_dict(r) for r in rows]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
stmt = (
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
)
|
||||
)
|
||||
.order_by(WorkflowRunRow.created_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
heartbeat_at=now,
|
||||
attempt=WorkflowRunRow.attempt + 1,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
started_at=func.coalesce(WorkflowRunRow.started_at, now),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.lease_owner == lease_owner,
|
||||
WorkflowRunRow.status == "running",
|
||||
)
|
||||
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return False
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
|
||||
for key, value in fields.items():
|
||||
if key in _RUN_UPDATABLE:
|
||||
values[key] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
values["lease_owner"] = None
|
||||
values["lease_until"] = None
|
||||
values["resume_token"] = None
|
||||
values["pending_input_json"] = None
|
||||
values.setdefault("finished_at", now)
|
||||
where = [WorkflowRunRow.id == run_id]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
|
||||
)
|
||||
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
finished_at=now,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
values: dict[str, Any] = {
|
||||
"status": "awaiting_input",
|
||||
"pending_input_json": pending_input,
|
||||
"resume_token": resume_token,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"version": WorkflowRunRow.version + 1,
|
||||
"updated_at": now,
|
||||
}
|
||||
if context is not None:
|
||||
values["context_json"] = context
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
if context is None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
if row is None:
|
||||
return None
|
||||
context = dict(row.context_json or {})
|
||||
new_context = {**context, "resume_payload": payload}
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
context_json=new_context,
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(WorkflowNodeRunRow).where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id == node_id,
|
||||
WorkflowNodeRunRow.attempt == attempt,
|
||||
WorkflowNodeRunRow.iteration == iteration,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = WorkflowNodeRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
node_id=node_id,
|
||||
node_type=str(data.get("node_type") or "unknown"),
|
||||
attempt=attempt,
|
||||
iteration=iteration,
|
||||
)
|
||||
session.add(row)
|
||||
row.status = str(data.get("status") or row.status)
|
||||
row.node_type = str(data.get("node_type") or row.node_type)
|
||||
row.is_current = bool(data.get("is_current", True))
|
||||
if "input" in data:
|
||||
row.input_snapshot = _dump_text_json(data.get("input"))
|
||||
if "output" in data:
|
||||
row.output_snapshot = _dump_text_json(data.get("output"))
|
||||
if "error" in data:
|
||||
row.error_json = data.get("error")
|
||||
if data.get("parent_node_run_id"):
|
||||
row.parent_node_run_id = str(data["parent_node_run_id"])
|
||||
if data.get("duration_ms") is not None:
|
||||
row.duration_ms = int(data["duration_ms"])
|
||||
if data.get("started_at") is not None:
|
||||
row.started_at = data["started_at"]
|
||||
if data.get("finished_at") is not None:
|
||||
row.finished_at = data["finished_at"]
|
||||
await session.flush()
|
||||
out = node_run_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
|
||||
if current_only:
|
||||
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
|
||||
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [node_run_to_dict(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
if not node_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowNodeRunRow)
|
||||
.where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id.in_(node_ids),
|
||||
WorkflowNodeRunRow.is_current.is_(True),
|
||||
)
|
||||
.values(is_current=False)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = WorkflowRunArtifactRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=str(data["run_id"]),
|
||||
node_id=str(data.get("node_id") or ""),
|
||||
name=str(data.get("name") or "")[:512],
|
||||
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
|
||||
path=str(data.get("path") or ""),
|
||||
size_bytes=data.get("size_bytes"),
|
||||
preview=data.get("preview"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = artifact_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [artifact_to_dict(r) for r in rows]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(WorkflowRunArtifactRow).where(
|
||||
WorkflowRunArtifactRow.id == artifact_id,
|
||||
WorkflowRunArtifactRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return artifact_to_dict(row) if row is not None else None
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
run_ids = (
|
||||
(
|
||||
await session.execute(
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
|
||||
WorkflowRunRow.finished_at.is_not(None),
|
||||
WorkflowRunRow.finished_at < cutoff,
|
||||
)
|
||||
.limit(_PURGE_BATCH)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not run_ids:
|
||||
return []
|
||||
ids = list(run_ids)
|
||||
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
|
||||
await session.commit()
|
||||
return ids
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
"""Workflow definition / version persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
WorkflowVersionNotFoundError,
|
||||
)
|
||||
from deerflow.persistence.workflows.memory import MemoryWorkflowStore
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
from deerflow.persistence.workflows.sql import WorkflowRepository
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowStore",
|
||||
"WorkflowDefinitionRow",
|
||||
"WorkflowDraftConflictError",
|
||||
"WorkflowNotFoundError",
|
||||
"WorkflowRepository",
|
||||
"WorkflowStore",
|
||||
"WorkflowVersionNotFoundError",
|
||||
"WorkflowVersionRow",
|
||||
"make_workflow_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowStore()
|
||||
return WorkflowRepository(session_factory)
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
"""Abstract workflow definition / version store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowDraftConflictError(Exception):
|
||||
"""Raised when ``expected_revision`` does not match the stored draft revision."""
|
||||
|
||||
def __init__(self, workflow_id: str, current_revision: int) -> None:
|
||||
super().__init__(f"draft conflict for {workflow_id}: current={current_revision}")
|
||||
self.workflow_id = workflow_id
|
||||
self.current_revision = current_revision
|
||||
|
||||
|
||||
class WorkflowNotFoundError(Exception):
|
||||
def __init__(self, workflow_id: str) -> None:
|
||||
super().__init__(f"workflow not found: {workflow_id}")
|
||||
self.workflow_id = workflow_id
|
||||
|
||||
|
||||
class WorkflowVersionNotFoundError(Exception):
|
||||
def __init__(self, version_id: str) -> None:
|
||||
super().__init__(f"workflow version not found: {version_id}")
|
||||
self.version_id = version_id
|
||||
|
||||
|
||||
class WorkflowStore(ABC):
|
||||
@abstractmethod
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""CAS update of ``draft_graph_json``. Raises ``WorkflowDraftConflictError``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Atomically CAS-update BOTH the execution graph and the canvas document.
|
||||
|
||||
A single conditional UPDATE bumps ``draft_revision`` and writes
|
||||
``draft_graph_json`` + ``draft_canvas_schema_json`` together, so the two
|
||||
representations can never diverge across revisions. Raises
|
||||
``WorkflowDraftConflictError`` on a stale ``expected_revision``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
"""Lightweight list for subworkflow resource catalog."""
|
||||
raise NotImplementedError
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"""In-memory workflow store for tests / no-DB mode."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat()
|
||||
|
||||
|
||||
class MemoryWorkflowStore(WorkflowStore):
|
||||
def __init__(self) -> None:
|
||||
self._definitions: dict[str, dict[str, Any]] = {}
|
||||
self._versions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = list(self._definitions.values())
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
elif not include_archived:
|
||||
rows = [r for r in rows if r["status"] != "archived"]
|
||||
rows.sort(key=lambda r: r.get("updated_at") or "", reverse=True)
|
||||
return [
|
||||
{k: v for k, v in r.items() if k not in ("draft_graph", "draft_canvas_schema", "next_version_number")}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
out = copy.deepcopy(row)
|
||||
if not include_draft:
|
||||
out.pop("draft_graph", None)
|
||||
return out
|
||||
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
|
||||
graph = copy.deepcopy(data.get("draft_graph") or data.get("graph") or {
|
||||
"schemaVersion": "1.0",
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or "未命名工作流",
|
||||
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
|
||||
"edges": [],
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
"outputSchema": {"type": "object", "properties": {}},
|
||||
"settings": {},
|
||||
})
|
||||
graph.setdefault("id", workflow_id)
|
||||
now = _now()
|
||||
row = {
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or graph.get("name") or "未命名工作流",
|
||||
"description": data.get("description") or "",
|
||||
"owner_id": data["owner_id"],
|
||||
"status": data.get("status") or "draft",
|
||||
"draft_revision": 0,
|
||||
"next_version_number": 1,
|
||||
"draft_graph": graph,
|
||||
"draft_canvas_schema": str(data.get("draft_canvas_schema") or "{}"),
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._definitions[workflow_id] = row
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data and data["name"] is not None:
|
||||
row["name"] = data["name"]
|
||||
if "description" in data and data["description"] is not None:
|
||||
row["description"] = data["description"]
|
||||
if "status" in data and data["status"] is not None:
|
||||
row["status"] = data["status"]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
return await self.update_definition(workflow_id, {"status": "archived"})
|
||||
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row["draft_revision"] != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
|
||||
payload = copy.deepcopy(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
row["draft_graph"] = payload
|
||||
row["draft_revision"] = expected_revision + 1
|
||||
if payload.get("name"):
|
||||
row["name"] = str(payload["name"])[:255]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by
|
||||
row = self._definitions.get(workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row["draft_revision"] != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row["draft_revision"])
|
||||
payload = copy.deepcopy(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
row["draft_graph"] = payload
|
||||
row["draft_canvas_schema"] = canvas_schema or "{}"
|
||||
row["draft_revision"] = expected_revision + 1
|
||||
if payload.get("name"):
|
||||
row["name"] = str(payload["name"])[:255]
|
||||
row["updated_at"] = _now()
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
definition = self._definitions.get(workflow_id)
|
||||
if definition is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
existing = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
|
||||
# Allocate from the definition's own counter (mirrors the SQL store) so
|
||||
# the number never goes backwards even if events are purged/replayed.
|
||||
counter = int(definition.get("next_version_number") or 1)
|
||||
version_number = max(counter, max((v["version_number"] for v in existing), default=0) + 1)
|
||||
definition["next_version_number"] = version_number + 1
|
||||
version_id = f"wv_{uuid.uuid4().hex[:12]}"
|
||||
version = {
|
||||
"id": version_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_number": version_number,
|
||||
"graph": copy.deepcopy(graph),
|
||||
"graph_hash": graph_hash,
|
||||
"input_schema": graph.get("inputSchema") or graph.get("input_schema") or {},
|
||||
"output_schema": graph.get("outputSchema") or graph.get("output_schema") or {},
|
||||
"published_by": published_by,
|
||||
"published_at": _now(),
|
||||
"change_note": change_note or "",
|
||||
}
|
||||
self._versions[version_id] = version
|
||||
definition["status"] = "active"
|
||||
if canvas_schema_hash:
|
||||
definition["published_canvas_schema_hash"] = str(canvas_schema_hash)[:64]
|
||||
definition["updated_at"] = _now()
|
||||
return copy.deepcopy(version)
|
||||
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
rows = [v for v in self._versions.values() if v["workflow_id"] == workflow_id]
|
||||
rows.sort(key=lambda r: r["version_number"], reverse=True)
|
||||
return [{k: v for k, v in r.items() if k != "graph"} for r in rows]
|
||||
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
row = self._versions.get(version_id)
|
||||
return copy.deepcopy(row) if row is not None else None
|
||||
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
by_wf: dict[str, dict[str, Any]] = {}
|
||||
for version in self._versions.values():
|
||||
wf_id = version["workflow_id"]
|
||||
if exclude_workflow_id and wf_id == exclude_workflow_id:
|
||||
continue
|
||||
definition = self._definitions.get(wf_id)
|
||||
if definition is None or definition.get("status") == "archived":
|
||||
continue
|
||||
current = by_wf.get(wf_id)
|
||||
if current is None or version["version_number"] > current["version_number"]:
|
||||
by_wf[wf_id] = {
|
||||
"workflow_id": wf_id,
|
||||
"name": definition["name"],
|
||||
"version_id": version["id"],
|
||||
"version_number": version["version_number"],
|
||||
"graph_hash": version["graph_hash"],
|
||||
"published_at": version["published_at"],
|
||||
}
|
||||
return sorted(by_wf.values(), key=lambda r: r.get("published_at") or "", reverse=True)
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
"""ORM models for workflow definitions and immutable published versions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowDefinitionRow(Base):
|
||||
"""Mutable workflow metadata + current draft canvas."""
|
||||
|
||||
__tablename__ = "workflow_definitions"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
description: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# draft | active | archived
|
||||
status: Mapped[str] = mapped_column(String(32), default="draft", nullable=False, index=True)
|
||||
draft_revision: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
|
||||
# Monotonic version allocator. "1" = no version published yet; publish takes
|
||||
# the value then CAS-bumps it, replacing the old MAX(version_number)+1 race.
|
||||
next_version_number: Mapped[int] = mapped_column(
|
||||
Integer, nullable=False, default=1, server_default=sa_text("1")
|
||||
)
|
||||
draft_graph_json: Mapped[str] = mapped_column(PortableLongText(), default="{}", nullable=False)
|
||||
# Coze canvas document (opaque JSON text). Never interpreted by the runtime;
|
||||
# the executable WorkflowGraph lives in draft_graph_json above.
|
||||
draft_canvas_schema_json: Mapped[str] = mapped_column(
|
||||
PortableLongText(), default="{}", server_default=sa_text("'{}'"), nullable=False
|
||||
)
|
||||
# sha256 of the canvas document the current published version was compiled
|
||||
# from (audit trail only; publish always freezes the execution graph).
|
||||
published_canvas_schema_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowVersionRow(Base):
|
||||
"""Immutable published graph snapshot. Runs bind to version id."""
|
||||
|
||||
__tablename__ = "workflow_versions"
|
||||
__table_args__ = (UniqueConstraint("workflow_id", "version_number", name="uq_workflow_versions_wf_num"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(
|
||||
String(64),
|
||||
ForeignKey("workflow_definitions.id", ondelete="CASCADE"),
|
||||
nullable=False,
|
||||
index=True,
|
||||
)
|
||||
version_number: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
graph_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False)
|
||||
graph_hash: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
input_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_schema: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
published_by: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
published_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
change_note: Mapped[str] = mapped_column(Text, default="", nullable=False)
|
||||
+385
@@ -0,0 +1,385 @@
|
||||
"""SQLAlchemy-backed workflow definition / version store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflows.base import (
|
||||
WorkflowDraftConflictError,
|
||||
WorkflowNotFoundError,
|
||||
WorkflowStore,
|
||||
)
|
||||
from deerflow.persistence.workflows.model import WorkflowDefinitionRow, WorkflowVersionRow
|
||||
|
||||
# Concurrent publishers CAS-retry against the workflow's own counter; a handful
|
||||
# of attempts is far more than any realistic contention window needs.
|
||||
_PUBLISH_ATTEMPTS = 6
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _parse_graph(raw: str | dict[str, Any] | None) -> dict[str, Any]:
|
||||
if raw is None:
|
||||
return {}
|
||||
if isinstance(raw, dict):
|
||||
return raw
|
||||
text = str(raw).strip()
|
||||
if not text:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _dump_graph(graph: dict[str, Any]) -> str:
|
||||
return json.dumps(graph, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
||||
|
||||
|
||||
def _definition_dict(row: WorkflowDefinitionRow, *, include_draft: bool) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": row.id,
|
||||
"name": row.name,
|
||||
"description": row.description or "",
|
||||
"owner_id": row.owner_id,
|
||||
"status": row.status,
|
||||
"draft_revision": row.draft_revision,
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
if include_draft:
|
||||
data["draft_graph"] = _parse_graph(row.draft_graph_json)
|
||||
# Canvas document as stored (opaque JSON text; "{}" when never saved).
|
||||
data["draft_canvas_schema"] = row.draft_canvas_schema_json or "{}"
|
||||
return data
|
||||
|
||||
|
||||
def _version_dict(row: WorkflowVersionRow, *, include_graph: bool = True) -> dict[str, Any]:
|
||||
data = {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_number": row.version_number,
|
||||
"graph_hash": row.graph_hash,
|
||||
"input_schema": row.input_schema or {},
|
||||
"output_schema": row.output_schema or {},
|
||||
"published_by": row.published_by,
|
||||
"published_at": _iso(row.published_at),
|
||||
"change_note": row.change_note or "",
|
||||
}
|
||||
if include_graph:
|
||||
data["graph"] = _parse_graph(row.graph_json)
|
||||
return data
|
||||
|
||||
|
||||
class WorkflowRepository(WorkflowStore):
|
||||
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def list_definitions(
|
||||
self,
|
||||
*,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
include_archived: bool = False,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowDefinitionRow)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.status == status)
|
||||
elif not include_archived:
|
||||
stmt = stmt.where(WorkflowDefinitionRow.status != "archived")
|
||||
stmt = stmt.order_by(WorkflowDefinitionRow.updated_at.desc())
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_definition_dict(row, include_draft=False) for row in result.scalars()]
|
||||
|
||||
async def get_definition(self, workflow_id: str, *, include_draft: bool = True) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
return _definition_dict(row, include_draft=include_draft) if row is not None else None
|
||||
|
||||
async def create_definition(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
workflow_id = data.get("id") or f"wf_{uuid.uuid4().hex[:12]}"
|
||||
graph = data.get("draft_graph") or data.get("graph") or {
|
||||
"schemaVersion": "1.0",
|
||||
"id": workflow_id,
|
||||
"name": data.get("name") or "未命名工作流",
|
||||
"nodes": [{"id": "start_1", "type": "start", "name": "开始", "config": {}}],
|
||||
"edges": [],
|
||||
"inputSchema": {"type": "object", "properties": {}},
|
||||
"outputSchema": {"type": "object", "properties": {}},
|
||||
"settings": {},
|
||||
}
|
||||
if isinstance(graph, dict) and not graph.get("id"):
|
||||
graph = {**graph, "id": workflow_id}
|
||||
if isinstance(graph, dict) and data.get("name") and not graph.get("name"):
|
||||
graph = {**graph, "name": data["name"]}
|
||||
row = WorkflowDefinitionRow(
|
||||
id=workflow_id,
|
||||
name=data.get("name") or graph.get("name") or "未命名工作流",
|
||||
description=data.get("description") or "",
|
||||
owner_id=data["owner_id"],
|
||||
status=data.get("status") or "draft",
|
||||
draft_revision=0,
|
||||
draft_graph_json=_dump_graph(graph),
|
||||
draft_canvas_schema_json=str(data.get("draft_canvas_schema") or "{}"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = _definition_dict(row, include_draft=True)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def update_definition(self, workflow_id: str, data: dict[str, Any]) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
return None
|
||||
if "name" in data and data["name"] is not None:
|
||||
row.name = data["name"]
|
||||
if "description" in data and data["description"] is not None:
|
||||
row.description = data["description"]
|
||||
if "status" in data and data["status"] is not None:
|
||||
row.status = data["status"]
|
||||
row.updated_at = datetime.now(UTC)
|
||||
await session.flush()
|
||||
out = _definition_dict(row, include_draft=True)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def archive_definition(self, workflow_id: str) -> dict[str, Any] | None:
|
||||
return await self.update_definition(workflow_id, {"status": "archived"})
|
||||
|
||||
async def save_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by # audit reserved for later
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row.draft_revision != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
|
||||
payload = dict(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
if row.name and not payload.get("name"):
|
||||
payload["name"] = row.name
|
||||
# Conditional UPDATE on the revision itself: two sessions that both
|
||||
# read the same revision cannot both win — the loser's WHERE clause
|
||||
# matches zero rows and surfaces a deterministic conflict instead of
|
||||
# silently overwriting the other editor's draft.
|
||||
values: dict[str, Any] = {
|
||||
"draft_graph_json": _dump_graph(payload),
|
||||
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
if payload.get("name"):
|
||||
values["name"] = str(payload["name"])[:255]
|
||||
result = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.draft_revision == expected_revision,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
current = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if current is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
|
||||
await session.commit()
|
||||
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
assert fresh is not None
|
||||
return _definition_dict(fresh, include_draft=True)
|
||||
|
||||
async def save_studio_draft(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
expected_revision: int,
|
||||
graph: dict[str, Any],
|
||||
canvas_schema: str = "{}",
|
||||
updated_by: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del updated_by # audit reserved for later
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if row is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
if row.draft_revision != expected_revision:
|
||||
raise WorkflowDraftConflictError(workflow_id, row.draft_revision)
|
||||
payload = dict(graph)
|
||||
payload.setdefault("id", workflow_id)
|
||||
if row.name and not payload.get("name"):
|
||||
payload["name"] = row.name
|
||||
# One conditional UPDATE writes graph + canvas + revision together:
|
||||
# the two representations are always from the same save, and a
|
||||
# concurrent editor loses deterministically instead of interleaving
|
||||
# a new canvas onto someone else's graph.
|
||||
values: dict[str, Any] = {
|
||||
"draft_graph_json": _dump_graph(payload),
|
||||
"draft_canvas_schema_json": canvas_schema or "{}",
|
||||
"draft_revision": WorkflowDefinitionRow.draft_revision + 1,
|
||||
"updated_at": datetime.now(UTC),
|
||||
}
|
||||
if payload.get("name"):
|
||||
values["name"] = str(payload["name"])[:255]
|
||||
result = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.draft_revision == expected_revision,
|
||||
)
|
||||
.values(**values)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
current = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if current is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
raise WorkflowDraftConflictError(workflow_id, current.draft_revision)
|
||||
await session.commit()
|
||||
fresh = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
assert fresh is not None
|
||||
return _definition_dict(fresh, include_draft=True)
|
||||
|
||||
async def publish_version(
|
||||
self,
|
||||
workflow_id: str,
|
||||
*,
|
||||
graph: dict[str, Any],
|
||||
graph_hash: str,
|
||||
published_by: str | None = None,
|
||||
change_note: str = "",
|
||||
canvas_schema_hash: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Allocate the version number from the workflow's own counter.
|
||||
|
||||
``MAX(version_number) + 1`` races between concurrent publishes; here the
|
||||
number is claimed by CAS-bumping ``next_version_number`` inside the same
|
||||
transaction that inserts the version row, and the unique
|
||||
``(workflow_id, version_number)`` constraint is the final safety net.
|
||||
Concurrent publishers get adjacent, distinct numbers.
|
||||
"""
|
||||
last_error: Exception | None = None
|
||||
for _ in range(_PUBLISH_ATTEMPTS):
|
||||
try:
|
||||
async with self._sf() as session:
|
||||
definition = await session.get(WorkflowDefinitionRow, workflow_id)
|
||||
if definition is None:
|
||||
raise WorkflowNotFoundError(workflow_id)
|
||||
counter_seen = int(definition.next_version_number or 1)
|
||||
# Legacy rows got the column auto-added with server_default 1
|
||||
# while versions already exist — never allocate below the max.
|
||||
latest = (
|
||||
await session.execute(
|
||||
select(func.max(WorkflowVersionRow.version_number)).where(
|
||||
WorkflowVersionRow.workflow_id == workflow_id
|
||||
)
|
||||
)
|
||||
).scalar() or 0
|
||||
number = max(counter_seen, int(latest) + 1)
|
||||
claimed = await session.execute(
|
||||
update(WorkflowDefinitionRow)
|
||||
.where(
|
||||
WorkflowDefinitionRow.id == workflow_id,
|
||||
WorkflowDefinitionRow.next_version_number == counter_seen,
|
||||
)
|
||||
.values(next_version_number=number + 1)
|
||||
)
|
||||
if claimed.rowcount == 0:
|
||||
# Another publish landed first — release nothing (the
|
||||
# counter bump never happened) and retry with fresh state.
|
||||
await session.rollback()
|
||||
last_error = None
|
||||
continue
|
||||
version = WorkflowVersionRow(
|
||||
id=f"wv_{uuid.uuid4().hex[:12]}",
|
||||
workflow_id=workflow_id,
|
||||
version_number=number,
|
||||
graph_json=_dump_graph(graph),
|
||||
graph_hash=graph_hash,
|
||||
input_schema=graph.get("inputSchema") or graph.get("input_schema") or {},
|
||||
output_schema=graph.get("outputSchema") or graph.get("output_schema") or {},
|
||||
published_by=published_by,
|
||||
change_note=change_note or "",
|
||||
)
|
||||
definition.status = "active"
|
||||
definition.updated_at = datetime.now(UTC)
|
||||
if canvas_schema_hash:
|
||||
definition.published_canvas_schema_hash = canvas_schema_hash[:64]
|
||||
session.add(version)
|
||||
await session.flush()
|
||||
out = _version_dict(version, include_graph=True)
|
||||
await session.commit()
|
||||
return out
|
||||
except IntegrityError as exc: # pragma: no cover - CAS makes this near-impossible
|
||||
last_error = exc
|
||||
continue
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError(f"failed to allocate a workflow version number for {workflow_id}")
|
||||
|
||||
async def list_versions(self, workflow_id: str) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(WorkflowVersionRow)
|
||||
.where(WorkflowVersionRow.workflow_id == workflow_id)
|
||||
.order_by(WorkflowVersionRow.version_number.desc())
|
||||
)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
return [_version_dict(row, include_graph=False) for row in result.scalars()]
|
||||
|
||||
async def get_version(self, version_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowVersionRow, version_id)
|
||||
return _version_dict(row, include_graph=True) if row is not None else None
|
||||
|
||||
async def list_published_summaries(self, *, exclude_workflow_id: str | None = None) -> list[dict[str, Any]]:
|
||||
stmt = (
|
||||
select(WorkflowVersionRow, WorkflowDefinitionRow)
|
||||
.join(WorkflowDefinitionRow, WorkflowDefinitionRow.id == WorkflowVersionRow.workflow_id)
|
||||
.where(WorkflowDefinitionRow.status != "archived")
|
||||
.order_by(WorkflowVersionRow.published_at.desc())
|
||||
)
|
||||
if exclude_workflow_id:
|
||||
stmt = stmt.where(WorkflowVersionRow.workflow_id != exclude_workflow_id)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(stmt)
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for version, definition in result.all():
|
||||
# Latest version per workflow only.
|
||||
if version.workflow_id in seen:
|
||||
continue
|
||||
seen.add(version.workflow_id)
|
||||
out.append(
|
||||
{
|
||||
"workflow_id": definition.id,
|
||||
"name": definition.name,
|
||||
"version_id": version.id,
|
||||
"version_number": version.version_number,
|
||||
"graph_hash": version.graph_hash,
|
||||
"published_at": _iso(version.published_at),
|
||||
}
|
||||
)
|
||||
return out
|
||||
Reference in new issue
Block a user