代码提交

This commit is contained in:
董宏凯 committed 2026-08-30 01:25:20 +08:00
1 parent 4ac7bc21cd
commit 17cf849361
110 files changed
+17007 -518

No files matched your search

@@ -0,0 +1,38 @@
"""Workflow run / node-run / artifact persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunConflictError,
WorkflowRunNotFoundError,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
__all__ = [
"CANCELLABLE_RUN_STATUSES",
"MemoryWorkflowRunStore",
"SqlWorkflowRunStore",
"TERMINAL_RUN_STATUSES",
"WorkflowNodeRunRow",
"WorkflowRunArtifactRow",
"WorkflowRunConflictError",
"WorkflowRunNotFoundError",
"WorkflowRunRow",
"WorkflowRunStore",
"make_workflow_run_store",
]
def make_workflow_run_store(session_factory):
if session_factory is None:
return MemoryWorkflowRunStore()
return SqlWorkflowRunStore(session_factory)
@@ -0,0 +1,161 @@
"""Abstract workflow run / node-run / artifact store."""
from __future__ import annotations
from abc import ABC, abstractmethod
from datetime import datetime
from typing import Any
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
class WorkflowRunNotFoundError(Exception):
def __init__(self, run_id: str) -> None:
super().__init__(f"workflow run not found: {run_id}")
self.run_id = run_id
class WorkflowRunConflictError(Exception):
"""Raised when a state transition is not legal for the current status."""
def __init__(self, run_id: str, status: str) -> None:
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
self.run_id = run_id
self.status = status
class WorkflowRunStore(ABC):
# ── runs ────────────────────────────────────────────────────────────
@abstractmethod
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
"""Create a run. Returns ``(run, created)``; an existing
``idempotency_key`` yields the stored run with ``created=False``."""
raise NotImplementedError
@abstractmethod
async def get_run(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
raise NotImplementedError
@abstractmethod
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
raise NotImplementedError
@abstractmethod
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
"""CAS update. When ``lease_owner`` is given the row must still be leased
by that owner. Terminal statuses clear the lease."""
raise NotImplementedError
@abstractmethod
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
raise NotImplementedError
@abstractmethod
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Pause atomically: status + token + pending descriptor (+ optional run
context) in one conditional update, so a concurrent resume can never be
overwritten by a follow-up context write."""
raise NotImplementedError
@abstractmethod
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
``queued``. Returns None when the token is wrong/already used."""
raise NotImplementedError
@abstractmethod
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic human-input resume in a single conditional update.
Validates the token, stores ``payload`` as ``context.resume_payload``
(replacing the whole run context with ``context`` when given), clears the
token/pending descriptor and flips the run to ``queued`` — all under
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
the resumed run on success, ``None`` when the token is wrong, already
used, or the run is no longer awaiting input (caller maps that to 409).
"""
raise NotImplementedError
# ── node runs ───────────────────────────────────────────────────────
@abstractmethod
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
raise NotImplementedError
@abstractmethod
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
"""Mark node runs stale (used when a loop re-enters its body)."""
raise NotImplementedError
# ── artifacts ───────────────────────────────────────────────────────
@abstractmethod
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
raise NotImplementedError
@abstractmethod
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
"""Fetch one artifact row scoped to its run (authorization key)."""
raise NotImplementedError
# ── retention ───────────────────────────────────────────────────────
@abstractmethod
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
"""Delete finished runs older than the cutoff, with their node runs and
artifact rows. Only terminal runs are eligible. Returns deleted run ids
(bounded per call so a sweeper can loop)."""
raise NotImplementedError
@@ -0,0 +1,388 @@
"""In-memory workflow run store (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
class MemoryWorkflowRunStore(WorkflowRunStore):
def __init__(self) -> None:
self._runs: dict[str, dict[str, Any]] = {}
self._node_runs: dict[str, list[dict[str, Any]]] = {}
self._artifacts: dict[str, list[dict[str, Any]]] = {}
self._lock = asyncio.Lock()
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
async with self._lock:
key = data.get("idempotency_key")
if key:
for run in self._runs.values():
if run.get("idempotency_key") == str(key):
return copy.deepcopy(run), False
now = datetime.now(UTC)
run_id = str(data.get("id") or uuid.uuid4())
run = {
"id": run_id,
"workflow_id": str(data["workflow_id"]),
"workflow_version_id": str(data["workflow_version_id"]),
"status": str(data.get("status") or "queued"),
"owner_id": str(data["owner_id"]),
"idempotency_key": str(key) if key else None,
"input": data.get("input") or {},
"output": None,
"error": None,
"context": data.get("context") or {},
"pending_input": None,
"resume_token": None,
"lease_owner": None,
"lease_until": None,
"heartbeat_at": None,
"attempt": 0,
"max_attempts": int(data.get("max_attempts") or 3),
"version": 0,
"retry_of_run_id": data.get("retry_of_run_id"),
"next_event_seq": 0,
"started_at": None,
"finished_at": None,
"created_at": _iso(now),
"updated_at": _iso(now),
"_created": now,
}
self._runs[run_id] = run
return copy.deepcopy(run), True
async def get_run(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
return copy.deepcopy(run) if run else None
async def note_event_seq(self, run_id: str, seq: int) -> None:
"""Mirror the SQL counter semantics for no-DB mode: the memory event
store reports each handed-out seq so run views can expose lastSeq."""
async with self._lock:
run = self._runs.get(run_id)
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
run["next_event_seq"] = int(seq)
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
if workflow_id:
rows = [r for r in rows if r["workflow_id"] == workflow_id]
if owner_id:
rows = [r for r in rows if r["owner_id"] == owner_id]
if status:
rows = [r for r in rows if r["status"] == status]
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
out: list[str] = []
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
if run["status"] == "queued":
out.append(run["id"])
elif run["status"] == "running" and run["lease_until"]:
if datetime.fromisoformat(run["lease_until"]) < now:
out.append(run["id"])
if len(out) >= limit:
break
return out
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
now = datetime.now(UTC)
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
if not claimable:
return None
run.update(
status="running",
lease_owner=lease_owner,
lease_until=_iso(lease_until),
heartbeat_at=_iso(now),
attempt=run["attempt"] + 1,
version=run["version"] + 1,
started_at=run["started_at"] or _iso(now),
updated_at=_iso(now),
)
return copy.deepcopy(run)
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
run = self._runs.get(run_id)
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
return False
run["lease_until"] = _iso(lease_until)
run["heartbeat_at"] = _iso(datetime.now(UTC))
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None:
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
mapping = {
"output_json": "output",
"error_json": "error",
"context_json": "context",
"pending_input_json": "pending_input",
}
for key, value in fields.items():
run[mapping.get(key, key)] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
run["lease_owner"] = None
run["lease_until"] = None
run["resume_token"] = None
run["pending_input"] = None
run["finished_at"] = run.get("finished_at") or _iso(now)
for key in ("started_at", "finished_at"):
if isinstance(run.get(key), datetime):
run[key] = _iso(run[key])
run["version"] += 1
run["updated_at"] = _iso(now)
return copy.deepcopy(run)
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None:
return None
if run["status"] in ("cancel_requested", "cancelled"):
return copy.deepcopy(run)
if run["status"] not in CANCELLABLE_RUN_STATUSES:
return None
run["status"] = "cancel_requested"
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
run = self._runs.get(run_id)
if run is None or run["status"] != "cancel_requested":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input=None,
finished_at=_iso(now),
updated_at=_iso(now),
)
run["version"] += 1
return copy.deepcopy(run)
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "running":
return None
if lease_owner is not None and run["lease_owner"] != lease_owner:
return None
now = datetime.now(UTC)
run.update(
status="awaiting_input",
pending_input=copy.deepcopy(pending_input),
resume_token=resume_token,
lease_owner=None,
lease_until=None,
updated_at=_iso(now),
)
if context is not None:
run["context"] = copy.deepcopy(context)
run["version"] += 1
return copy.deepcopy(run)
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
run.update(status="queued", resume_token=None, pending_input=None)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
async with self._lock:
run = self._runs.get(run_id)
if run is None or run["status"] != "awaiting_input":
return None
if not run.get("resume_token") or run["resume_token"] != resume_token:
return None
base_context = context if context is not None else run.get("context") or {}
run.update(
context={**base_context, "resume_payload": copy.deepcopy(payload)},
status="queued",
resume_token=None,
pending_input=None,
)
run["version"] += 1
run["updated_at"] = _iso(datetime.now(UTC))
return copy.deepcopy(run)
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
rows = self._node_runs.setdefault(run_id, [])
existing = next(
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
None,
)
if existing is None:
existing = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": run_id,
"node_id": node_id,
"node_type": str(data.get("node_type") or "unknown"),
"status": "queued",
"attempt": attempt,
"iteration": iteration,
"is_current": True,
"input": None,
"output": None,
"error": None,
"parent_node_run_id": None,
"duration_ms": None,
"started_at": None,
"finished_at": None,
"created_at": _iso(datetime.now(UTC)),
}
rows.append(existing)
for key in (
"status",
"node_type",
"input",
"output",
"error",
"parent_node_run_id",
"duration_ms",
"started_at",
"finished_at",
):
if key in data and data[key] is not None:
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
if "is_current" in data:
existing["is_current"] = bool(data["is_current"])
return copy.deepcopy(existing)
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
rows = self._node_runs.get(run_id, [])
if current_only:
rows = [r for r in rows if r["is_current"]]
return [copy.deepcopy(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
count = 0
for row in self._node_runs.get(run_id, []):
if row["node_id"] in node_ids and row["is_current"]:
row["is_current"] = False
count += 1
return count
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = {
"id": str(data.get("id") or uuid.uuid4()),
"run_id": str(data["run_id"]),
"node_id": str(data.get("node_id") or ""),
"name": str(data.get("name") or ""),
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
"path": str(data.get("path") or ""),
"size_bytes": data.get("size_bytes"),
"preview": data.get("preview"),
"created_at": _iso(datetime.now(UTC)),
}
self._artifacts.setdefault(row["run_id"], []).append(row)
return copy.deepcopy(row)
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
for row in self._artifacts.get(run_id, []):
if row["id"] == artifact_id:
return copy.deepcopy(row)
return None
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted: list[str] = []
for run_id, run in list(self._runs.items()):
if run["status"] not in TERMINAL_RUN_STATUSES:
continue
finished = _parse_dt(run.get("finished_at"))
if finished is None or finished >= cutoff:
continue
deleted.append(run_id)
for run_id in deleted:
self._runs.pop(run_id, None)
self._node_runs.pop(run_id, None)
self._artifacts.pop(run_id, None)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,96 @@
"""ORM models for workflow runs and per-node runs."""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy import text as sa_text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
class WorkflowRunRow(Base):
"""One workflow execution bound to an immutable published version."""
__tablename__ = "workflow_runs"
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
# Non-sensitive run context (never credentials).
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
# Pending human-input descriptor when status == awaiting_input.
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
# Last seq handed out by the event log for this run (0 = none yet). The
# event store increments it in the same transaction that inserts the event
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
# first append self-heals from MAX(seq) on conflict.
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
class WorkflowNodeRunRow(Base):
"""Per-node execution record (also the resume checkpoint for a paused run)."""
__tablename__ = "workflow_node_runs"
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
# queued | running | completed | failed | skipped | cancelled
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
class WorkflowRunArtifactRow(Base):
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
__tablename__ = "workflow_run_artifacts"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
name: Mapped[str] = mapped_column(String(512), nullable=False)
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,566 @@
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
from __future__ import annotations
import json
import uuid
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_runs.base import (
CANCELLABLE_RUN_STATUSES,
TERMINAL_RUN_STATUSES,
WorkflowRunStore,
)
from deerflow.persistence.workflow_runs.model import (
WorkflowNodeRunRow,
WorkflowRunArtifactRow,
WorkflowRunRow,
)
_RUN_UPDATABLE = {
"status",
"output_json",
"error_json",
"context_json",
"pending_input_json",
"resume_token",
"started_at",
"finished_at",
}
# Retention deletes in bounded batches so a long-neglected table cannot produce
# one enormous transaction.
_PURGE_BATCH = 500
def _iso(value: datetime | None) -> str | None:
return value.isoformat() if isinstance(value, datetime) else None
def _load_text_json(raw: str | None) -> Any:
if not raw:
return None
try:
return json.loads(raw)
except (TypeError, json.JSONDecodeError):
return None
def _dump_text_json(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
return {
"id": row.id,
"workflow_id": row.workflow_id,
"workflow_version_id": row.workflow_version_id,
"status": row.status,
"owner_id": row.owner_id,
"idempotency_key": row.idempotency_key,
"input": row.input_json or {},
"output": row.output_json,
"error": row.error_json,
"context": row.context_json or {},
"pending_input": row.pending_input_json,
"resume_token": row.resume_token,
"lease_owner": row.lease_owner,
"lease_until": _iso(row.lease_until),
"heartbeat_at": _iso(row.heartbeat_at),
"attempt": row.attempt,
"max_attempts": row.max_attempts,
"version": row.version,
"retry_of_run_id": row.retry_of_run_id,
# Last seq handed out by the event log (0 = no events yet).
"next_event_seq": int(row.next_event_seq or 0),
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
"updated_at": _iso(row.updated_at),
}
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"node_type": row.node_type,
"status": row.status,
"attempt": row.attempt,
"iteration": row.iteration,
"is_current": bool(row.is_current),
"input": _load_text_json(row.input_snapshot),
"output": _load_text_json(row.output_snapshot),
"error": row.error_json,
"parent_node_run_id": row.parent_node_run_id,
"duration_ms": row.duration_ms,
"started_at": _iso(row.started_at),
"finished_at": _iso(row.finished_at),
"created_at": _iso(row.created_at),
}
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
return {
"id": row.id,
"run_id": row.run_id,
"node_id": row.node_id,
"name": row.name,
"mime_type": row.mime_type,
"path": row.path,
"size_bytes": row.size_bytes,
"preview": row.preview,
"created_at": _iso(row.created_at),
}
class SqlWorkflowRunStore(WorkflowRunStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
# ── runs ────────────────────────────────────────────────────────────
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
key = data.get("idempotency_key")
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
row = WorkflowRunRow(
id=str(data.get("id") or uuid.uuid4()),
workflow_id=str(data["workflow_id"]),
workflow_version_id=str(data["workflow_version_id"]),
status=str(data.get("status") or "queued"),
owner_id=str(data["owner_id"]),
idempotency_key=str(key) if key else None,
input_json=data.get("input") or {},
context_json=data.get("context") or {},
max_attempts=int(data.get("max_attempts") or 3),
retry_of_run_id=data.get("retry_of_run_id"),
)
async with self._sf() as session:
session.add(row)
try:
await session.flush()
except IntegrityError:
await session.rollback()
if key:
existing = await self._get_by_idempotency(str(key))
if existing is not None:
return existing, False
raise
out = run_to_dict(row)
await session.commit()
return out, True
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
return run_to_dict(row) if row else None
async def get_run(self, run_id: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def list_runs(
self,
*,
workflow_id: str | None = None,
owner_id: str | None = None,
status: str | None = None,
limit: int = 50,
offset: int = 0,
) -> list[dict[str, Any]]:
stmt = select(WorkflowRunRow)
if workflow_id:
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
if owner_id:
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
if status:
stmt = stmt.where(WorkflowRunRow.status == status)
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [run_to_dict(r) for r in rows]
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
stmt = (
select(WorkflowRunRow.id)
.where(
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
)
)
.order_by(WorkflowRunRow.created_at.asc())
.limit(limit)
)
async with self._sf() as session:
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
or_(
WorkflowRunRow.status == "queued",
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
),
)
.values(
status="running",
lease_owner=lease_owner,
lease_until=lease_until,
heartbeat_at=now,
attempt=WorkflowRunRow.attempt + 1,
version=WorkflowRunRow.version + 1,
started_at=func.coalesce(WorkflowRunRow.started_at, now),
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.lease_owner == lease_owner,
WorkflowRunRow.status == "running",
)
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
return False
await session.commit()
return True
async def update_run(
self,
run_id: str,
*,
lease_owner: str | None = None,
**fields: Any,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
for key, value in fields.items():
if key in _RUN_UPDATABLE:
values[key] = value
status = fields.get("status")
if status in TERMINAL_RUN_STATUSES:
values["lease_owner"] = None
values["lease_until"] = None
values["resume_token"] = None
values["pending_input_json"] = None
values.setdefault("finished_at", now)
where = [WorkflowRunRow.id == run_id]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
)
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
)
if result.rowcount == 0:
await session.rollback()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(*where)
.values(
status="cancelled",
lease_owner=None,
lease_until=None,
resume_token=None,
pending_input_json=None,
finished_at=now,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def set_awaiting_input(
self,
run_id: str,
*,
lease_owner: str | None,
pending_input: dict[str, Any],
resume_token: str,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
if lease_owner is not None:
where.append(WorkflowRunRow.lease_owner == lease_owner)
values: dict[str, Any] = {
"status": "awaiting_input",
"pending_input_json": pending_input,
"resume_token": resume_token,
"lease_owner": None,
"lease_until": None,
"version": WorkflowRunRow.version + 1,
"updated_at": now,
}
if context is not None:
values["context_json"] = context
async with self._sf() as session:
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
now = datetime.now(UTC)
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
async def resume_with_payload(
self,
run_id: str,
resume_token: str,
payload: dict[str, Any],
*,
context: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
now = datetime.now(UTC)
if context is None:
async with self._sf() as session:
row = await session.get(WorkflowRunRow, run_id)
if row is None:
return None
context = dict(row.context_json or {})
new_context = {**context, "resume_payload": payload}
async with self._sf() as session:
result = await session.execute(
update(WorkflowRunRow)
.where(
WorkflowRunRow.id == run_id,
WorkflowRunRow.status == "awaiting_input",
WorkflowRunRow.resume_token == resume_token,
)
.values(
context_json=new_context,
status="queued",
resume_token=None,
pending_input_json=None,
version=WorkflowRunRow.version + 1,
updated_at=now,
)
)
if result.rowcount == 0:
await session.rollback()
return None
await session.commit()
row = await session.get(WorkflowRunRow, run_id)
return run_to_dict(row) if row else None
# ── node runs ───────────────────────────────────────────────────────
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
run_id = str(data["run_id"])
node_id = str(data["node_id"])
attempt = int(data.get("attempt") or 1)
iteration = int(data.get("iteration") or 0)
async with self._sf() as session:
row = (
await session.execute(
select(WorkflowNodeRunRow).where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id == node_id,
WorkflowNodeRunRow.attempt == attempt,
WorkflowNodeRunRow.iteration == iteration,
)
)
).scalar_one_or_none()
if row is None:
row = WorkflowNodeRunRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=run_id,
node_id=node_id,
node_type=str(data.get("node_type") or "unknown"),
attempt=attempt,
iteration=iteration,
)
session.add(row)
row.status = str(data.get("status") or row.status)
row.node_type = str(data.get("node_type") or row.node_type)
row.is_current = bool(data.get("is_current", True))
if "input" in data:
row.input_snapshot = _dump_text_json(data.get("input"))
if "output" in data:
row.output_snapshot = _dump_text_json(data.get("output"))
if "error" in data:
row.error_json = data.get("error")
if data.get("parent_node_run_id"):
row.parent_node_run_id = str(data["parent_node_run_id"])
if data.get("duration_ms") is not None:
row.duration_ms = int(data["duration_ms"])
if data.get("started_at") is not None:
row.started_at = data["started_at"]
if data.get("finished_at") is not None:
row.finished_at = data["finished_at"]
await session.flush()
out = node_run_to_dict(row)
await session.commit()
return out
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
if current_only:
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [node_run_to_dict(r) for r in rows]
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
if not node_ids:
return 0
async with self._sf() as session:
result = await session.execute(
update(WorkflowNodeRunRow)
.where(
WorkflowNodeRunRow.run_id == run_id,
WorkflowNodeRunRow.node_id.in_(node_ids),
WorkflowNodeRunRow.is_current.is_(True),
)
.values(is_current=False)
)
await session.commit()
return int(result.rowcount or 0)
# ── artifacts ───────────────────────────────────────────────────────
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
row = WorkflowRunArtifactRow(
id=str(data.get("id") or uuid.uuid4()),
run_id=str(data["run_id"]),
node_id=str(data.get("node_id") or ""),
name=str(data.get("name") or "")[:512],
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
path=str(data.get("path") or ""),
size_bytes=data.get("size_bytes"),
preview=data.get("preview"),
)
async with self._sf() as session:
session.add(row)
await session.flush()
out = artifact_to_dict(row)
await session.commit()
return out
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [artifact_to_dict(r) for r in rows]
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
stmt = select(WorkflowRunArtifactRow).where(
WorkflowRunArtifactRow.id == artifact_id,
WorkflowRunArtifactRow.run_id == run_id,
)
async with self._sf() as session:
row = (await session.execute(stmt)).scalar_one_or_none()
return artifact_to_dict(row) if row is not None else None
# ── retention ───────────────────────────────────────────────────────
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
run_ids = (
(
await session.execute(
select(WorkflowRunRow.id)
.where(
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
WorkflowRunRow.finished_at.is_not(None),
WorkflowRunRow.finished_at < cutoff,
)
.limit(_PURGE_BATCH)
)
)
.scalars()
.all()
)
if not run_ids:
return []
ids = list(run_ids)
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
await session.commit()
return ids