初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+38
@@ -0,0 +1,38 @@
|
||||
"""Workflow run / node-run / artifact persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunConflictError,
|
||||
WorkflowRunNotFoundError,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.memory import MemoryWorkflowRunStore
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.sql import SqlWorkflowRunStore
|
||||
|
||||
__all__ = [
|
||||
"CANCELLABLE_RUN_STATUSES",
|
||||
"MemoryWorkflowRunStore",
|
||||
"SqlWorkflowRunStore",
|
||||
"TERMINAL_RUN_STATUSES",
|
||||
"WorkflowNodeRunRow",
|
||||
"WorkflowRunArtifactRow",
|
||||
"WorkflowRunConflictError",
|
||||
"WorkflowRunNotFoundError",
|
||||
"WorkflowRunRow",
|
||||
"WorkflowRunStore",
|
||||
"make_workflow_run_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_run_store(session_factory):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowRunStore()
|
||||
return SqlWorkflowRunStore(session_factory)
|
||||
+161
@@ -0,0 +1,161 @@
|
||||
"""Abstract workflow run / node-run / artifact store."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
TERMINAL_RUN_STATUSES = ("completed", "failed", "cancelled")
|
||||
CANCELLABLE_RUN_STATUSES = ("queued", "running", "awaiting_input")
|
||||
|
||||
|
||||
class WorkflowRunNotFoundError(Exception):
|
||||
def __init__(self, run_id: str) -> None:
|
||||
super().__init__(f"workflow run not found: {run_id}")
|
||||
self.run_id = run_id
|
||||
|
||||
|
||||
class WorkflowRunConflictError(Exception):
|
||||
"""Raised when a state transition is not legal for the current status."""
|
||||
|
||||
def __init__(self, run_id: str, status: str) -> None:
|
||||
super().__init__(f"workflow run {run_id} not in a valid state (status={status})")
|
||||
self.run_id = run_id
|
||||
self.status = status
|
||||
|
||||
|
||||
class WorkflowRunStore(ABC):
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
"""Create a run. Returns ``(run, created)``; an existing
|
||||
``idempotency_key`` yields the stored run with ``created=False``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
"""Run ids that are ``queued`` or ``running`` with an expired lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
"""CAS update. When ``lease_owner`` is given the row must still be leased
|
||||
by that owner. Terminal statuses clear the lease."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pause atomically: status + token + pending descriptor (+ optional run
|
||||
context) in one conditional update, so a concurrent resume can never be
|
||||
overwritten by a follow-up context write."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
"""Atomically consume a one-shot resume token: ``awaiting_input`` →
|
||||
``queued``. Returns None when the token is wrong/already used."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Atomic human-input resume in a single conditional update.
|
||||
|
||||
Validates the token, stores ``payload`` as ``context.resume_payload``
|
||||
(replacing the whole run context with ``context`` when given), clears the
|
||||
token/pending descriptor and flips the run to ``queued`` — all under
|
||||
``WHERE status = 'awaiting_input' AND resume_token = :token``. Returns
|
||||
the resumed run on success, ``None`` when the token is wrong, already
|
||||
used, or the run is no longer awaiting input (caller maps that to 409).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Insert-or-update a node run keyed by (run_id, node_id, attempt, iteration)."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
"""Mark node runs stale (used when a loop re-enters its body)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
"""Fetch one artifact row scoped to its run (authorization key)."""
|
||||
raise NotImplementedError
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
@abstractmethod
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
"""Delete finished runs older than the cutoff, with their node runs and
|
||||
artifact rows. Only terminal runs are eligible. Returns deleted run ids
|
||||
(bounded per call so a sweeper can loop)."""
|
||||
raise NotImplementedError
|
||||
+388
@@ -0,0 +1,388 @@
|
||||
"""In-memory workflow run store (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
class MemoryWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self) -> None:
|
||||
self._runs: dict[str, dict[str, Any]] = {}
|
||||
self._node_runs: dict[str, list[dict[str, Any]]] = {}
|
||||
self._artifacts: dict[str, list[dict[str, Any]]] = {}
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
async with self._lock:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
for run in self._runs.values():
|
||||
if run.get("idempotency_key") == str(key):
|
||||
return copy.deepcopy(run), False
|
||||
now = datetime.now(UTC)
|
||||
run_id = str(data.get("id") or uuid.uuid4())
|
||||
run = {
|
||||
"id": run_id,
|
||||
"workflow_id": str(data["workflow_id"]),
|
||||
"workflow_version_id": str(data["workflow_version_id"]),
|
||||
"status": str(data.get("status") or "queued"),
|
||||
"owner_id": str(data["owner_id"]),
|
||||
"idempotency_key": str(key) if key else None,
|
||||
"input": data.get("input") or {},
|
||||
"output": None,
|
||||
"error": None,
|
||||
"context": data.get("context") or {},
|
||||
"pending_input": None,
|
||||
"resume_token": None,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"heartbeat_at": None,
|
||||
"attempt": 0,
|
||||
"max_attempts": int(data.get("max_attempts") or 3),
|
||||
"version": 0,
|
||||
"retry_of_run_id": data.get("retry_of_run_id"),
|
||||
"next_event_seq": 0,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(now),
|
||||
"updated_at": _iso(now),
|
||||
"_created": now,
|
||||
}
|
||||
self._runs[run_id] = run
|
||||
return copy.deepcopy(run), True
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
return copy.deepcopy(run) if run else None
|
||||
|
||||
async def note_event_seq(self, run_id: str, seq: int) -> None:
|
||||
"""Mirror the SQL counter semantics for no-DB mode: the memory event
|
||||
store reports each handed-out seq so run views can expose lastSeq."""
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is not None and int(seq) > int(run.get("next_event_seq") or 0):
|
||||
run["next_event_seq"] = int(seq)
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
rows = sorted(self._runs.values(), key=lambda r: r["_created"], reverse=True)
|
||||
if workflow_id:
|
||||
rows = [r for r in rows if r["workflow_id"] == workflow_id]
|
||||
if owner_id:
|
||||
rows = [r for r in rows if r["owner_id"] == owner_id]
|
||||
if status:
|
||||
rows = [r for r in rows if r["status"] == status]
|
||||
return [copy.deepcopy(r) for r in rows[offset : offset + limit]]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
out: list[str] = []
|
||||
for run in sorted(self._runs.values(), key=lambda r: r["_created"]):
|
||||
if run["status"] == "queued":
|
||||
out.append(run["id"])
|
||||
elif run["status"] == "running" and run["lease_until"]:
|
||||
if datetime.fromisoformat(run["lease_until"]) < now:
|
||||
out.append(run["id"])
|
||||
if len(out) >= limit:
|
||||
break
|
||||
return out
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
claimable = run["status"] == "queued" or (run["status"] == "running" and run["lease_until"] and datetime.fromisoformat(run["lease_until"]) < now)
|
||||
if not claimable:
|
||||
return None
|
||||
run.update(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=_iso(lease_until),
|
||||
heartbeat_at=_iso(now),
|
||||
attempt=run["attempt"] + 1,
|
||||
version=run["version"] + 1,
|
||||
started_at=run["started_at"] or _iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["lease_owner"] != lease_owner or run["status"] != "running":
|
||||
return False
|
||||
run["lease_until"] = _iso(lease_until)
|
||||
run["heartbeat_at"] = _iso(datetime.now(UTC))
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
mapping = {
|
||||
"output_json": "output",
|
||||
"error_json": "error",
|
||||
"context_json": "context",
|
||||
"pending_input_json": "pending_input",
|
||||
}
|
||||
for key, value in fields.items():
|
||||
run[mapping.get(key, key)] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
run["lease_owner"] = None
|
||||
run["lease_until"] = None
|
||||
run["resume_token"] = None
|
||||
run["pending_input"] = None
|
||||
run["finished_at"] = run.get("finished_at") or _iso(now)
|
||||
for key in ("started_at", "finished_at"):
|
||||
if isinstance(run.get(key), datetime):
|
||||
run[key] = _iso(run[key])
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(now)
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None:
|
||||
return None
|
||||
if run["status"] in ("cancel_requested", "cancelled"):
|
||||
return copy.deepcopy(run)
|
||||
if run["status"] not in CANCELLABLE_RUN_STATUSES:
|
||||
return None
|
||||
run["status"] = "cancel_requested"
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "cancel_requested":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
finished_at=_iso(now),
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "running":
|
||||
return None
|
||||
if lease_owner is not None and run["lease_owner"] != lease_owner:
|
||||
return None
|
||||
now = datetime.now(UTC)
|
||||
run.update(
|
||||
status="awaiting_input",
|
||||
pending_input=copy.deepcopy(pending_input),
|
||||
resume_token=resume_token,
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
updated_at=_iso(now),
|
||||
)
|
||||
if context is not None:
|
||||
run["context"] = copy.deepcopy(context)
|
||||
run["version"] += 1
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
run.update(status="queued", resume_token=None, pending_input=None)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
async with self._lock:
|
||||
run = self._runs.get(run_id)
|
||||
if run is None or run["status"] != "awaiting_input":
|
||||
return None
|
||||
if not run.get("resume_token") or run["resume_token"] != resume_token:
|
||||
return None
|
||||
base_context = context if context is not None else run.get("context") or {}
|
||||
run.update(
|
||||
context={**base_context, "resume_payload": copy.deepcopy(payload)},
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input=None,
|
||||
)
|
||||
run["version"] += 1
|
||||
run["updated_at"] = _iso(datetime.now(UTC))
|
||||
return copy.deepcopy(run)
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
rows = self._node_runs.setdefault(run_id, [])
|
||||
existing = next(
|
||||
(r for r in rows if r["node_id"] == node_id and r["attempt"] == attempt and r["iteration"] == iteration),
|
||||
None,
|
||||
)
|
||||
if existing is None:
|
||||
existing = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": run_id,
|
||||
"node_id": node_id,
|
||||
"node_type": str(data.get("node_type") or "unknown"),
|
||||
"status": "queued",
|
||||
"attempt": attempt,
|
||||
"iteration": iteration,
|
||||
"is_current": True,
|
||||
"input": None,
|
||||
"output": None,
|
||||
"error": None,
|
||||
"parent_node_run_id": None,
|
||||
"duration_ms": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
rows.append(existing)
|
||||
for key in (
|
||||
"status",
|
||||
"node_type",
|
||||
"input",
|
||||
"output",
|
||||
"error",
|
||||
"parent_node_run_id",
|
||||
"duration_ms",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
):
|
||||
if key in data and data[key] is not None:
|
||||
existing[key] = _iso(data[key]) if isinstance(data[key], datetime) else data[key]
|
||||
if "is_current" in data:
|
||||
existing["is_current"] = bool(data["is_current"])
|
||||
return copy.deepcopy(existing)
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
rows = self._node_runs.get(run_id, [])
|
||||
if current_only:
|
||||
rows = [r for r in rows if r["is_current"]]
|
||||
return [copy.deepcopy(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
count = 0
|
||||
for row in self._node_runs.get(run_id, []):
|
||||
if row["node_id"] in node_ids and row["is_current"]:
|
||||
row["is_current"] = False
|
||||
count += 1
|
||||
return count
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = {
|
||||
"id": str(data.get("id") or uuid.uuid4()),
|
||||
"run_id": str(data["run_id"]),
|
||||
"node_id": str(data.get("node_id") or ""),
|
||||
"name": str(data.get("name") or ""),
|
||||
"mime_type": str(data.get("mime_type") or "application/octet-stream"),
|
||||
"path": str(data.get("path") or ""),
|
||||
"size_bytes": data.get("size_bytes"),
|
||||
"preview": data.get("preview"),
|
||||
"created_at": _iso(datetime.now(UTC)),
|
||||
}
|
||||
self._artifacts.setdefault(row["run_id"], []).append(row)
|
||||
return copy.deepcopy(row)
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
return [copy.deepcopy(r) for r in self._artifacts.get(run_id, [])]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
for row in self._artifacts.get(run_id, []):
|
||||
if row["id"] == artifact_id:
|
||||
return copy.deepcopy(row)
|
||||
return None
|
||||
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted: list[str] = []
|
||||
for run_id, run in list(self._runs.items()):
|
||||
if run["status"] not in TERMINAL_RUN_STATUSES:
|
||||
continue
|
||||
finished = _parse_dt(run.get("finished_at"))
|
||||
if finished is None or finished >= cutoff:
|
||||
continue
|
||||
deleted.append(run_id)
|
||||
for run_id in deleted:
|
||||
self._runs.pop(run_id, None)
|
||||
self._node_runs.pop(run_id, None)
|
||||
self._artifacts.pop(run_id, None)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
"""ORM models for workflow runs and per-node runs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
|
||||
from sqlalchemy import text as sa_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableJSON, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunRow(Base):
|
||||
"""One workflow execution bound to an immutable published version."""
|
||||
|
||||
__tablename__ = "workflow_runs"
|
||||
__table_args__ = (UniqueConstraint("idempotency_key", name="uq_workflow_runs_idempotency"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_version_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
# queued | running | awaiting_input | completed | failed | cancel_requested | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
owner_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
idempotency_key: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
input_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
output_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
# Non-sensitive run context (never credentials).
|
||||
context_json: Mapped[dict] = mapped_column(PortableJSON(), default=dict)
|
||||
# Pending human-input descriptor when status == awaiting_input.
|
||||
pending_input_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
resume_token: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
|
||||
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3, server_default=sa_text("3"))
|
||||
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
# Last seq handed out by the event log for this run (0 = none yet). The
|
||||
# event store increments it in the same transaction that inserts the event
|
||||
# row, replacing the MAX(seq)+1 race; "0" backfills existing rows and the
|
||||
# first append self-heals from MAX(seq) on conflict.
|
||||
next_event_seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
retry_of_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
BeijingDateTime(),
|
||||
default=lambda: datetime.now(UTC),
|
||||
onupdate=lambda: datetime.now(UTC),
|
||||
)
|
||||
|
||||
|
||||
class WorkflowNodeRunRow(Base):
|
||||
"""Per-node execution record (also the resume checkpoint for a paused run)."""
|
||||
|
||||
__tablename__ = "workflow_node_runs"
|
||||
__table_args__ = (UniqueConstraint("run_id", "node_id", "attempt", "iteration", name="uq_workflow_node_runs_key"),)
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
node_type: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
# queued | running | completed | failed | skipped | cancelled
|
||||
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
|
||||
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1, server_default=sa_text("1"))
|
||||
iteration: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa_text("0"))
|
||||
is_current: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
output_snapshot: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
error_json: Mapped[dict | None] = mapped_column(PortableJSON(), nullable=True)
|
||||
parent_node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
duration_ms: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
|
||||
|
||||
class WorkflowRunArtifactRow(Base):
|
||||
"""Artifact reference produced by a run (bytes live in the sandbox/thread dir)."""
|
||||
|
||||
__tablename__ = "workflow_run_artifacts"
|
||||
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
node_id: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
name: Mapped[str] = mapped_column(String(512), nullable=False)
|
||||
mime_type: Mapped[str] = mapped_column(String(191), nullable=False, default="application/octet-stream")
|
||||
path: Mapped[str] = mapped_column(Text, nullable=False, default="")
|
||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
preview: Mapped[str | None] = mapped_column(PortableLongText(), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+566
@@ -0,0 +1,566 @@
|
||||
"""SQLAlchemy-backed workflow run store (lease dispatch + resume checkpoints)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, or_, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_runs.base import (
|
||||
CANCELLABLE_RUN_STATUSES,
|
||||
TERMINAL_RUN_STATUSES,
|
||||
WorkflowRunStore,
|
||||
)
|
||||
from deerflow.persistence.workflow_runs.model import (
|
||||
WorkflowNodeRunRow,
|
||||
WorkflowRunArtifactRow,
|
||||
WorkflowRunRow,
|
||||
)
|
||||
|
||||
_RUN_UPDATABLE = {
|
||||
"status",
|
||||
"output_json",
|
||||
"error_json",
|
||||
"context_json",
|
||||
"pending_input_json",
|
||||
"resume_token",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
}
|
||||
|
||||
# Retention deletes in bounded batches so a long-neglected table cannot produce
|
||||
# one enormous transaction.
|
||||
_PURGE_BATCH = 500
|
||||
|
||||
|
||||
def _iso(value: datetime | None) -> str | None:
|
||||
return value.isoformat() if isinstance(value, datetime) else None
|
||||
|
||||
|
||||
def _load_text_json(raw: str | None) -> Any:
|
||||
if not raw:
|
||||
return None
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return None
|
||||
|
||||
|
||||
def _dump_text_json(value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
return json.dumps(value, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def run_to_dict(row: WorkflowRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"workflow_version_id": row.workflow_version_id,
|
||||
"status": row.status,
|
||||
"owner_id": row.owner_id,
|
||||
"idempotency_key": row.idempotency_key,
|
||||
"input": row.input_json or {},
|
||||
"output": row.output_json,
|
||||
"error": row.error_json,
|
||||
"context": row.context_json or {},
|
||||
"pending_input": row.pending_input_json,
|
||||
"resume_token": row.resume_token,
|
||||
"lease_owner": row.lease_owner,
|
||||
"lease_until": _iso(row.lease_until),
|
||||
"heartbeat_at": _iso(row.heartbeat_at),
|
||||
"attempt": row.attempt,
|
||||
"max_attempts": row.max_attempts,
|
||||
"version": row.version,
|
||||
"retry_of_run_id": row.retry_of_run_id,
|
||||
# Last seq handed out by the event log (0 = no events yet).
|
||||
"next_event_seq": int(row.next_event_seq or 0),
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
"updated_at": _iso(row.updated_at),
|
||||
}
|
||||
|
||||
|
||||
def node_run_to_dict(row: WorkflowNodeRunRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"node_type": row.node_type,
|
||||
"status": row.status,
|
||||
"attempt": row.attempt,
|
||||
"iteration": row.iteration,
|
||||
"is_current": bool(row.is_current),
|
||||
"input": _load_text_json(row.input_snapshot),
|
||||
"output": _load_text_json(row.output_snapshot),
|
||||
"error": row.error_json,
|
||||
"parent_node_run_id": row.parent_node_run_id,
|
||||
"duration_ms": row.duration_ms,
|
||||
"started_at": _iso(row.started_at),
|
||||
"finished_at": _iso(row.finished_at),
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
def artifact_to_dict(row: WorkflowRunArtifactRow) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row.id,
|
||||
"run_id": row.run_id,
|
||||
"node_id": row.node_id,
|
||||
"name": row.name,
|
||||
"mime_type": row.mime_type,
|
||||
"path": row.path,
|
||||
"size_bytes": row.size_bytes,
|
||||
"preview": row.preview,
|
||||
"created_at": _iso(row.created_at),
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowRunStore(WorkflowRunStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
# ── runs ────────────────────────────────────────────────────────────
|
||||
async def create_run(self, data: dict[str, Any]) -> tuple[dict[str, Any], bool]:
|
||||
key = data.get("idempotency_key")
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
row = WorkflowRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
workflow_id=str(data["workflow_id"]),
|
||||
workflow_version_id=str(data["workflow_version_id"]),
|
||||
status=str(data.get("status") or "queued"),
|
||||
owner_id=str(data["owner_id"]),
|
||||
idempotency_key=str(key) if key else None,
|
||||
input_json=data.get("input") or {},
|
||||
context_json=data.get("context") or {},
|
||||
max_attempts=int(data.get("max_attempts") or 3),
|
||||
retry_of_run_id=data.get("retry_of_run_id"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
await session.rollback()
|
||||
if key:
|
||||
existing = await self._get_by_idempotency(str(key))
|
||||
if existing is not None:
|
||||
return existing, False
|
||||
raise
|
||||
out = run_to_dict(row)
|
||||
await session.commit()
|
||||
return out, True
|
||||
|
||||
async def _get_by_idempotency(self, key: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(select(WorkflowRunRow).where(WorkflowRunRow.idempotency_key == key))).scalar_one_or_none()
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def get_run(self, run_id: str) -> dict[str, Any] | None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def list_runs(
|
||||
self,
|
||||
*,
|
||||
workflow_id: str | None = None,
|
||||
owner_id: str | None = None,
|
||||
status: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunRow)
|
||||
if workflow_id:
|
||||
stmt = stmt.where(WorkflowRunRow.workflow_id == workflow_id)
|
||||
if owner_id:
|
||||
stmt = stmt.where(WorkflowRunRow.owner_id == owner_id)
|
||||
if status:
|
||||
stmt = stmt.where(WorkflowRunRow.status == status)
|
||||
stmt = stmt.order_by(WorkflowRunRow.created_at.desc()).limit(limit).offset(offset)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [run_to_dict(r) for r in rows]
|
||||
|
||||
async def list_claimable(self, *, now: datetime, limit: int = 10) -> list[str]:
|
||||
stmt = (
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
)
|
||||
)
|
||||
.order_by(WorkflowRunRow.created_at.asc())
|
||||
.limit(limit)
|
||||
)
|
||||
async with self._sf() as session:
|
||||
return [str(v) for v in (await session.execute(stmt)).scalars().all()]
|
||||
|
||||
async def claim_run(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
or_(
|
||||
WorkflowRunRow.status == "queued",
|
||||
(WorkflowRunRow.status == "running") & (WorkflowRunRow.lease_until.is_not(None)) & (WorkflowRunRow.lease_until < now),
|
||||
),
|
||||
)
|
||||
.values(
|
||||
status="running",
|
||||
lease_owner=lease_owner,
|
||||
lease_until=lease_until,
|
||||
heartbeat_at=now,
|
||||
attempt=WorkflowRunRow.attempt + 1,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
started_at=func.coalesce(WorkflowRunRow.started_at, now),
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def renew_lease(self, run_id: str, *, lease_owner: str, lease_until: datetime) -> bool:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.lease_owner == lease_owner,
|
||||
WorkflowRunRow.status == "running",
|
||||
)
|
||||
.values(lease_until=lease_until, heartbeat_at=now, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return False
|
||||
await session.commit()
|
||||
return True
|
||||
|
||||
async def update_run(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None = None,
|
||||
**fields: Any,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
values: dict[str, Any] = {"updated_at": now, "version": WorkflowRunRow.version + 1}
|
||||
for key, value in fields.items():
|
||||
if key in _RUN_UPDATABLE:
|
||||
values[key] = value
|
||||
status = fields.get("status")
|
||||
if status in TERMINAL_RUN_STATUSES:
|
||||
values["lease_owner"] = None
|
||||
values["lease_until"] = None
|
||||
values["resume_token"] = None
|
||||
values["pending_input_json"] = None
|
||||
values.setdefault("finished_at", now)
|
||||
where = [WorkflowRunRow.id == run_id]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def request_cancel(self, run_id: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status.in_(list(CANCELLABLE_RUN_STATUSES)),
|
||||
)
|
||||
.values(status="cancel_requested", version=WorkflowRunRow.version + 1, updated_at=now)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row and row.status in ("cancel_requested", "cancelled") else None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def finalize_cancel(self, run_id: str, *, lease_owner: str | None = None) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "cancel_requested"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(*where)
|
||||
.values(
|
||||
status="cancelled",
|
||||
lease_owner=None,
|
||||
lease_until=None,
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
finished_at=now,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def set_awaiting_input(
|
||||
self,
|
||||
run_id: str,
|
||||
*,
|
||||
lease_owner: str | None,
|
||||
pending_input: dict[str, Any],
|
||||
resume_token: str,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
where = [WorkflowRunRow.id == run_id, WorkflowRunRow.status == "running"]
|
||||
if lease_owner is not None:
|
||||
where.append(WorkflowRunRow.lease_owner == lease_owner)
|
||||
values: dict[str, Any] = {
|
||||
"status": "awaiting_input",
|
||||
"pending_input_json": pending_input,
|
||||
"resume_token": resume_token,
|
||||
"lease_owner": None,
|
||||
"lease_until": None,
|
||||
"version": WorkflowRunRow.version + 1,
|
||||
"updated_at": now,
|
||||
}
|
||||
if context is not None:
|
||||
values["context_json"] = context
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(update(WorkflowRunRow).where(*where).values(**values))
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def consume_resume_token(self, run_id: str, resume_token: str) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
async def resume_with_payload(
|
||||
self,
|
||||
run_id: str,
|
||||
resume_token: str,
|
||||
payload: dict[str, Any],
|
||||
*,
|
||||
context: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
now = datetime.now(UTC)
|
||||
if context is None:
|
||||
async with self._sf() as session:
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
if row is None:
|
||||
return None
|
||||
context = dict(row.context_json or {})
|
||||
new_context = {**context, "resume_payload": payload}
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(
|
||||
WorkflowRunRow.id == run_id,
|
||||
WorkflowRunRow.status == "awaiting_input",
|
||||
WorkflowRunRow.resume_token == resume_token,
|
||||
)
|
||||
.values(
|
||||
context_json=new_context,
|
||||
status="queued",
|
||||
resume_token=None,
|
||||
pending_input_json=None,
|
||||
version=WorkflowRunRow.version + 1,
|
||||
updated_at=now,
|
||||
)
|
||||
)
|
||||
if result.rowcount == 0:
|
||||
await session.rollback()
|
||||
return None
|
||||
await session.commit()
|
||||
row = await session.get(WorkflowRunRow, run_id)
|
||||
return run_to_dict(row) if row else None
|
||||
|
||||
# ── node runs ───────────────────────────────────────────────────────
|
||||
async def record_node_run(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
run_id = str(data["run_id"])
|
||||
node_id = str(data["node_id"])
|
||||
attempt = int(data.get("attempt") or 1)
|
||||
iteration = int(data.get("iteration") or 0)
|
||||
async with self._sf() as session:
|
||||
row = (
|
||||
await session.execute(
|
||||
select(WorkflowNodeRunRow).where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id == node_id,
|
||||
WorkflowNodeRunRow.attempt == attempt,
|
||||
WorkflowNodeRunRow.iteration == iteration,
|
||||
)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if row is None:
|
||||
row = WorkflowNodeRunRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=run_id,
|
||||
node_id=node_id,
|
||||
node_type=str(data.get("node_type") or "unknown"),
|
||||
attempt=attempt,
|
||||
iteration=iteration,
|
||||
)
|
||||
session.add(row)
|
||||
row.status = str(data.get("status") or row.status)
|
||||
row.node_type = str(data.get("node_type") or row.node_type)
|
||||
row.is_current = bool(data.get("is_current", True))
|
||||
if "input" in data:
|
||||
row.input_snapshot = _dump_text_json(data.get("input"))
|
||||
if "output" in data:
|
||||
row.output_snapshot = _dump_text_json(data.get("output"))
|
||||
if "error" in data:
|
||||
row.error_json = data.get("error")
|
||||
if data.get("parent_node_run_id"):
|
||||
row.parent_node_run_id = str(data["parent_node_run_id"])
|
||||
if data.get("duration_ms") is not None:
|
||||
row.duration_ms = int(data["duration_ms"])
|
||||
if data.get("started_at") is not None:
|
||||
row.started_at = data["started_at"]
|
||||
if data.get("finished_at") is not None:
|
||||
row.finished_at = data["finished_at"]
|
||||
await session.flush()
|
||||
out = node_run_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_node_runs(self, run_id: str, *, current_only: bool = False) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id == run_id)
|
||||
if current_only:
|
||||
stmt = stmt.where(WorkflowNodeRunRow.is_current.is_(True))
|
||||
stmt = stmt.order_by(WorkflowNodeRunRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [node_run_to_dict(r) for r in rows]
|
||||
|
||||
async def invalidate_node_runs(self, run_id: str, node_ids: list[str]) -> int:
|
||||
if not node_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(
|
||||
update(WorkflowNodeRunRow)
|
||||
.where(
|
||||
WorkflowNodeRunRow.run_id == run_id,
|
||||
WorkflowNodeRunRow.node_id.in_(node_ids),
|
||||
WorkflowNodeRunRow.is_current.is_(True),
|
||||
)
|
||||
.values(is_current=False)
|
||||
)
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
# ── artifacts ───────────────────────────────────────────────────────
|
||||
async def add_artifact(self, data: dict[str, Any]) -> dict[str, Any]:
|
||||
row = WorkflowRunArtifactRow(
|
||||
id=str(data.get("id") or uuid.uuid4()),
|
||||
run_id=str(data["run_id"]),
|
||||
node_id=str(data.get("node_id") or ""),
|
||||
name=str(data.get("name") or "")[:512],
|
||||
mime_type=str(data.get("mime_type") or "application/octet-stream")[:191],
|
||||
path=str(data.get("path") or ""),
|
||||
size_bytes=data.get("size_bytes"),
|
||||
preview=data.get("preview"),
|
||||
)
|
||||
async with self._sf() as session:
|
||||
session.add(row)
|
||||
await session.flush()
|
||||
out = artifact_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
|
||||
async def list_artifacts(self, run_id: str) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id == run_id).order_by(WorkflowRunArtifactRow.created_at.asc())
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [artifact_to_dict(r) for r in rows]
|
||||
|
||||
async def get_artifact(self, run_id: str, artifact_id: str) -> dict[str, Any] | None:
|
||||
stmt = select(WorkflowRunArtifactRow).where(
|
||||
WorkflowRunArtifactRow.id == artifact_id,
|
||||
WorkflowRunArtifactRow.run_id == run_id,
|
||||
)
|
||||
async with self._sf() as session:
|
||||
row = (await session.execute(stmt)).scalar_one_or_none()
|
||||
return artifact_to_dict(row) if row is not None else None
|
||||
|
||||
# ── retention ───────────────────────────────────────────────────────
|
||||
async def purge_terminal_before(self, *, older_than_days: int) -> list[str]:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
run_ids = (
|
||||
(
|
||||
await session.execute(
|
||||
select(WorkflowRunRow.id)
|
||||
.where(
|
||||
WorkflowRunRow.status.in_(TERMINAL_RUN_STATUSES),
|
||||
WorkflowRunRow.finished_at.is_not(None),
|
||||
WorkflowRunRow.finished_at < cutoff,
|
||||
)
|
||||
.limit(_PURGE_BATCH)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not run_ids:
|
||||
return []
|
||||
ids = list(run_ids)
|
||||
await session.execute(delete(WorkflowNodeRunRow).where(WorkflowNodeRunRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunArtifactRow).where(WorkflowRunArtifactRow.run_id.in_(ids)))
|
||||
await session.execute(delete(WorkflowRunRow).where(WorkflowRunRow.id.in_(ids)))
|
||||
await session.commit()
|
||||
return ids
|
||||
Reference in new issue
Block a user