代码提交

This commit is contained in:
董宏凯 committed 2026-08-30 01:25:20 +08:00
1 parent 4ac7bc21cd
commit 17cf849361
110 files changed
+17007 -518

No files matched your search

@@ -0,0 +1,22 @@
"""Workflow run event-log persistence."""
from __future__ import annotations
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
__all__ = [
"MemoryWorkflowEventStore",
"SqlWorkflowEventStore",
"WorkflowEventStore",
"WorkflowRunEventRow",
"make_workflow_event_store",
]
def make_workflow_event_store(session_factory, run_store=None):
if session_factory is None:
return MemoryWorkflowEventStore(run_store=run_store)
return SqlWorkflowEventStore(session_factory)
@@ -0,0 +1,42 @@
"""Abstract workflow run event store (durable, replayable)."""
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any
class WorkflowEventStore(ABC):
@abstractmethod
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
"""Persist one event, assigning the next per-run ``seq``."""
raise NotImplementedError
@abstractmethod
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
raise NotImplementedError
@abstractmethod
async def max_seq(self, run_id: str) -> int:
raise NotImplementedError
@abstractmethod
async def purge_before(self, *, older_than_days: int) -> int:
"""Retention purge by event age; returns deleted row count."""
raise NotImplementedError
@abstractmethod
async def purge_for_runs(self, run_ids: list[str]) -> int:
"""Delete every event belonging to the given runs (used when the runs
themselves are retained away). Returns deleted row count."""
raise NotImplementedError
@@ -0,0 +1,103 @@
"""In-memory workflow event log (tests / no-database deployments)."""
from __future__ import annotations
import asyncio
import copy
from datetime import UTC, datetime, timedelta
from typing import Any
from deerflow.persistence.workflow_events.base import WorkflowEventStore
class MemoryWorkflowEventStore(WorkflowEventStore):
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
self._events: dict[str, list[dict[str, Any]]] = {}
self._max_per_run = max_per_run
self._run_store = run_store
self._lock = asyncio.Lock()
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
) -> dict[str, Any]:
async with self._lock:
rows = self._events.setdefault(run_id, [])
event = {
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
"run_id": run_id,
"workflow_id": workflow_id,
"version_id": version_id,
"event_type": event_type,
"node_id": node_id,
"node_run_id": node_run_id,
"payload": payload or {},
"created_at": datetime.now(UTC).isoformat(),
}
rows.append(event)
if len(rows) > self._max_per_run:
del rows[: len(rows) - self._max_per_run]
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
# where the counter bump and the insert share one transaction).
try:
await self._run_store.note_event_seq(run_id, event["seq"])
except Exception: # noqa: BLE001 - counter sync is best effort
pass
return copy.deepcopy(event)
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
return [copy.deepcopy(r) for r in rows[:limit]]
async def max_seq(self, run_id: str) -> int:
rows = self._events.get(run_id, [])
return int(rows[-1]["seq"]) if rows else 0
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._lock:
deleted = 0
for run_id, rows in list(self._events.items()):
kept: list[dict[str, Any]] = []
for row in rows:
created = _parse_dt(row.get("created_at"))
if created is not None and created < cutoff:
deleted += 1
else:
kept.append(row)
if kept:
self._events[run_id] = kept
else:
del self._events[run_id]
return deleted
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._lock:
deleted = 0
for run_id in run_ids:
rows = self._events.pop(run_id, None)
if rows:
deleted += len(rows)
return deleted
def _parse_dt(value: Any) -> datetime | None:
if isinstance(value, datetime):
return value if value.tzinfo else value.replace(tzinfo=UTC)
if isinstance(value, str) and value:
try:
parsed = datetime.fromisoformat(value)
except ValueError:
return None
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
return None
@@ -0,0 +1,34 @@
"""ORM model for the durable workflow run event log."""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class WorkflowRunEventRow(Base):
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
__tablename__ = "workflow_run_events"
__table_args__ = (
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
@@ -0,0 +1,157 @@
"""SQLAlchemy-backed workflow event log."""
from __future__ import annotations
import json
from datetime import UTC, datetime, timedelta
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import async_sessionmaker
from deerflow.persistence.workflow_events.base import WorkflowEventStore
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
_MAX_PAYLOAD_CHARS = 64_000
_APPEND_ATTEMPTS = 5
def _dump_payload(payload: dict[str, Any] | None) -> str:
text = json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
if len(text) > _MAX_PAYLOAD_CHARS:
return json.dumps(
{"truncated": True, "preview": text[:_MAX_PAYLOAD_CHARS]},
ensure_ascii=False,
separators=(",", ":"),
)
return text
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
try:
payload = json.loads(row.payload_json or "{}")
except json.JSONDecodeError:
payload = {}
return {
"seq": row.seq,
"run_id": row.run_id,
"workflow_id": row.workflow_id,
"version_id": row.version_id,
"event_type": row.event_type,
"node_id": row.node_id,
"node_run_id": row.node_run_id,
"payload": payload if isinstance(payload, dict) else {},
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
}
class SqlWorkflowEventStore(WorkflowEventStore):
def __init__(self, session_factory: async_sessionmaker) -> None:
self._sf = session_factory
async def append(
self,
*,
run_id: str,
workflow_id: str,
version_id: str,
event_type: str,
payload: dict[str, Any] | None = None,
node_id: str | None = None,
node_run_id: str | None = None,
max_retries: int = _APPEND_ATTEMPTS,
) -> dict[str, Any]:
"""Persist one event, taking its ``seq`` from the run's own counter.
``MAX(seq) + 1`` let two appenders draw the same number and burn a
unique-constraint retry; here the counter increment and the insert share
one transaction, and the row write-lock held by the UPDATE serialises
concurrent appenders until commit — so seqs are strictly increasing,
unique and gap-free per run, in commit order.
"""
text = _dump_payload(payload)
for _ in range(max_retries):
async with self._sf() as session:
seq = await self._take_seq(session, run_id)
row = WorkflowRunEventRow(
run_id=run_id,
workflow_id=workflow_id,
version_id=version_id,
seq=seq,
event_type=event_type,
node_id=node_id,
node_run_id=node_run_id,
payload_json=text,
)
session.add(row)
try:
await session.flush()
except IntegrityError:
# Only reachable for legacy rows whose counter was backfilled
# to 0 while events already existed. Re-sync the counter to
# MAX(seq) in its own committed transaction, then retry.
await session.rollback()
await self._resync_counter(run_id)
continue
out = event_to_dict(row)
await session.commit()
return out
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
async def _take_seq(self, session: Any, run_id: str) -> int:
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
missing (event store used standalone in tests)."""
result = await session.execute(
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
)
if result.rowcount:
# Our own uncommitted increment; the row lock guards it until commit.
current = (
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
).scalar()
return int(current or 1)
current = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar()
return int(current or 0) + 1
async def _resync_counter(self, run_id: str) -> None:
"""Point a legacy/stale run counter back at MAX(seq)."""
async with self._sf() as session:
max_seq = (
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
).scalar() or 0
await session.execute(
update(WorkflowRunRow)
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
.values(next_event_seq=int(max_seq))
)
await session.commit()
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
async with self._sf() as session:
rows = (await session.execute(stmt)).scalars().all()
return [event_to_dict(r) for r in rows]
async def max_seq(self, run_id: str) -> int:
async with self._sf() as session:
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
return int(value or 0)
async def purge_before(self, *, older_than_days: int) -> int:
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
await session.commit()
return int(result.rowcount or 0)
async def purge_for_runs(self, run_ids: list[str]) -> int:
if not run_ids:
return 0
async with self._sf() as session:
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
await session.commit()
return int(result.rowcount or 0)