初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+22
@@ -0,0 +1,22 @@
|
||||
"""Workflow run event-log persistence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.memory import MemoryWorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_events.sql import SqlWorkflowEventStore
|
||||
|
||||
__all__ = [
|
||||
"MemoryWorkflowEventStore",
|
||||
"SqlWorkflowEventStore",
|
||||
"WorkflowEventStore",
|
||||
"WorkflowRunEventRow",
|
||||
"make_workflow_event_store",
|
||||
]
|
||||
|
||||
|
||||
def make_workflow_event_store(session_factory, run_store=None):
|
||||
if session_factory is None:
|
||||
return MemoryWorkflowEventStore(run_store=run_store)
|
||||
return SqlWorkflowEventStore(session_factory)
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
"""Abstract workflow run event store (durable, replayable)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
|
||||
class WorkflowEventStore(ABC):
|
||||
@abstractmethod
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, assigning the next per-run ``seq``."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
"""Retention purge by event age; returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
"""Delete every event belonging to the given runs (used when the runs
|
||||
themselves are retained away). Returns deleted row count."""
|
||||
raise NotImplementedError
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""In-memory workflow event log (tests / no-database deployments)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
|
||||
|
||||
class MemoryWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, *, max_per_run: int = 20_000, run_store: Any = None) -> None:
|
||||
self._events: dict[str, list[dict[str, Any]]] = {}
|
||||
self._max_per_run = max_per_run
|
||||
self._run_store = run_store
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
async with self._lock:
|
||||
rows = self._events.setdefault(run_id, [])
|
||||
event = {
|
||||
"seq": len(rows) + 1 if not rows else rows[-1]["seq"] + 1,
|
||||
"run_id": run_id,
|
||||
"workflow_id": workflow_id,
|
||||
"version_id": version_id,
|
||||
"event_type": event_type,
|
||||
"node_id": node_id,
|
||||
"node_run_id": node_run_id,
|
||||
"payload": payload or {},
|
||||
"created_at": datetime.now(UTC).isoformat(),
|
||||
}
|
||||
rows.append(event)
|
||||
if len(rows) > self._max_per_run:
|
||||
del rows[: len(rows) - self._max_per_run]
|
||||
if self._run_store is not None and hasattr(self._run_store, "note_event_seq"):
|
||||
# Keep the run row's lastSeq counter in sync (mirrors the SQL store,
|
||||
# where the counter bump and the insert share one transaction).
|
||||
try:
|
||||
await self._run_store.note_event_seq(run_id, event["seq"])
|
||||
except Exception: # noqa: BLE001 - counter sync is best effort
|
||||
pass
|
||||
return copy.deepcopy(event)
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
rows = [r for r in self._events.get(run_id, []) if r["seq"] > after_seq]
|
||||
return [copy.deepcopy(r) for r in rows[:limit]]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
rows = self._events.get(run_id, [])
|
||||
return int(rows[-1]["seq"]) if rows else 0
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id, rows in list(self._events.items()):
|
||||
kept: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
created = _parse_dt(row.get("created_at"))
|
||||
if created is not None and created < cutoff:
|
||||
deleted += 1
|
||||
else:
|
||||
kept.append(row)
|
||||
if kept:
|
||||
self._events[run_id] = kept
|
||||
else:
|
||||
del self._events[run_id]
|
||||
return deleted
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._lock:
|
||||
deleted = 0
|
||||
for run_id in run_ids:
|
||||
rows = self._events.pop(run_id, None)
|
||||
if rows:
|
||||
deleted += len(rows)
|
||||
return deleted
|
||||
|
||||
|
||||
def _parse_dt(value: Any) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
return value if value.tzinfo else value.replace(tzinfo=UTC)
|
||||
if isinstance(value, str) and value:
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=UTC)
|
||||
return None
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
"""ORM model for the durable workflow run event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy import Integer, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from deerflow.persistence.base import Base
|
||||
from deerflow.persistence.types import BeijingDateTime, PortableLongText
|
||||
|
||||
|
||||
class WorkflowRunEventRow(Base):
|
||||
"""A single persisted event. ``seq`` is monotonic per run and gap-free."""
|
||||
|
||||
__tablename__ = "workflow_run_events"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("run_id", "seq", name="uq_workflow_run_events_run_seq"),
|
||||
sa.Index("ix_workflow_run_events_run_seq", "run_id", "seq"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
|
||||
run_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
|
||||
workflow_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
version_id: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
seq: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
node_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
|
||||
node_run_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
# JSON text rather than PortableJSON: payloads can hold bounded deltas.
|
||||
payload_json: Mapped[str] = mapped_column(PortableLongText(), nullable=False, default="{}")
|
||||
created_at: Mapped[datetime] = mapped_column(BeijingDateTime(), default=lambda: datetime.now(UTC))
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
"""SQLAlchemy-backed workflow event log."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, func, select, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import async_sessionmaker
|
||||
|
||||
from deerflow.persistence.workflow_events.base import WorkflowEventStore
|
||||
from deerflow.persistence.workflow_events.model import WorkflowRunEventRow
|
||||
from deerflow.persistence.workflow_runs.model import WorkflowRunRow
|
||||
|
||||
_APPEND_ATTEMPTS = 5
|
||||
|
||||
|
||||
def _dump_payload(payload: dict[str, Any] | None) -> str:
|
||||
"""Serialize the durable event exactly as it was emitted.
|
||||
|
||||
``node.message`` carries DeerFlow's native LangGraph messages frame. It
|
||||
contains tool results, clarification metadata, and sometimes large search
|
||||
responses. Replacing it with a preview makes an SSE reconnect render a
|
||||
different conversation from the live stream, so the event log must not
|
||||
crop payloads. The column is ``PortableLongText`` specifically for this
|
||||
kind of replayable payload.
|
||||
"""
|
||||
return json.dumps(payload or {}, ensure_ascii=False, separators=(",", ":"), default=str)
|
||||
|
||||
|
||||
def event_to_dict(row: WorkflowRunEventRow) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(row.payload_json or "{}")
|
||||
except json.JSONDecodeError:
|
||||
payload = {}
|
||||
return {
|
||||
"seq": row.seq,
|
||||
"run_id": row.run_id,
|
||||
"workflow_id": row.workflow_id,
|
||||
"version_id": row.version_id,
|
||||
"event_type": row.event_type,
|
||||
"node_id": row.node_id,
|
||||
"node_run_id": row.node_run_id,
|
||||
"payload": payload if isinstance(payload, dict) else {},
|
||||
"created_at": row.created_at.isoformat() if isinstance(row.created_at, datetime) else None,
|
||||
}
|
||||
|
||||
|
||||
class SqlWorkflowEventStore(WorkflowEventStore):
|
||||
def __init__(self, session_factory: async_sessionmaker) -> None:
|
||||
self._sf = session_factory
|
||||
|
||||
async def append(
|
||||
self,
|
||||
*,
|
||||
run_id: str,
|
||||
workflow_id: str,
|
||||
version_id: str,
|
||||
event_type: str,
|
||||
payload: dict[str, Any] | None = None,
|
||||
node_id: str | None = None,
|
||||
node_run_id: str | None = None,
|
||||
max_retries: int = _APPEND_ATTEMPTS,
|
||||
) -> dict[str, Any]:
|
||||
"""Persist one event, taking its ``seq`` from the run's own counter.
|
||||
|
||||
``MAX(seq) + 1`` let two appenders draw the same number and burn a
|
||||
unique-constraint retry; here the counter increment and the insert share
|
||||
one transaction, and the row write-lock held by the UPDATE serialises
|
||||
concurrent appenders until commit — so seqs are strictly increasing,
|
||||
unique and gap-free per run, in commit order.
|
||||
"""
|
||||
text = _dump_payload(payload)
|
||||
for _ in range(max_retries):
|
||||
async with self._sf() as session:
|
||||
seq = await self._take_seq(session, run_id)
|
||||
row = WorkflowRunEventRow(
|
||||
run_id=run_id,
|
||||
workflow_id=workflow_id,
|
||||
version_id=version_id,
|
||||
seq=seq,
|
||||
event_type=event_type,
|
||||
node_id=node_id,
|
||||
node_run_id=node_run_id,
|
||||
payload_json=text,
|
||||
)
|
||||
session.add(row)
|
||||
try:
|
||||
await session.flush()
|
||||
except IntegrityError:
|
||||
# Only reachable for legacy rows whose counter was backfilled
|
||||
# to 0 while events already existed. Re-sync the counter to
|
||||
# MAX(seq) in its own committed transaction, then retry.
|
||||
await session.rollback()
|
||||
await self._resync_counter(run_id)
|
||||
continue
|
||||
out = event_to_dict(row)
|
||||
await session.commit()
|
||||
return out
|
||||
raise RuntimeError(f"failed to append workflow event after {max_retries} attempts (run={run_id})")
|
||||
|
||||
async def _take_seq(self, session: Any, run_id: str) -> int:
|
||||
"""Draw the next seq. Falls back to MAX(seq)+1 when the run row is
|
||||
missing (event store used standalone in tests)."""
|
||||
result = await session.execute(
|
||||
update(WorkflowRunRow).where(WorkflowRunRow.id == run_id).values(next_event_seq=WorkflowRunRow.next_event_seq + 1)
|
||||
)
|
||||
if result.rowcount:
|
||||
# Our own uncommitted increment; the row lock guards it until commit.
|
||||
current = (
|
||||
await session.execute(select(WorkflowRunRow.next_event_seq).where(WorkflowRunRow.id == run_id))
|
||||
).scalar()
|
||||
return int(current or 1)
|
||||
current = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar()
|
||||
return int(current or 0) + 1
|
||||
|
||||
async def _resync_counter(self, run_id: str) -> None:
|
||||
"""Point a legacy/stale run counter back at MAX(seq)."""
|
||||
async with self._sf() as session:
|
||||
max_seq = (
|
||||
await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))
|
||||
).scalar() or 0
|
||||
await session.execute(
|
||||
update(WorkflowRunRow)
|
||||
.where(WorkflowRunRow.id == run_id, WorkflowRunRow.next_event_seq < int(max_seq))
|
||||
.values(next_event_seq=int(max_seq))
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
async def list_after(self, run_id: str, *, after_seq: int = 0, limit: int = 500) -> list[dict[str, Any]]:
|
||||
stmt = select(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id == run_id, WorkflowRunEventRow.seq > after_seq).order_by(WorkflowRunEventRow.seq.asc()).limit(limit)
|
||||
async with self._sf() as session:
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [event_to_dict(r) for r in rows]
|
||||
|
||||
async def max_seq(self, run_id: str) -> int:
|
||||
async with self._sf() as session:
|
||||
value = (await session.execute(select(func.max(WorkflowRunEventRow.seq)).where(WorkflowRunEventRow.run_id == run_id))).scalar()
|
||||
return int(value or 0)
|
||||
|
||||
async def purge_before(self, *, older_than_days: int) -> int:
|
||||
cutoff = datetime.now(UTC) - timedelta(days=max(1, older_than_days))
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.created_at < cutoff))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def purge_for_runs(self, run_ids: list[str]) -> int:
|
||||
if not run_ids:
|
||||
return 0
|
||||
async with self._sf() as session:
|
||||
result = await session.execute(delete(WorkflowRunEventRow).where(WorkflowRunEventRow.run_id.in_(list(run_ids))))
|
||||
await session.commit()
|
||||
return int(result.rowcount or 0)
|
||||
Reference in new issue
Block a user