代码提交

This commit is contained in:
董宏凯 committed 2026-08-15 12:51:58 +08:00
1 parent 500dab29ab
commit 70da7c5649
59 files changed
+12108 -148

No files matched your search

@@ -0,0 +1,33 @@
"""Deep-research event-log persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
from deerflow.persistence.deep_research_events.sql import DeepResearchEventRepository
def make_deep_research_event_store(session_factory) -> DeepResearchEventRepository | None:
if session_factory is None:
return None
return DeepResearchEventRepository(session_factory)
_default_store: DeepResearchEventRepository | None = None
def register_default_store(store: DeepResearchEventRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchEventRepository | None:
return _default_store
__all__ = [
"DeepResearchEventRepository",
"DeepResearchEventRow",
"get_default_store",
"make_deep_research_event_store",
"register_default_store",
]
@@ -0,0 +1,43 @@
"""ORM model for the deep-research event log.
Unlike ``roundtable_jobs`` (whole-row DB-poll SSE), deep research persists a
genuine **append-only event stream** so the frontend can reconnect with
``?after=<seq>`` and replay missed events (§15.3). ``seq`` is monotonic per
job and enforced unique by a composite constraint.
Events are small: payloads carry summaries + locators, never full material
text or model reasoning (§15.2 / §19.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchEventRow(Base):
__tablename__ = "deep_research_events"
__table_args__ = (
# Monotonic per-job sequence — the SSE ``Last-Event-ID`` / ``?after`` cursor.
UniqueConstraint("job_id", "seq", name="uq_deep_research_events_job_seq"),
# Composite index for "all events of a job in seq order".
sa.Index("ix_deep_research_events_job_seq", "job_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
job_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,143 @@
"""SQLAlchemy repository for the deep-research event log.
The ``append()`` method assigns the next monotonic ``seq`` per job inside the
same transaction as the INSERT (``SELECT MAX(seq) … FOR`` + insert). Because
only **one** worker holds the job lease at a time, contention on seq is
low; the unique ``(job_id, seq)`` constraint catches any collision as a safety net.
"""
from __future__ import annotations
import json
import logging
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
logger = logging.getLogger(__name__)
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
class DeepResearchEventRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchEventRow) -> dict[str, Any]:
d = row.to_dict()
if "payload" in d:
d["payload"] = _loads(d["payload"])
# created_at → ISO for SSE.
if d.get("created_at"):
d["created_at"] = d["created_at"].isoformat() if hasattr(d["created_at"], "isoformat") else d["created_at"]
return d
async def append(
self,
*,
session_id: str,
job_id: str,
event_type: str,
phase: str = "initializing",
payload: dict[str, Any] | None = None,
max_retries: int = 3,
) -> dict[str, Any]:
"""Insert the next event, assigning ``seq = max(seq) + 1`` for this job.
Retries on the unique-constraint collision (only happens if two writers
race for the same seq — shouldn't occur under single-lease ownership
but is handled defensively).
"""
for attempt in range(max_retries):
async with self._sf() as session:
current_max = (
await session.execute(
select(func.max(DeepResearchEventRow.seq)).where(
DeepResearchEventRow.job_id == job_id
)
)
).scalar() or 0
next_seq = int(current_max) + 1
row = DeepResearchEventRow(
job_id=job_id,
session_id=session_id,
seq=next_seq,
event_type=event_type,
phase=phase,
payload=_dumps(payload),
)
session.add(row)
try:
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
except IntegrityError:
await session.rollback()
continue # retry with recomputed max
raise RuntimeError(
f"deep_research_events.append: seq collision persisted after {max_retries} retries (job={job_id})"
)
async def list_after(
self, job_id: str, *, after: int = 0, limit: int = 200
) -> list[dict[str, Any]]:
"""Events with ``seq > after`` in ascending order (SSE replay, §15.3)."""
async with self._sf() as session:
stmt = (
select(DeepResearchEventRow)
.where(DeepResearchEventRow.job_id == job_id)
.where(DeepResearchEventRow.seq > after)
.order_by(DeepResearchEventRow.seq.asc())
.limit(limit)
)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
async def last_seq(self, job_id: str) -> int:
async with self._sf() as session:
mx = (
await session.execute(
select(func.max(DeepResearchEventRow.seq)).where(
DeepResearchEventRow.job_id == job_id
)
)
).scalar()
return int(mx or 0)
async def delete_by_job(self, job_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id)
)
await session.commit()
return result.rowcount
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchEventRow).where(DeepResearchEventRow.session_id == session_id)
)
await session.commit()
return result.rowcount