初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,33 @@
"""Deep-research event-log persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
from deerflow.persistence.deep_research_events.sql import DeepResearchEventRepository
def make_deep_research_event_store(session_factory) -> DeepResearchEventRepository | None:
if session_factory is None:
return None
return DeepResearchEventRepository(session_factory)
_default_store: DeepResearchEventRepository | None = None
def register_default_store(store: DeepResearchEventRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchEventRepository | None:
return _default_store
__all__ = [
"DeepResearchEventRepository",
"DeepResearchEventRow",
"get_default_store",
"make_deep_research_event_store",
"register_default_store",
]
@@ -0,0 +1,43 @@
"""ORM model for the deep-research event log.
Unlike ``roundtable_jobs`` (whole-row DB-poll SSE), deep research persists a
genuine **append-only event stream** so the frontend can reconnect with
``?after=<seq>`` and replay missed events (§15.3). ``seq`` is monotonic per
job and enforced unique by a composite constraint.
Events are small: payloads carry summaries + locators, never full material
text or model reasoning (§15.2 / §19.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchEventRow(Base):
__tablename__ = "deep_research_events"
__table_args__ = (
# Monotonic per-job sequence — the SSE ``Last-Event-ID`` / ``?after`` cursor.
UniqueConstraint("job_id", "seq", name="uq_deep_research_events_job_seq"),
# Composite index for "all events of a job in seq order".
sa.Index("ix_deep_research_events_job_seq", "job_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
job_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,143 @@
"""SQLAlchemy repository for the deep-research event log.
The ``append()`` method assigns the next monotonic ``seq`` per job inside the
same transaction as the INSERT (``SELECT MAX(seq) … FOR`` + insert). Because
only **one** worker holds the job lease at a time, contention on seq is
low; the unique ``(job_id, seq)`` constraint catches any collision as a safety net.
"""
from __future__ import annotations
import json
import logging
from datetime import date, datetime
from enum import Enum
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
logger = logging.getLogger(__name__)
def _json_default(value: Any) -> Any:
"""Make event payloads replayable when source metadata has native dates.
Search adapters intentionally preserve ``published_at`` as a ``datetime``
until it reaches the source store. Events carry the same source envelope,
so the append-only log must normalise it instead of silently falling back to
a live-only frame.
"""
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Enum):
return value.value
if hasattr(value, "model_dump"):
return value.model_dump(mode="json")
return str(value)
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False, default=_json_default)
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
class DeepResearchEventRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchEventRow) -> dict[str, Any]:
d = row.to_dict()
if "payload" in d:
d["payload"] = _loads(d["payload"])
# created_at → ISO for SSE.
if d.get("created_at"):
d["created_at"] = d["created_at"].isoformat() if hasattr(d["created_at"], "isoformat") else d["created_at"]
return d
async def append(
self,
*,
session_id: str,
job_id: str,
event_type: str,
phase: str = "initializing",
payload: dict[str, Any] | None = None,
max_retries: int = 3,
) -> dict[str, Any]:
"""Insert the next event, assigning ``seq = max(seq) + 1`` for this job.
Retries on the unique-constraint collision (only happens if two writers
race for the same seq — shouldn't occur under single-lease ownership
but is handled defensively).
"""
for attempt in range(max_retries):
async with self._sf() as session:
current_max = (await session.execute(select(func.max(DeepResearchEventRow.seq)).where(DeepResearchEventRow.job_id == job_id))).scalar() or 0
next_seq = int(current_max) + 1
row = DeepResearchEventRow(
job_id=job_id,
session_id=session_id,
seq=next_seq,
event_type=event_type,
phase=phase,
payload=_dumps(payload),
)
session.add(row)
try:
# Do not issue ``session.refresh(row)`` after commit. On a
# MySQL read/write-splitting deployment that read can land
# on a lagging replica, which cannot see the just-committed
# event and raises ``Could not refresh instance``. Flush
# assigns generated primary-key/default values on the writer
# connection, so the plain dict is complete before commit.
await session.flush()
result = self._row_to_dict(row)
await session.commit()
return result
except IntegrityError:
await session.rollback()
continue # retry with recomputed max
raise RuntimeError(f"deep_research_events.append: seq collision persisted after {max_retries} retries (job={job_id})")
async def list_after(self, job_id: str, *, after: int = 0, limit: int = 200) -> list[dict[str, Any]]:
"""Events with ``seq > after`` in ascending order (SSE replay, §15.3)."""
async with self._sf() as session:
stmt = select(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id).where(DeepResearchEventRow.seq > after).order_by(DeepResearchEventRow.seq.asc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
async def last_seq(self, job_id: str) -> int:
async with self._sf() as session:
mx = (await session.execute(select(func.max(DeepResearchEventRow.seq)).where(DeepResearchEventRow.job_id == job_id))).scalar()
return int(mx or 0)
async def delete_by_job(self, job_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(delete(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id))
await session.commit()
return result.rowcount
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(delete(DeepResearchEventRow).where(DeepResearchEventRow.session_id == session_id))
await session.commit()
return result.rowcount