代码提交

This commit is contained in:
董宏凯 committed 2026-08-15 12:51:58 +08:00
1 parent 500dab29ab
commit 70da7c5649
59 files changed
+12108 -148

No files matched your search

@@ -0,0 +1,33 @@
"""Deep-research event-log persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
from deerflow.persistence.deep_research_events.sql import DeepResearchEventRepository
def make_deep_research_event_store(session_factory) -> DeepResearchEventRepository | None:
if session_factory is None:
return None
return DeepResearchEventRepository(session_factory)
_default_store: DeepResearchEventRepository | None = None
def register_default_store(store: DeepResearchEventRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchEventRepository | None:
return _default_store
__all__ = [
"DeepResearchEventRepository",
"DeepResearchEventRow",
"get_default_store",
"make_deep_research_event_store",
"register_default_store",
]
@@ -0,0 +1,43 @@
"""ORM model for the deep-research event log.
Unlike ``roundtable_jobs`` (whole-row DB-poll SSE), deep research persists a
genuine **append-only event stream** so the frontend can reconnect with
``?after=<seq>`` and replay missed events (§15.3). ``seq`` is monotonic per
job and enforced unique by a composite constraint.
Events are small: payloads carry summaries + locators, never full material
text or model reasoning (§15.2 / §19.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchEventRow(Base):
__tablename__ = "deep_research_events"
__table_args__ = (
# Monotonic per-job sequence — the SSE ``Last-Event-ID`` / ``?after`` cursor.
UniqueConstraint("job_id", "seq", name="uq_deep_research_events_job_seq"),
# Composite index for "all events of a job in seq order".
sa.Index("ix_deep_research_events_job_seq", "job_id", "seq"),
)
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
job_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False)
event_type: Mapped[str] = mapped_column(String(64), nullable=False)
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,143 @@
"""SQLAlchemy repository for the deep-research event log.
The ``append()`` method assigns the next monotonic ``seq`` per job inside the
same transaction as the INSERT (``SELECT MAX(seq) … FOR`` + insert). Because
only **one** worker holds the job lease at a time, contention on seq is
low; the unique ``(job_id, seq)`` constraint catches any collision as a safety net.
"""
from __future__ import annotations
import json
import logging
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
logger = logging.getLogger(__name__)
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
class DeepResearchEventRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchEventRow) -> dict[str, Any]:
d = row.to_dict()
if "payload" in d:
d["payload"] = _loads(d["payload"])
# created_at → ISO for SSE.
if d.get("created_at"):
d["created_at"] = d["created_at"].isoformat() if hasattr(d["created_at"], "isoformat") else d["created_at"]
return d
async def append(
self,
*,
session_id: str,
job_id: str,
event_type: str,
phase: str = "initializing",
payload: dict[str, Any] | None = None,
max_retries: int = 3,
) -> dict[str, Any]:
"""Insert the next event, assigning ``seq = max(seq) + 1`` for this job.
Retries on the unique-constraint collision (only happens if two writers
race for the same seq — shouldn't occur under single-lease ownership
but is handled defensively).
"""
for attempt in range(max_retries):
async with self._sf() as session:
current_max = (
await session.execute(
select(func.max(DeepResearchEventRow.seq)).where(
DeepResearchEventRow.job_id == job_id
)
)
).scalar() or 0
next_seq = int(current_max) + 1
row = DeepResearchEventRow(
job_id=job_id,
session_id=session_id,
seq=next_seq,
event_type=event_type,
phase=phase,
payload=_dumps(payload),
)
session.add(row)
try:
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
except IntegrityError:
await session.rollback()
continue # retry with recomputed max
raise RuntimeError(
f"deep_research_events.append: seq collision persisted after {max_retries} retries (job={job_id})"
)
async def list_after(
self, job_id: str, *, after: int = 0, limit: int = 200
) -> list[dict[str, Any]]:
"""Events with ``seq > after`` in ascending order (SSE replay, §15.3)."""
async with self._sf() as session:
stmt = (
select(DeepResearchEventRow)
.where(DeepResearchEventRow.job_id == job_id)
.where(DeepResearchEventRow.seq > after)
.order_by(DeepResearchEventRow.seq.asc())
.limit(limit)
)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
async def last_seq(self, job_id: str) -> int:
async with self._sf() as session:
mx = (
await session.execute(
select(func.max(DeepResearchEventRow.seq)).where(
DeepResearchEventRow.job_id == job_id
)
)
).scalar()
return int(mx or 0)
async def delete_by_job(self, job_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchEventRow).where(DeepResearchEventRow.job_id == job_id)
)
await session.commit()
return result.rowcount
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchEventRow).where(DeepResearchEventRow.session_id == session_id)
)
await session.commit()
return result.rowcount
@@ -0,0 +1,48 @@
"""Deep-research durable-job persistence.
Mirrors ``roundtable_jobs``: store factory + process-level default-store singleton
so harness-side runner code can write progress without reaching into FastAPI
``request.app.state`` (harness → app imports forbidden).
"""
from __future__ import annotations
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.persistence.deep_research_jobs.sql import (
ACTIVE_STATUSES,
CANCELLABLE_STATUSES,
TERMINAL_STATUSES,
DeepResearchJobRepository,
compute_active_dedupe_key,
)
def make_deep_research_job_store(session_factory) -> DeepResearchJobRepository | None:
if session_factory is None:
return None
return DeepResearchJobRepository(session_factory)
_default_store: DeepResearchJobRepository | None = None
def register_default_store(store: DeepResearchJobRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchJobRepository | None:
return _default_store
__all__ = [
"ACTIVE_STATUSES",
"CANCELLABLE_STATUSES",
"DeepResearchJobRepository",
"DeepResearchJobRow",
"TERMINAL_STATUSES",
"compute_active_dedupe_key",
"get_default_store",
"make_deep_research_job_store",
"register_default_store",
]
@@ -0,0 +1,89 @@
"""ORM model for deep-research durable jobs.
A *job* is one **backend execution** of the research pipeline against a session.
Detailed / deep / multi-agent research can run for minutes, so jobs are
**durable**: persisted to the DB, claimed via atomic lease, heartbeat-renewed,
and recoverable after a crash or server restart (§14).
The design mirrors ``roundtable_jobs`` (Phase 3 lease + dedupe):
- ``active_dedupe_key`` — unique index; occupied while the job is non-terminal,
cleared (NULL) on terminal. INSERT conflict = idempotent "active job exists".
- ``lease_owner`` / ``lease_until`` — atomic conditional claim + heartbeat.
- ``version`` — row CAS counter, bumped on every state-changing UPDATE.
- Two-phase cancel: ``cancel_requested`` flag → executor watcher → ``cancelled``.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchJobRow(Base):
__tablename__ = "deep_research_jobs"
__table_args__ = (
# Same trick as roundtable_jobs: unique key occupied only while active.
UniqueConstraint("active_dedupe_key", name="uq_deep_research_jobs_active_dedupe_key"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# ── idempotency + dedupe ────────────────────────────────────────────────
request_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
request_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
active_dedupe_key: Mapped[str | None] = mapped_column(String(64), nullable=True)
# ── frozen input (rebuilt by any worker after a takeover) ───────────────
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# ── lease / ownership ───────────────────────────────────────────────────
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
# ── status machine (§14.2) ──────────────────────────────────────────────
# queued / running / awaiting_input / completed / failed / cancelled.
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
# Fine-grained phase (§14.3): initializing / planning / collecting / …
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
current_query: Mapped[str | None] = mapped_column(Text, nullable=True)
# ── two-phase cancel ────────────────────────────────────────────────────
cancel_requested: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
cancel_requested_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
# ── interrupt / human-review resume (§11.2) ─────────────────────────────
awaiting_interrupt_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
awaiting_payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# ── error classification (§14.5) ────────────────────────────────────────
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
# ── result locator ──────────────────────────────────────────────────────
result_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
@@ -0,0 +1,506 @@
"""SQLAlchemy repository for deep-research durable jobs.
Mirrors ``roundtable_jobs`` Phase-3 lease/dedupe/CAS logic exactly:
- ``compute_active_dedupe_key(session_id, user_id)`` → sha256 scope; NULL on terminal.
- ``try_create_or_get_active()`` — INSERT-conflict = idempotent active hit.
- ``claim_job()`` — atomic conditional UPDATE (rowcount-based mutual exclusion).
- ``renew_lease()`` — heartbeat (only lease owner, only while ``running``).
- ``update_progress()`` — lease-gated conditional UPDATE (CAS via ``version``).
- ``request_cancel()`` → ``finalize_cancel()`` — two-phase cancel.
- ``queue_for_resume()`` — atomic ``awaiting_input`` → ``queued``.
Recovery story (§14): ``input_snapshot`` lets any worker rebuild execution
after a takeover; expired-lease ``running`` rows are re-claimable via
``list_claimable``.
"""
from __future__ import annotations
import hashlib
import json
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
_UNSET = object()
# ── status constants (§14.2) ────────────────────────────────────────────────
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
ACTIVE_STATUSES = ("queued", "running", "awaiting_input")
CANCELLABLE_STATUSES = ("queued", "running", "awaiting_input")
def compute_active_dedupe_key(session_id: str, user_id: str | None) -> str:
"""``sha256(session_id + user_id)`` — one active job per session+user scope."""
scope = f"session:{session_id}:{user_id or ''}"
return hashlib.sha256(scope.encode("utf-8")).hexdigest()
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class DeepResearchJobRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── row ↔ dict ──────────────────────────────────────────────────────────
@staticmethod
def _row_to_dict(row: DeepResearchJobRow) -> dict[str, Any]:
d = row.to_dict()
for k in ("input_snapshot", "result_snapshot", "awaiting_payload"):
if k in d:
d[k] = _loads(d[k])
return d
# ── create / idempotent active lookup ───────────────────────────────────
async def try_create_or_get_active(
self,
*,
id: str,
session_id: str,
user_id: str | _AutoSentinel = AUTO,
request_id: str | None = None,
request_hash: str | None = None,
input_snapshot: dict[str, Any] | None = None,
status: str = "queued",
phase: str = "initializing",
) -> tuple[dict[str, Any], bool]:
"""Insert a new job, or return the existing active job for this scope.
Returns ``(job_dict, created)``. ``created=False`` = an active job
already exists (idempotent hit), driven by the ``active_dedupe_key``
unique index.
"""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.try_create_or_get_active")
dedupe_key = compute_active_dedupe_key(session_id, resolved)
for _ in range(2):
try:
async with self._sf() as session:
row = DeepResearchJobRow(
id=id,
session_id=session_id,
user_id=resolved,
request_id=request_id,
request_hash=request_hash,
active_dedupe_key=dedupe_key,
input_snapshot=_dumps(input_snapshot),
status=status,
phase=phase,
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row), True
except IntegrityError:
# Unique-index conflict → an active job already exists.
pass
existing = await self._get_by_dedupe_key(dedupe_key)
if existing is not None:
return existing, False
# Winner entered terminal between conflict and readback → retry INSERT.
# Should not happen in practice.
raise RuntimeError("try_create_or_get_active: race condition not resolved after retries")
async def _get_by_dedupe_key(self, dedupe_key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchJobRow).where(
DeepResearchJobRow.active_dedupe_key == dedupe_key
)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── read ────────────────────────────────────────────────────────────────
async def get(
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
if resolved is not None:
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def get_unscoped(self, job_id: str) -> dict[str, Any] | None:
"""Read without user filter (executor/dispatcher use this)."""
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def get_active_for_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get_active_for_session")
dedupe_key = compute_active_dedupe_key(session_id, resolved)
return await self._get_by_dedupe_key(dedupe_key)
async def list_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 20
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.session_id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
stmt = stmt.order_by(DeepResearchJobRow.created_at.desc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
# ── dispatcher: scan + claim ────────────────────────────────────────────
async def list_claimable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""Candidate job ids: queued, or running with an expired lease (zombie)."""
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now),
)
)
.order_by(DeepResearchJobRow.updated_at.asc())
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
async def claim_job(
self, job_id: str, *, lease_owner: str, lease_until: datetime
) -> dict[str, Any] | None:
"""Atomic conditional claim — rowcount==0 means another worker won."""
async with self._sf() as session:
values = {
"status": "running",
"lease_owner": lease_owner,
"lease_until": lease_until,
"attempt": DeepResearchJobRow.attempt + 1,
"version": DeepResearchJobRow.version + 1,
"started_at": datetime.now(UTC),
"updated_at": datetime.now(UTC),
"cancel_requested": False,
}
where = [
DeepResearchJobRow.id == job_id,
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < datetime.now(UTC)),
),
]
result = await session.execute(
update(DeepResearchJobRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── heartbeat ───────────────────────────────────────────────────────────
async def renew_lease(
self, job_id: str, *, lease_owner: str, lease_until: datetime
) -> bool:
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow)
.where(
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.lease_owner == lease_owner,
DeepResearchJobRow.status == "running",
)
.values(lease_until=lease_until, updated_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
async def release_lease(self, job_id: str, *, lease_owner: str | None = None) -> bool:
async with self._sf() as session:
where = [DeepResearchJobRow.id == job_id]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(lease_owner=None, lease_until=None, updated_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
# ── progress update (lease-gated CAS) ───────────────────────────────────
async def update_progress(
self,
job_id: str,
*,
lease_owner: str | None = None,
user_id: str | _AutoSentinel = AUTO,
status: str | object = _UNSET,
phase: str | object = _UNSET,
progress: int | object = _UNSET,
current_query: str | None | object = _UNSET,
error_code: str | None | object = _UNSET,
error_message: str | None | object = _UNSET,
result_snapshot: dict | None | object = _UNSET,
awaiting_interrupt_id: str | None | object = _UNSET,
awaiting_payload: dict | None | object = _UNSET,
) -> dict[str, Any] | None:
"""Lease-gated conditional progress update.
When ``lease_owner`` is given, the write is rejected unless this worker
still owns the lease and the job hasn't been cancelled — preventing a
zombie worker from clobbering a takeover. Returns ``None`` on rejection.
Note: when ``lease_owner`` is set, the lease ownership *is* the
authorization (the executor calls this from a background task with no
user context), so ``user_id`` is left unresolved in that path.
"""
if lease_owner is not None and isinstance(user_id, _AutoSentinel):
resolved = None # lease-gated path — ownership is the gate, not user_id
else:
resolved = resolve_user_id(
user_id, method_name="DeepResearchJobRepository.update_progress"
)
values: dict[str, Any] = {"updated_at": datetime.now(UTC)}
if status is not _UNSET and status is not None:
values["status"] = status
if status in TERMINAL_STATUSES:
# Release the scope key + lease on terminal.
values["active_dedupe_key"] = None
values["lease_owner"] = None
values["lease_until"] = None
values["finished_at"] = datetime.now(UTC)
elif status == "awaiting_input":
# Paused: release the lease so a worker isn't pinned, keep the scope key.
values["lease_owner"] = None
values["lease_until"] = None
if phase is not _UNSET:
values["phase"] = phase
if progress is not _UNSET:
values["progress"] = progress
if current_query is not _UNSET:
values["current_query"] = current_query
if error_code is not _UNSET:
values["error_code"] = error_code
if error_message is not _UNSET:
values["error_message"] = error_message
if result_snapshot is not _UNSET:
values["result_snapshot"] = _dumps(result_snapshot)
if awaiting_interrupt_id is not _UNSET:
values["awaiting_interrupt_id"] = awaiting_interrupt_id
if awaiting_payload is not _UNSET:
values["awaiting_payload"] = _dumps(awaiting_payload)
values["version"] = DeepResearchJobRow.version + 1
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.not_in(["cancelled"]),
]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
if resolved is not None and lease_owner is None:
where.append(DeepResearchJobRow.user_id == resolved)
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── two-phase cancel ────────────────────────────────────────────────────
async def request_cancel(
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
"""Phase 1: atomically flip ``cancel_requested`` + flag for the executor."""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.request_cancel")
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.in_(list(CANCELLABLE_STATUSES)),
]
if resolved is not None:
where.append(DeepResearchJobRow.user_id == resolved)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
cancel_requested=True,
cancel_requested_at=datetime.now(UTC),
version=DeepResearchJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def finalize_cancel(
self, job_id: str, *, lease_owner: str | None = None
) -> dict[str, Any] | None:
"""Phase 2: ``cancel_requested`` → terminal ``cancelled`` (clears dedupe key)."""
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.cancel_requested.is_(True),
]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
status="cancelled",
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
version=DeepResearchJobRow.version + 1,
finished_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── resume after human-review interrupt ─────────────────────────────────
async def queue_for_resume(
self,
job_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
resume_payload: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic ``awaiting_input`` → ``queued`` so the dispatcher re-claims it."""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.queue_for_resume")
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status == "awaiting_input",
]
if resolved is not None:
where.append(DeepResearchJobRow.user_id == resolved)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
status="queued",
awaiting_interrupt_id=None,
awaiting_payload=_dumps(resume_payload),
lease_owner=None,
lease_until=None,
version=DeepResearchJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── cleanup ─────────────────────────────────────────────────────────────
async def list_older_than(self, *, cutoff: datetime, limit: int = 100) -> list[str]:
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(DeepResearchJobRow.status.in_(list(TERMINAL_STATUSES)))
.where(DeepResearchJobRow.updated_at < cutoff)
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""``cancel_requested`` jobs whose lease expired or was never held.
These are orphaned cancellations (the worker that set the flag crashed
before finalizing). The dispatcher reaps them so they don't stick in a
half-cancelled state forever (§14.3).
"""
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(DeepResearchJobRow.cancel_requested.is_(True))
.where(DeepResearchJobRow.status.not_in(list(TERMINAL_STATUSES)))
.where(
or_(
DeepResearchJobRow.lease_owner.is_(None),
(
DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now)
),
)
)
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
@@ -0,0 +1,33 @@
"""Deep-research follow-up message persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
from deerflow.persistence.deep_research_messages.sql import DeepResearchMessageRepository
def make_deep_research_message_store(session_factory) -> DeepResearchMessageRepository | None:
if session_factory is None:
return None
return DeepResearchMessageRepository(session_factory)
_default_store: DeepResearchMessageRepository | None = None
def register_default_store(store: DeepResearchMessageRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchMessageRepository | None:
return _default_store
__all__ = [
"DeepResearchMessageRepository",
"DeepResearchMessageRow",
"get_default_store",
"make_deep_research_message_store",
"register_default_store",
]
@@ -0,0 +1,35 @@
"""ORM model for deep-research follow-up messages.
After a report is complete, the user can ask follow-up questions that are
answered against the session's report + sources only (``allow_new_research=false``
default; §13.5 / §16.6). This table persists that conversation.
"""
from __future__ import annotations
from datetime import UTC, datetime
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchMessageRow(Base):
__tablename__ = "deep_research_messages"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
role: Mapped[str] = mapped_column(String(16), nullable=False) # user / assistant
content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
# Source ids cited in an assistant reply.
citation_source_ids: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Token/cost usage for this message (JSON).
usage_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
seq: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
@@ -0,0 +1,105 @@
"""SQLAlchemy repository for deep-research follow-up messages."""
from __future__ import annotations
import json
import logging
from typing import Any
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
class DeepResearchMessageRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchMessageRow) -> dict[str, Any]:
d = row.to_dict()
if "citation_source_ids" in d:
d["citation_source_ids"] = _loads(d["citation_source_ids"]) or []
if "usage_snapshot" in d:
d["usage_snapshot"] = _loads(d["usage_snapshot"])
return d
async def append(
self,
*,
id: str,
session_id: str,
user_id: str | _AutoSentinel = AUTO,
role: str,
content: str,
citation_source_ids: list[str] | None = None,
usage_snapshot: dict[str, Any] | None = None,
) -> dict[str, Any]:
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.append")
async with self._sf() as session:
current_max = (
await session.execute(
select(func.max(DeepResearchMessageRow.seq)).where(
DeepResearchMessageRow.session_id == session_id
)
)
).scalar() or 0
row = DeepResearchMessageRow(
id=id,
session_id=session_id,
user_id=resolved,
role=role,
content=content,
citation_source_ids=_dumps(citation_source_ids),
usage_snapshot=_dumps(usage_snapshot),
seq=int(current_max) + 1,
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
async def list_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 100
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchMessageRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchMessageRow).where(
DeepResearchMessageRow.session_id == session_id
)
if resolved is not None:
stmt = stmt.where(DeepResearchMessageRow.user_id == resolved)
stmt = stmt.order_by(DeepResearchMessageRow.seq.asc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchMessageRow).where(
DeepResearchMessageRow.session_id == session_id
)
)
await session.commit()
return result.rowcount
@@ -0,0 +1,40 @@
"""Deep-research session persistence.
Exposes a store factory + a process-level default-store singleton, mirroring
``roundtable_jobs``: harness-side graph nodes call ``get_default_store()``;
``app/gateway/deps.py`` registers it at startup.
"""
from __future__ import annotations
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
from deerflow.persistence.deep_research_sessions.sql import DeepResearchSessionRepository
def make_deep_research_session_store(
session_factory,
) -> DeepResearchSessionRepository | None:
if session_factory is None:
return None
return DeepResearchSessionRepository(session_factory)
_default_store: DeepResearchSessionRepository | None = None
def register_default_store(store: DeepResearchSessionRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchSessionRepository | None:
return _default_store
__all__ = [
"DeepResearchSessionRepository",
"DeepResearchSessionRow",
"get_default_store",
"make_deep_research_session_store",
"register_default_store",
]
@@ -0,0 +1,62 @@
"""ORM model for deep-research sessions.
A *session* is one research topic owned by a user: it holds the frozen config
snapshot, the latest report/plan, and points at the hidden runtime thread that
owns the artifact files. Multiple *jobs* (runs) can execute against a session
(re-run with different settings, resume after human review), but at most one
job is active at a time (enforced by the jobs table's ``active_dedupe_key``).
Field design follows ``roundtable_drafts`` / ``ai_writing_sessions``:
big JSON/text columns use ``PortableLongText``; timestamps use ``BeijingDateTime``.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Integer, String
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchSessionRow(Base):
__tablename__ = "deep_research_sessions"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
title: Mapped[str] = mapped_column(String(256), nullable=False, default="")
query: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
# Research mode (basic/detailed/deep/multi_agent/quick) — denormalised for list views.
mode: Mapped[str] = mapped_column(String(32), nullable=False, default="basic", index=True)
# Session status machine: draft / running / awaiting_input / completed / failed / cancelled.
status: Mapped[str] = mapped_column(String(32), nullable=False, default="draft", index=True)
# Frozen config at creation/start time (full DeepResearchConfig JSON).
config_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# The hidden runtime thread that owns the artifacts (metadata.thread_type=deep_research).
runtime_thread_id: Mapped[str | None] = mapped_column(String(64), nullable=True, unique=True)
# Latest research plan (JSON) — updated by the planner node.
plan_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Latest report outputs.
report_markdown: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
report_html: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Denormalised counts for list views (avoid joining sources table).
source_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
# Token/cost usage snapshot (JSON: {model: {prompt, completion, total}}).
usage_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# Current active job id (null when none running).
active_job_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
# User-facing error summary on failure.
error: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,222 @@
"""SQLAlchemy repository for deep-research sessions.
Mirrors the ``roundtable_jobs`` / ``ai_writing_sessions`` repository conventions:
constructed with an ``async_sessionmaker``, methods are ``async``, open
``async with self._sf() as session:`` blocks, and return **plain dicts**.
JSON columns round-trip through ``json.dumps`` / ``json.loads``.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import delete, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
# Only these states have a durable job that consumes a per-user research slot.
# A draft is just an unsent composer record; counting it here makes a few saved
# drafts permanently trigger the start-job 429 guard.
IN_FLIGHT_STATUSES = ("running", "awaiting_input")
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class DeepResearchSessionRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── row ↔ dict ──────────────────────────────────────────────────────────
@staticmethod
def _row_to_dict(row: DeepResearchSessionRow) -> dict[str, Any]:
d = row.to_dict()
# Deserialize JSON columns.
for k in ("config_snapshot", "plan_snapshot", "usage_snapshot"):
if k in d:
d[k] = _loads(d[k])
return d
# ── create ──────────────────────────────────────────────────────────────
async def create(
self,
*,
id: str,
user_id: str | _AutoSentinel = AUTO,
title: str = "",
query: str = "",
mode: str = "basic",
status: str = "draft",
config_snapshot: dict[str, Any] | None = None,
runtime_thread_id: str | None = None,
) -> dict[str, Any]:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.create")
async with self._sf() as session:
row = DeepResearchSessionRow(
id=id,
user_id=resolved,
title=title,
query=query,
mode=mode,
status=status,
config_snapshot=_dumps(config_snapshot),
runtime_thread_id=runtime_thread_id,
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
# ── read ────────────────────────────────────────────────────────────────
async def get(self, session_id: str, *, user_id: str | _AutoSentinel = AUTO) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def list_by_user(
self,
*,
user_id: str | _AutoSentinel = AUTO,
status: str | None = None,
limit: int = 20,
offset: int = 0,
) -> list[dict[str, Any]]:
"""Paginated session list for the history panel (excludes heavy report text)."""
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.list_by_user")
async with self._sf() as session:
stmt = select(DeepResearchSessionRow)
if resolved is not None:
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
if status:
stmt = stmt.where(DeepResearchSessionRow.status == status)
stmt = stmt.order_by(DeepResearchSessionRow.updated_at.desc()).limit(limit).offset(offset)
rows = (await session.execute(stmt)).scalars().all()
# Strip heavy columns for list views.
out: list[dict[str, Any]] = []
for r in rows:
d = self._row_to_dict(r)
d.pop("report_markdown", None)
d.pop("report_html", None)
d.pop("config_snapshot", None)
d.pop("plan_snapshot", None)
d.pop("usage_snapshot", None)
d.pop("error", None)
out.append(d)
return out
async def count_by_user(
self, *, user_id: str | _AutoSentinel = AUTO, status: str | None = None
) -> int:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.count_by_user")
async with self._sf() as session:
stmt = select(func.count(DeepResearchSessionRow.id))
if resolved is not None:
stmt = stmt.where(DeepResearchSessionRow.user_id == resolved)
if status:
stmt = stmt.where(DeepResearchSessionRow.status == status)
return int((await session.execute(stmt)).scalar() or 0)
async def count_active_by_user(self, *, user_id: str | _AutoSentinel = AUTO) -> int:
"""Count sessions with an in-flight job for the per-user concurrency limit.
Draft sessions deliberately do not count: they have no job row and a
user may safely keep any number of unfinished research prompts.
"""
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.count_active_by_user")
async with self._sf() as session:
stmt = (
select(func.count(DeepResearchSessionRow.id))
.where(DeepResearchSessionRow.user_id == resolved)
.where(DeepResearchSessionRow.status.in_(list(IN_FLIGHT_STATUSES)))
)
return int((await session.execute(stmt)).scalar() or 0)
# ── update ──────────────────────────────────────────────────────────────
async def update(
self,
session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
**fields: Any,
) -> dict[str, Any] | None:
"""Generic field update; JSON-valued fields are auto-serialized."""
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.update")
# Serialize known JSON columns.
if "config_snapshot" in fields and fields["config_snapshot"] is not None:
fields["config_snapshot"] = _dumps(fields["config_snapshot"])
if "plan_snapshot" in fields and fields["plan_snapshot"] is not None:
fields["plan_snapshot"] = _dumps(fields["plan_snapshot"])
if "usage_snapshot" in fields and fields["usage_snapshot"] is not None:
fields["usage_snapshot"] = _dumps(fields["usage_snapshot"])
fields["updated_at"] = datetime.now(UTC)
async with self._sf() as session:
where = [DeepResearchSessionRow.id == session_id]
if resolved is not None:
where.append(DeepResearchSessionRow.user_id == resolved)
result = await session.execute(
update(DeepResearchSessionRow).where(*where).values(**fields)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchSessionRow).where(DeepResearchSessionRow.id == session_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── delete ──────────────────────────────────────────────────────────────
async def delete(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> bool:
resolved = resolve_user_id(user_id, method_name="DeepResearchSessionRepository.delete")
async with self._sf() as session:
where = [DeepResearchSessionRow.id == session_id]
if resolved is not None:
where.append(DeepResearchSessionRow.user_id == resolved)
result = await session.execute(delete(DeepResearchSessionRow).where(*where))
await session.commit()
return result.rowcount > 0
async def get_runtime_thread_id(self, session_id: str) -> str | None:
"""Look up the hidden thread id (for artifact cleanup on delete)."""
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchSessionRow.runtime_thread_id).where(
DeepResearchSessionRow.id == session_id
)
)
).scalar_one_or_none()
return row
@@ -0,0 +1,33 @@
"""Deep-research source-material persistence."""
from __future__ import annotations
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.persistence.deep_research_sources.sql import DeepResearchSourceRepository
def make_deep_research_source_store(session_factory) -> DeepResearchSourceRepository | None:
if session_factory is None:
return None
return DeepResearchSourceRepository(session_factory)
_default_store: DeepResearchSourceRepository | None = None
def register_default_store(store: DeepResearchSourceRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchSourceRepository | None:
return _default_store
__all__ = [
"DeepResearchSourceRepository",
"DeepResearchSourceRow",
"get_default_store",
"make_deep_research_source_store",
"register_default_store",
]
@@ -0,0 +1,61 @@
"""ORM model for deep-research source materials.
Stores every :class:`~deerflow.agents.deep_research.types.ResearchMaterial`
collected during a session, with full provenance (title, url, source,
``rec_uuid``, publish time, relevance) so report citations can trace back to
the originating DeerFlow record (§8.2 / §13.4).
Full text uses ``PortableLongText`` (MySQL ``LONGTEXT``) since web pages can be
large; the list endpoint returns summaries only (§16.5).
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Float, String, Text
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchSourceRow(Base):
__tablename__ = "deep_research_sources"
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
job_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# The query that first surfaced this material.
first_seen_query: Mapped[str | None] = mapped_column(Text, nullable=True)
title: Mapped[str] = mapped_column(Text, nullable=False, default="")
url: Mapped[str | None] = mapped_column(Text, nullable=True, index=True)
raw_content: Mapped[str] = mapped_column(PortableLongText, nullable=False, default="")
snippet: Mapped[str | None] = mapped_column(Text, nullable=True)
source: Mapped[str] = mapped_column(String(256), nullable=False, default="")
source_type: Mapped[str] = mapped_column(String(32), nullable=False, default="web")
published_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
relevance_score: Mapped[float | None] = mapped_column(Float, nullable=True)
rec_uuid: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True)
content_hash: Mapped[str] = mapped_column(String(64), nullable=False, default="")
# Whether this source was selected into the final context.
selected: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
selection_reason: Mapped[str | None] = mapped_column(Text, nullable=True)
# Stable citation key assigned during report writing (e.g. "1", "2").
citation_key: Mapped[str | None] = mapped_column(String(16), nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
@@ -0,0 +1,166 @@
"""SQLAlchemy repository for deep-research source materials."""
from __future__ import annotations
import logging
from typing import Any
from sqlalchemy import func, select, update
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
_JSON_COLS = ("metadata",)
class DeepResearchSourceRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
@staticmethod
def _row_to_dict(row: DeepResearchSourceRow, *, include_content: bool = True) -> dict[str, Any]:
d = row.to_dict()
if not include_content:
d.pop("raw_content", None)
d.pop("snippet", None)
return d
# ── upsert (dedupe by rec_uuid / url / content_hash) ────────────────────
async def upsert(self, *, session_id: str, job_id: str | None = None, **fields: Any) -> dict[str, Any]:
"""Insert a source, or return the existing one if rec_uuid/url/hash matches."""
# Try insert.
async with self._sf() as session:
row = DeepResearchSourceRow(session_id=session_id, job_id=job_id, **fields)
session.add(row)
try:
await session.commit()
await session.refresh(row)
return self._row_to_dict(row)
except Exception:
await session.rollback()
# Fallback: find existing by rec_uuid or url.
existing = await self._find_existing(
session_id=session_id,
rec_uuid=fields.get("rec_uuid"),
url=fields.get("url"),
content_hash=fields.get("content_hash", ""),
)
return existing or self._row_to_dict(row)
async def _find_existing(
self, *, session_id: str, rec_uuid: str | None, url: str | None, content_hash: str
) -> dict[str, Any] | None:
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(
DeepResearchSourceRow.session_id == session_id
)
if rec_uuid:
stmt = stmt.where(DeepResearchSourceRow.rec_uuid == rec_uuid)
elif url:
stmt = stmt.where(DeepResearchSourceRow.url == url)
elif content_hash:
stmt = stmt.where(DeepResearchSourceRow.content_hash == content_hash)
else:
return None
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── read ────────────────────────────────────────────────────────────────
async def get(
self,
source_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
include_content: bool = True,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row, include_content=include_content) if row else None
async def list_by_session(
self,
session_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
selected: bool | None = None,
limit: int = 50,
offset: int = 0,
include_content: bool = False,
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
if selected is not None:
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
stmt = stmt.order_by(DeepResearchSourceRow.created_at.asc()).limit(limit).offset(offset)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r, include_content=include_content) for r in rows]
async def count_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, selected: bool | None = None
) -> int:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.count_by_session")
async with self._sf() as session:
stmt = select(func.count(DeepResearchSourceRow.id)).where(
DeepResearchSourceRow.session_id == session_id
)
if resolved is not None:
stmt = stmt.where(DeepResearchSourceRow.user_id == resolved)
if selected is not None:
stmt = stmt.where(DeepResearchSourceRow.selected == selected)
return int((await session.execute(stmt)).scalar() or 0)
# ── update selection ────────────────────────────────────────────────────
async def set_selected(
self,
source_id: str,
*,
selected: bool,
reason: str | None = None,
citation_key: str | None = None,
user_id: str | _AutoSentinel = AUTO,
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchSourceRepository.set_selected")
async with self._sf() as session:
where = [DeepResearchSourceRow.id == source_id]
if resolved is not None:
where.append(DeepResearchSourceRow.user_id == resolved)
values: dict[str, Any] = {"selected": selected}
if reason is not None:
values["selection_reason"] = reason
if citation_key is not None:
values["citation_key"] = citation_key
result = await session.execute(
update(DeepResearchSourceRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchSourceRow).where(DeepResearchSourceRow.id == source_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def delete_by_session(self, session_id: str) -> int:
from sqlalchemy import delete
async with self._sf() as session:
result = await session.execute(
delete(DeepResearchSourceRow).where(DeepResearchSourceRow.session_id == session_id)
)
await session.commit()
return result.rowcount
@@ -0,0 +1,34 @@
"""Placeholder for a locally-stamped revision that never landed in git.
Some existing databases (including local sqlite) have ``alembic_version``
stamped at ``20260620_01``. That file was never committed; ``20260620_02``
(gather_first) originally revised ``20260619_01`` directly, so
``alembic upgrade head`` failed with ``Can't locate revision identified by
'20260620_01'`` and later schema patches (including ``task_buttons.login_params_json``)
never ran.
This no-op restores the missing node so those DBs can walk
``20260620_01 → 20260620_02 → … → head``. Fresh DBs at ``20260619_01`` just
pass through. ``20260620_02`` is still idempotent (``_has_column``), so a DB
that already applied gather_first under the old id is safe to re-run.
Revision ID: 20260620_01
Revises: 20260619_01
Create Date: 2026-06-20
"""
from __future__ import annotations
revision = "20260620_01"
down_revision = "20260619_01"
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -12,7 +12,7 @@ sweep adds it to already-existing tables (sqlite/mysql/postgres). This revision
alembic graph consistent with the rest of the chain columns and documents the change.
Revision ID: 20260620_02
Revises: 20260619_01
Revises: 20260620_01
Create Date: 2026-06-20
"""
@@ -24,7 +24,7 @@ from sqlalchemy import inspect
revision = "20260620_02"
down_revision = "20260619_01"
down_revision = "20260620_01"
branch_labels = None
depends_on = None
@@ -38,6 +38,11 @@ from deerflow.persistence.agents.model import (
from deerflow.persistence.business_mapping.model import BusinessMappingRow
from deerflow.persistence.concurrency.model import ConcurrencySampleRow
from deerflow.persistence.dashboard_sessions.model import DashboardSessionRow
from deerflow.persistence.deep_research_events.model import DeepResearchEventRow
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.persistence.deep_research_messages.model import DeepResearchMessageRow
from deerflow.persistence.deep_research_sessions.model import DeepResearchSessionRow
from deerflow.persistence.deep_research_sources.model import DeepResearchSourceRow
from deerflow.persistence.embed_sessions.model import EmbedSessionRow
from deerflow.persistence.feedback.model import FeedbackRow
from deerflow.persistence.intent_turns.model import IntentTurnRow
@@ -98,6 +103,11 @@ __all__ = [
"BusinessMappingRow",
"ConcurrencySampleRow",
"DashboardSessionRow",
"DeepResearchEventRow",
"DeepResearchJobRow",
"DeepResearchMessageRow",
"DeepResearchSessionRow",
"DeepResearchSourceRow",
"KnowledgeEmbeddingRow",
"KnowledgeEntityRow",
"KnowledgeExtractTemplateRow",
@@ -51,8 +51,10 @@ class TaskButtonRow(Base):
# ``url`` 外链追加的登录参数(与应用管理一致):在拼好的地址后再依次追加
# ``?{key}={value}``,``value`` 由前端运行时从 ``userInfo`` 按 ``source`` 解析
# (``other`` 时取 ``custom_value`` 字面值)。每项 ``{key, source, custom_value}``。
# 旧行 / NULL 读时归一化为 ``[]``。
login_params_json: Mapped[list] = mapped_column(PortableJSON(), default=list)
# 旧行 / NULL 读时归一化为 ``[]``。必须 ``nullable=True``:否则在已存在
# ``task_buttons`` 表的旧库上,启动期 ``_ensure_orm_columns_sync`` 会跳过
# 「NOT NULL 且无 server_default」的列(见 engine.py),该列永远补不上。
login_params_json: Mapped[list | None] = mapped_column(PortableJSON(), default=list, nullable=True)
# ``url`` 跳转是否追加登录态(token 登录的上游 token + 登录用户名),供外链所在的外部系统
# 校验当前登录用户。仅 token 登录(前端存有 token1)时实际拼上;非 token 登录自动跳过。
# ``server_default`` 必须显式声明:否则在已存在 ``task_buttons`` 表的旧库上,