代码提交

This commit is contained in:
董宏凯 committed 2026-08-15 12:51:58 +08:00
1 parent 500dab29ab
commit 70da7c5649
59 files changed
+12108 -148

No files matched your search

@@ -0,0 +1,48 @@
"""Deep-research durable-job persistence.
Mirrors ``roundtable_jobs``: store factory + process-level default-store singleton
so harness-side runner code can write progress without reaching into FastAPI
``request.app.state`` (harness → app imports forbidden).
"""
from __future__ import annotations
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.persistence.deep_research_jobs.sql import (
ACTIVE_STATUSES,
CANCELLABLE_STATUSES,
TERMINAL_STATUSES,
DeepResearchJobRepository,
compute_active_dedupe_key,
)
def make_deep_research_job_store(session_factory) -> DeepResearchJobRepository | None:
if session_factory is None:
return None
return DeepResearchJobRepository(session_factory)
_default_store: DeepResearchJobRepository | None = None
def register_default_store(store: DeepResearchJobRepository | None) -> None:
global _default_store
_default_store = store
def get_default_store() -> DeepResearchJobRepository | None:
return _default_store
__all__ = [
"ACTIVE_STATUSES",
"CANCELLABLE_STATUSES",
"DeepResearchJobRepository",
"DeepResearchJobRow",
"TERMINAL_STATUSES",
"compute_active_dedupe_key",
"get_default_store",
"make_deep_research_job_store",
"register_default_store",
]
@@ -0,0 +1,89 @@
"""ORM model for deep-research durable jobs.
A *job* is one **backend execution** of the research pipeline against a session.
Detailed / deep / multi-agent research can run for minutes, so jobs are
**durable**: persisted to the DB, claimed via atomic lease, heartbeat-renewed,
and recoverable after a crash or server restart (§14).
The design mirrors ``roundtable_jobs`` (Phase 3 lease + dedupe):
- ``active_dedupe_key`` — unique index; occupied while the job is non-terminal,
cleared (NULL) on terminal. INSERT conflict = idempotent "active job exists".
- ``lease_owner`` / ``lease_until`` — atomic conditional claim + heartbeat.
- ``version`` — row CAS counter, bumped on every state-changing UPDATE.
- Two-phase cancel: ``cancel_requested`` flag → executor watcher → ``cancelled``.
"""
from __future__ import annotations
from datetime import UTC, datetime
import sqlalchemy as sa
from sqlalchemy import Boolean, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column
from deerflow.persistence.base import Base
from deerflow.persistence.types import BeijingDateTime, PortableLongText
class DeepResearchJobRow(Base):
__tablename__ = "deep_research_jobs"
__table_args__ = (
# Same trick as roundtable_jobs: unique key occupied only while active.
UniqueConstraint("active_dedupe_key", name="uq_deep_research_jobs_active_dedupe_key"),
)
id: Mapped[str] = mapped_column(String(64), primary_key=True)
session_id: Mapped[str] = mapped_column(String(64), nullable=False, index=True)
user_id: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True)
# ── idempotency + dedupe ────────────────────────────────────────────────
request_id: Mapped[str | None] = mapped_column(String(64), nullable=True)
request_hash: Mapped[str | None] = mapped_column(String(64), nullable=True)
active_dedupe_key: Mapped[str | None] = mapped_column(String(64), nullable=True)
# ── frozen input (rebuilt by any worker after a takeover) ───────────────
input_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# ── lease / ownership ───────────────────────────────────────────────────
lease_owner: Mapped[str | None] = mapped_column(String(64), nullable=True)
lease_until: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
version: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
# ── status machine (§14.2) ──────────────────────────────────────────────
# queued / running / awaiting_input / completed / failed / cancelled.
status: Mapped[str] = mapped_column(String(32), nullable=False, default="queued", index=True)
# Fine-grained phase (§14.3): initializing / planning / collecting / …
phase: Mapped[str] = mapped_column(String(32), nullable=False, default="initializing")
progress: Mapped[int] = mapped_column(Integer, nullable=False, default=0, server_default=sa.text("0"))
current_query: Mapped[str | None] = mapped_column(Text, nullable=True)
# ── two-phase cancel ────────────────────────────────────────────────────
cancel_requested: Mapped[bool] = mapped_column(
Boolean, nullable=False, default=False, server_default=sa.false()
)
cancel_requested_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
# ── interrupt / human-review resume (§11.2) ─────────────────────────────
awaiting_interrupt_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
awaiting_payload: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
# ── error classification (§14.5) ────────────────────────────────────────
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
# ── result locator ──────────────────────────────────────────────────────
result_snapshot: Mapped[str | None] = mapped_column(PortableLongText, nullable=True)
created_at: Mapped[datetime] = mapped_column(
BeijingDateTime(), nullable=False, default=lambda: datetime.now(UTC)
)
started_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
updated_at: Mapped[datetime] = mapped_column(
BeijingDateTime(),
nullable=False,
default=lambda: datetime.now(UTC),
onupdate=lambda: datetime.now(UTC),
)
finished_at: Mapped[datetime | None] = mapped_column(BeijingDateTime(), nullable=True)
@@ -0,0 +1,506 @@
"""SQLAlchemy repository for deep-research durable jobs.
Mirrors ``roundtable_jobs`` Phase-3 lease/dedupe/CAS logic exactly:
- ``compute_active_dedupe_key(session_id, user_id)`` → sha256 scope; NULL on terminal.
- ``try_create_or_get_active()`` — INSERT-conflict = idempotent active hit.
- ``claim_job()`` — atomic conditional UPDATE (rowcount-based mutual exclusion).
- ``renew_lease()`` — heartbeat (only lease owner, only while ``running``).
- ``update_progress()`` — lease-gated conditional UPDATE (CAS via ``version``).
- ``request_cancel()`` → ``finalize_cancel()`` — two-phase cancel.
- ``queue_for_resume()`` — atomic ``awaiting_input`` → ``queued``.
Recovery story (§14): ``input_snapshot`` lets any worker rebuild execution
after a takeover; expired-lease ``running`` rows are re-claimable via
``list_claimable``.
"""
from __future__ import annotations
import hashlib
import json
import logging
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import or_, select, update
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from deerflow.persistence.deep_research_jobs.model import DeepResearchJobRow
from deerflow.runtime.user_context import AUTO, _AutoSentinel, resolve_user_id
logger = logging.getLogger(__name__)
_UNSET = object()
# ── status constants (§14.2) ────────────────────────────────────────────────
TERMINAL_STATUSES = ("completed", "failed", "cancelled")
ACTIVE_STATUSES = ("queued", "running", "awaiting_input")
CANCELLABLE_STATUSES = ("queued", "running", "awaiting_input")
def compute_active_dedupe_key(session_id: str, user_id: str | None) -> str:
"""``sha256(session_id + user_id)`` — one active job per session+user scope."""
scope = f"session:{session_id}:{user_id or ''}"
return hashlib.sha256(scope.encode("utf-8")).hexdigest()
def _loads(value: Any) -> Any:
if not isinstance(value, str) or not value:
return None
try:
return json.loads(value)
except Exception:
return None
def _dumps(value: Any) -> str | None:
if value is None:
return None
return json.dumps(value, ensure_ascii=False)
class DeepResearchJobRepository:
def __init__(self, session_factory: async_sessionmaker[AsyncSession]) -> None:
self._sf = session_factory
# ── row ↔ dict ──────────────────────────────────────────────────────────
@staticmethod
def _row_to_dict(row: DeepResearchJobRow) -> dict[str, Any]:
d = row.to_dict()
for k in ("input_snapshot", "result_snapshot", "awaiting_payload"):
if k in d:
d[k] = _loads(d[k])
return d
# ── create / idempotent active lookup ───────────────────────────────────
async def try_create_or_get_active(
self,
*,
id: str,
session_id: str,
user_id: str | _AutoSentinel = AUTO,
request_id: str | None = None,
request_hash: str | None = None,
input_snapshot: dict[str, Any] | None = None,
status: str = "queued",
phase: str = "initializing",
) -> tuple[dict[str, Any], bool]:
"""Insert a new job, or return the existing active job for this scope.
Returns ``(job_dict, created)``. ``created=False`` = an active job
already exists (idempotent hit), driven by the ``active_dedupe_key``
unique index.
"""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.try_create_or_get_active")
dedupe_key = compute_active_dedupe_key(session_id, resolved)
for _ in range(2):
try:
async with self._sf() as session:
row = DeepResearchJobRow(
id=id,
session_id=session_id,
user_id=resolved,
request_id=request_id,
request_hash=request_hash,
active_dedupe_key=dedupe_key,
input_snapshot=_dumps(input_snapshot),
status=status,
phase=phase,
)
session.add(row)
await session.commit()
await session.refresh(row)
return self._row_to_dict(row), True
except IntegrityError:
# Unique-index conflict → an active job already exists.
pass
existing = await self._get_by_dedupe_key(dedupe_key)
if existing is not None:
return existing, False
# Winner entered terminal between conflict and readback → retry INSERT.
# Should not happen in practice.
raise RuntimeError("try_create_or_get_active: race condition not resolved after retries")
async def _get_by_dedupe_key(self, dedupe_key: str) -> dict[str, Any] | None:
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchJobRow).where(
DeepResearchJobRow.active_dedupe_key == dedupe_key
)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── read ────────────────────────────────────────────────────────────────
async def get(
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get")
async with self._sf() as session:
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
if resolved is not None:
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
row = (await session.execute(stmt)).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def get_unscoped(self, job_id: str) -> dict[str, Any] | None:
"""Read without user filter (executor/dispatcher use this)."""
async with self._sf() as session:
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def get_active_for_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.get_active_for_session")
dedupe_key = compute_active_dedupe_key(session_id, resolved)
return await self._get_by_dedupe_key(dedupe_key)
async def list_by_session(
self, session_id: str, *, user_id: str | _AutoSentinel = AUTO, limit: int = 20
) -> list[dict[str, Any]]:
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.list_by_session")
async with self._sf() as session:
stmt = select(DeepResearchJobRow).where(DeepResearchJobRow.session_id == session_id)
if resolved is not None:
stmt = stmt.where(DeepResearchJobRow.user_id == resolved)
stmt = stmt.order_by(DeepResearchJobRow.created_at.desc()).limit(limit)
rows = (await session.execute(stmt)).scalars().all()
return [self._row_to_dict(r) for r in rows]
# ── dispatcher: scan + claim ────────────────────────────────────────────
async def list_claimable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""Candidate job ids: queued, or running with an expired lease (zombie)."""
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now),
)
)
.order_by(DeepResearchJobRow.updated_at.asc())
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
async def claim_job(
self, job_id: str, *, lease_owner: str, lease_until: datetime
) -> dict[str, Any] | None:
"""Atomic conditional claim — rowcount==0 means another worker won."""
async with self._sf() as session:
values = {
"status": "running",
"lease_owner": lease_owner,
"lease_until": lease_until,
"attempt": DeepResearchJobRow.attempt + 1,
"version": DeepResearchJobRow.version + 1,
"started_at": datetime.now(UTC),
"updated_at": datetime.now(UTC),
"cancel_requested": False,
}
where = [
DeepResearchJobRow.id == job_id,
or_(
DeepResearchJobRow.status == "queued",
(DeepResearchJobRow.status == "running")
& DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < datetime.now(UTC)),
),
]
result = await session.execute(
update(DeepResearchJobRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── heartbeat ───────────────────────────────────────────────────────────
async def renew_lease(
self, job_id: str, *, lease_owner: str, lease_until: datetime
) -> bool:
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow)
.where(
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.lease_owner == lease_owner,
DeepResearchJobRow.status == "running",
)
.values(lease_until=lease_until, updated_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
async def release_lease(self, job_id: str, *, lease_owner: str | None = None) -> bool:
async with self._sf() as session:
where = [DeepResearchJobRow.id == job_id]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(lease_owner=None, lease_until=None, updated_at=datetime.now(UTC))
)
await session.commit()
return result.rowcount > 0
# ── progress update (lease-gated CAS) ───────────────────────────────────
async def update_progress(
self,
job_id: str,
*,
lease_owner: str | None = None,
user_id: str | _AutoSentinel = AUTO,
status: str | object = _UNSET,
phase: str | object = _UNSET,
progress: int | object = _UNSET,
current_query: str | None | object = _UNSET,
error_code: str | None | object = _UNSET,
error_message: str | None | object = _UNSET,
result_snapshot: dict | None | object = _UNSET,
awaiting_interrupt_id: str | None | object = _UNSET,
awaiting_payload: dict | None | object = _UNSET,
) -> dict[str, Any] | None:
"""Lease-gated conditional progress update.
When ``lease_owner`` is given, the write is rejected unless this worker
still owns the lease and the job hasn't been cancelled — preventing a
zombie worker from clobbering a takeover. Returns ``None`` on rejection.
Note: when ``lease_owner`` is set, the lease ownership *is* the
authorization (the executor calls this from a background task with no
user context), so ``user_id`` is left unresolved in that path.
"""
if lease_owner is not None and isinstance(user_id, _AutoSentinel):
resolved = None # lease-gated path — ownership is the gate, not user_id
else:
resolved = resolve_user_id(
user_id, method_name="DeepResearchJobRepository.update_progress"
)
values: dict[str, Any] = {"updated_at": datetime.now(UTC)}
if status is not _UNSET and status is not None:
values["status"] = status
if status in TERMINAL_STATUSES:
# Release the scope key + lease on terminal.
values["active_dedupe_key"] = None
values["lease_owner"] = None
values["lease_until"] = None
values["finished_at"] = datetime.now(UTC)
elif status == "awaiting_input":
# Paused: release the lease so a worker isn't pinned, keep the scope key.
values["lease_owner"] = None
values["lease_until"] = None
if phase is not _UNSET:
values["phase"] = phase
if progress is not _UNSET:
values["progress"] = progress
if current_query is not _UNSET:
values["current_query"] = current_query
if error_code is not _UNSET:
values["error_code"] = error_code
if error_message is not _UNSET:
values["error_message"] = error_message
if result_snapshot is not _UNSET:
values["result_snapshot"] = _dumps(result_snapshot)
if awaiting_interrupt_id is not _UNSET:
values["awaiting_interrupt_id"] = awaiting_interrupt_id
if awaiting_payload is not _UNSET:
values["awaiting_payload"] = _dumps(awaiting_payload)
values["version"] = DeepResearchJobRow.version + 1
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.not_in(["cancelled"]),
]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
if resolved is not None and lease_owner is None:
where.append(DeepResearchJobRow.user_id == resolved)
async with self._sf() as session:
result = await session.execute(
update(DeepResearchJobRow).where(*where).values(**values)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── two-phase cancel ────────────────────────────────────────────────────
async def request_cancel(
self, job_id: str, *, user_id: str | _AutoSentinel = AUTO
) -> dict[str, Any] | None:
"""Phase 1: atomically flip ``cancel_requested`` + flag for the executor."""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.request_cancel")
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status.in_(list(CANCELLABLE_STATUSES)),
]
if resolved is not None:
where.append(DeepResearchJobRow.user_id == resolved)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
cancel_requested=True,
cancel_requested_at=datetime.now(UTC),
version=DeepResearchJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
async def finalize_cancel(
self, job_id: str, *, lease_owner: str | None = None
) -> dict[str, Any] | None:
"""Phase 2: ``cancel_requested`` → terminal ``cancelled`` (clears dedupe key)."""
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.cancel_requested.is_(True),
]
if lease_owner is not None:
where.append(DeepResearchJobRow.lease_owner == lease_owner)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
status="cancelled",
active_dedupe_key=None,
lease_owner=None,
lease_until=None,
version=DeepResearchJobRow.version + 1,
finished_at=datetime.now(UTC),
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── resume after human-review interrupt ─────────────────────────────────
async def queue_for_resume(
self,
job_id: str,
*,
user_id: str | _AutoSentinel = AUTO,
resume_payload: dict[str, Any] | None = None,
) -> dict[str, Any] | None:
"""Atomic ``awaiting_input`` → ``queued`` so the dispatcher re-claims it."""
resolved = resolve_user_id(user_id, method_name="DeepResearchJobRepository.queue_for_resume")
async with self._sf() as session:
where = [
DeepResearchJobRow.id == job_id,
DeepResearchJobRow.status == "awaiting_input",
]
if resolved is not None:
where.append(DeepResearchJobRow.user_id == resolved)
result = await session.execute(
update(DeepResearchJobRow)
.where(*where)
.values(
status="queued",
awaiting_interrupt_id=None,
awaiting_payload=_dumps(resume_payload),
lease_owner=None,
lease_until=None,
version=DeepResearchJobRow.version + 1,
updated_at=datetime.now(UTC),
)
)
if result.rowcount == 0:
return None
await session.commit()
row = (
await session.execute(
select(DeepResearchJobRow).where(DeepResearchJobRow.id == job_id)
)
).scalar_one_or_none()
return self._row_to_dict(row) if row else None
# ── cleanup ─────────────────────────────────────────────────────────────
async def list_older_than(self, *, cutoff: datetime, limit: int = 100) -> list[str]:
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(DeepResearchJobRow.status.in_(list(TERMINAL_STATUSES)))
.where(DeepResearchJobRow.updated_at < cutoff)
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]
async def list_cancel_reapable(self, *, now: datetime, limit: int = 20) -> list[str]:
"""``cancel_requested`` jobs whose lease expired or was never held.
These are orphaned cancellations (the worker that set the flag crashed
before finalizing). The dispatcher reaps them so they don't stick in a
half-cancelled state forever (§14.3).
"""
async with self._sf() as session:
stmt = (
select(DeepResearchJobRow.id)
.where(DeepResearchJobRow.cancel_requested.is_(True))
.where(DeepResearchJobRow.status.not_in(list(TERMINAL_STATUSES)))
.where(
or_(
DeepResearchJobRow.lease_owner.is_(None),
(
DeepResearchJobRow.lease_until.is_not(None)
& (DeepResearchJobRow.lease_until < now)
),
)
)
.limit(limit)
)
return [r for r in (await session.execute(stmt)).scalars().all()]