代码提交

This commit is contained in:
董宏凯 committed 2026-08-15 12:51:58 +08:00
1 parent 500dab29ab
commit 70da7c5649
59 files changed
+12108 -148

No files matched your search

@@ -0,0 +1,558 @@
"""Deep-research background-job executor.
Mirrors the roundtable executor's proven pattern:
- ``start_job(snapshot, job_id, lease_owner)`` — idempotent ``asyncio.create_task``
+ ``done_callback`` self-cleanup.
- ``_run()`` — sets user context, builds the :class:`AdapterBundle`, spawns a
heartbeat (lease renewal) + a cancel-watcher, then ``await engine.run()``.
- Cooperative cancellation via :class:`CancellationToken` — the runner checks it
at every phase boundary; the watcher sets it when ``cancel_requested`` appears
in the DB (cross-process cancel from another worker / API call).
Key difference from roundtable: deep research persists an **append-only event
log** (not whole-row polling) and persists **sources** incrementally via a
:class:`RecordingEventSink` that wraps :class:`PersistingEventSink`.
The executor writes terminal state (completed / failed / cancelled) to **both**
the job row and the session row so a single SSE stream + session GET carry the
full result without extra joins.
"""
from __future__ import annotations
import asyncio
import logging
import os
import uuid as _uuid
from collections.abc import Awaitable, Callable
from contextlib import suppress
from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING, Any
from deerflow.agents.deep_research.cancellation import (
CancellationToken,
ResearchAwaitingInput,
ResearchCancelled,
)
from deerflow.agents.deep_research.config import DeepResearchConfig
from deerflow.agents.deep_research.engine import DeepResearchEngine
from deerflow.agents.deep_research.types import (
AdapterBundle,
DeepResearchRequest,
DeepResearchResult,
)
from deerflow.runtime.user_context import reset_current_user, set_current_user
if TYPE_CHECKING:
from deerflow.persistence.deep_research_events import DeepResearchEventRepository
from deerflow.persistence.deep_research_jobs import DeepResearchJobRepository
from deerflow.persistence.deep_research_sessions import DeepResearchSessionRepository
from deerflow.persistence.deep_research_sources import DeepResearchSourceRepository
logger = logging.getLogger(__name__)
# ── constants ───────────────────────────────────────────────────────────────
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
_LEASE_TTL_SECONDS = 60
_HEARTBEAT_INTERVAL = 20.0
_CANCEL_POLL_INTERVAL = 5.0
# Phase → progress% mapping for the job timeline.
_PHASE_PROGRESS: dict[str, int] = {
"initializing": 0,
"planning": 10,
"collecting": 30,
"curating": 60,
"compressing": 70,
"deepening": 50,
"writing": 85,
"reviewing": 92,
"illustrating": 94,
"exporting": 95,
"done": 100,
}
class _JobUser:
"""Minimal CurrentUser: satisfies user_context's ``.id`` protocol."""
def __init__(self, user_id: str) -> None:
self.id = user_id
# ── recording event sink (source persistence + progress fan-out) ─────────────
class RecordingEventSink:
"""Wraps :class:`PersistingEventSink`, adding DB side-effects on key events.
- ``source_added`` → upsert into the source store (incremental, live panel).
- ``phase_changed`` → update the job row's ``phase`` + ``progress``.
- ``sources_curated`` → mark ``selected`` on curated source ids.
All side-effects are best-effort (logged, not raised) so a DB hiccup never
kills the research pipeline — the event log (persisted by the delegate) is
the source of truth for the SSE stream; these writes are convenience
projections for the REST endpoints.
"""
def __init__(
self,
delegate,
*,
source_store: DeepResearchSourceRepository | None,
job_store: DeepResearchJobRepository,
job_id: str,
session_id: str,
user_id: str | None,
lease_owner: str | None,
) -> None:
self._delegate = delegate
self._source_store = source_store
self._job_store = job_store
self._job_id = job_id
self._session_id = session_id
self._user_id = user_id
self._lease_owner = lease_owner
async def emit(self, type, *, phase="initializing", payload=None): # noqa: A002
# Delegate first (persist-then-stream ordering).
ev = await self._delegate.emit(type, phase=phase, payload=payload or {})
# Side-effects.
if type == "source_added" and self._source_store is not None:
src = (payload or {}).get("source")
if isinstance(src, dict):
try:
await self._source_store.upsert(
session_id=self._session_id,
job_id=self._job_id,
user_id=self._user_id,
**src,
)
except Exception: # noqa: BLE001
logger.debug("source upsert failed for %s", src.get("id"), exc_info=True)
elif type == "phase_changed":
new_phase = (payload or {}).get("to", phase)
progress = _PHASE_PROGRESS.get(new_phase, 0)
try:
await self._job_store.update_progress(
self._job_id,
lease_owner=self._lease_owner,
phase=new_phase,
progress=progress,
)
except Exception: # noqa: BLE001
logger.debug("progress update failed for job %s", self._job_id, exc_info=True)
elif type == "sources_curated" and self._source_store is not None:
for sid in (payload or {}).get("selectedIds", []):
try:
await self._source_store.set_selected(
sid, selected=True, reason="curated", user_id=self._user_id
)
except Exception: # noqa: BLE001
pass
elif type == "awaiting_input":
payload = payload or {}
try:
await self._job_store.update_progress(
self._job_id,
lease_owner=self._lease_owner,
status="awaiting_input",
phase=phase,
progress=_PHASE_PROGRESS.get(phase, 10),
awaiting_interrupt_id=str(payload.get("interruptId") or ""),
awaiting_payload=payload,
)
except Exception: # noqa: BLE001
logger.debug("failed to pause deep-research job %s", self._job_id, exc_info=True)
return ev
async def publish_live(self, type, *, phase="initializing", payload=None): # noqa: A002
"""Forward non-durable model deltas without adding projection writes."""
await self._delegate.publish_live(type, phase=phase, payload=payload or {})
# ── executor ────────────────────────────────────────────────────────────────
class DeepResearchJobExecutor:
"""Runs deep-research jobs as background asyncio tasks (one per worker)."""
def __init__(
self,
job_store: DeepResearchJobRepository,
session_store: DeepResearchSessionRepository,
event_store: DeepResearchEventRepository,
source_store: DeepResearchSourceRepository | None = None,
live_publisher: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
) -> None:
self._job_store = job_store
self._session_store = session_store
self._event_store = event_store
self._source_store = source_store
self._live_publisher = live_publisher
self._tasks: dict[str, asyncio.Task] = {}
# ── public API ──────────────────────────────────────────────────────────
def start_job(
self,
snapshot: dict[str, Any],
*,
job_id: str,
lease_owner: str | None = None,
) -> bool:
"""Launch a background research task. Idempotent (same job → ignore).
Returns ``True`` if a new task was created.
"""
if job_id in self._tasks and not self._tasks[job_id].done():
return False
task = asyncio.create_task(self._run(snapshot, job_id=job_id, lease_owner=lease_owner))
self._tasks[job_id] = task
task.add_done_callback(lambda t, jid=job_id: self._tasks.pop(jid, None))
return True
def cancel(self, job_id: str) -> bool:
"""Cancel a running task. Returns ``True`` if the task was live."""
task = self._tasks.get(job_id)
if task is None or task.done():
return False
task.cancel()
return True
def is_running(self, job_id: str) -> bool:
task = self._tasks.get(job_id)
return task is not None and not task.done()
async def await_job(self, job_id: str) -> None:
"""Wait for a job to finish (tests only)."""
task = self._tasks.get(job_id)
if task is not None:
with suppress(Exception, asyncio.CancelledError):
await task
# ── the main run ────────────────────────────────────────────────────────
async def _run(self, snapshot: dict[str, Any], *, job_id: str, lease_owner: str | None) -> None:
session_id = str(snapshot.get("session_id") or "")
user_id = str(snapshot.get("user_id") or "")
query = str(snapshot.get("query") or "")
config_dict = snapshot.get("config") or {}
title = str(snapshot.get("title") or "")
token = set_current_user(_JobUser(user_id)) if user_id else None
cancel_token = CancellationToken()
try:
config = DeepResearchConfig.model_validate(config_dict)
except Exception: # noqa: BLE001
logger.exception("job %s: invalid config snapshot, marking failed", job_id)
await self._mark_failed(job_id, session_id, lease_owner, "CONFIG_INVALID", "配置无效")
if token is not None:
reset_current_user(token)
return
main_task = asyncio.current_task()
heartbeat = (
asyncio.create_task(self._heartbeat(job_id, lease_owner, main_task))
if lease_owner
else None
)
watcher = asyncio.create_task(
self._watch_cancellation(job_id, main_task, lease_owner, cancel_token)
)
try:
result = await self._execute(
job_id=job_id,
session_id=session_id,
user_id=user_id,
query=query,
title=title,
config=config,
lease_owner=lease_owner,
cancel_token=cancel_token,
)
await self._mark_completed(job_id, session_id, lease_owner, result)
except ResearchAwaitingInput as pause:
logger.info("deep-research job %s awaiting human plan review", job_id)
await self._mark_awaiting(job_id, session_id, pause)
except ResearchCancelled:
logger.info("deep-research job %s cancelled (cooperative)", job_id)
# Terminal state written by the watcher / cancel API.
except asyncio.CancelledError:
logger.info("deep-research job %s cancelled (task)", job_id)
raise
except Exception as exc: # noqa: BLE001
logger.exception("deep-research job %s crashed", job_id)
await self._mark_failed(job_id, session_id, lease_owner, type(exc).__name__, str(exc))
finally:
watcher.cancel()
with suppress(asyncio.CancelledError, Exception):
await watcher
if heartbeat is not None:
heartbeat.cancel()
with suppress(asyncio.CancelledError, Exception):
await heartbeat
if token is not None:
reset_current_user(token)
# ── adapter assembly + engine call ──────────────────────────────────────
async def _execute(
self,
*,
job_id: str,
session_id: str,
user_id: str,
query: str,
title: str,
config: DeepResearchConfig,
lease_owner: str | None,
cancel_token: CancellationToken,
) -> DeepResearchResult:
from deerflow.agents.deep_research.adapters.artifacts import ThreadOutputsArtifactWriter
from deerflow.agents.deep_research.adapters.context import DeerFlowContextCompressor
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink
from deerflow.agents.deep_research.adapters.image import build_image_provider
from deerflow.agents.deep_research.adapters.llm import DeerFlowCompletionBackend
from deerflow.agents.deep_research.adapters.material_provider import DeerFlowMaterialProvider
# Resolve / create the hidden runtime thread for artifacts.
runtime_thread_id = await self._ensure_runtime_thread(session_id, user_id)
# Build the event sink chain: Persisting → Recording (side-effects).
base_sink = PersistingEventSink(
self._event_store,
session_id=session_id,
job_id=job_id,
on_event=self._live_publisher,
)
recording_sink = RecordingEventSink(
base_sink,
source_store=self._source_store,
job_store=self._job_store,
job_id=job_id,
session_id=session_id,
user_id=user_id or None,
lease_owner=lease_owner,
)
# LLM backend (single chokepoint).
llm_backend = DeerFlowCompletionBackend(config, event_sink=recording_sink, cancellation=cancel_token)
# Material provider.
material_provider = DeerFlowMaterialProvider(
user_id=user_id,
session_id=session_id,
runtime_thread_id=runtime_thread_id,
model_name=config.fast_model or config.smart_model or config.strategic_model,
)
# Context compressor (shares the LLM for the summary path).
context_compressor = DeerFlowContextCompressor(llm=llm_backend)
# Artifact writer.
artifact_writer = ThreadOutputsArtifactWriter(runtime_thread_id, user_id=user_id or None)
bundle = AdapterBundle(
llm=llm_backend,
materials=material_provider,
context=context_compressor,
artifacts=artifact_writer,
events=recording_sink,
images=build_image_provider(),
)
job = await self._job_store.get_unscoped(job_id)
request = DeepResearchRequest(
session_id=session_id,
user_id=user_id,
query=query,
config=config,
resume_payload=(job or {}).get("awaiting_payload") or {},
interrupted_plan=await self._load_interrupted_plan(job_id),
)
engine = DeepResearchEngine()
result = await engine.run(request, adapters=bundle, cancellation=cancel_token)
return result
async def _load_interrupted_plan(self, job_id: str) -> dict[str, Any] | None:
"""Recover the original plan-review payload when a queued job resumes."""
events = await self._event_store.list_after(job_id, after=0, limit=200)
for event in reversed(events):
if event.get("event_type") != "awaiting_input":
continue
plan = (event.get("payload") or {}).get("plan")
if isinstance(plan, dict):
return plan
return None
# ── terminal-state writers ──────────────────────────────────────────────
async def _mark_completed(
self, job_id: str, session_id: str, lease_owner: str | None, result: DeepResearchResult
) -> None:
# Job row → completed.
await self._job_store.update_progress(
job_id,
lease_owner=lease_owner,
status="completed",
phase="done",
progress=100,
result_snapshot={
"report_markdown": result.report_markdown[:200] if result.report_markdown else "",
"source_ids": result.source_ids,
"plan": result.plan,
},
)
# Session row → completed + report.
source_count = len(result.source_ids)
with suppress(Exception): # noqa: BLE001
await self._session_store.update(
session_id,
status="completed",
report_markdown=result.report_markdown,
report_html=result.report_html,
plan_snapshot=result.plan,
# Keep the terminal job id alongside model usage. The
# frontend uses it only to replay the persisted research trace
# after a refresh; it does not affect billing or job state.
usage_snapshot={**(result.usage or {}), "lastJobId": job_id},
source_count=source_count,
active_job_id=None,
)
# Emit a terminal event for SSE consumers.
with suppress(Exception): # noqa: BLE001
await self._emit_terminal(job_id, session_id, "job_completed")
async def _mark_awaiting(
self, job_id: str, session_id: str, pause: ResearchAwaitingInput
) -> None:
"""Project a persisted plan-review interrupt onto the owning session."""
with suppress(Exception): # noqa: BLE001
await self._session_store.update(
session_id,
status="awaiting_input",
plan_snapshot=pause.plan,
active_job_id=job_id,
)
async def _mark_failed(
self, job_id: str, session_id: str, lease_owner: str | None, code: str, message: str
) -> None:
await self._job_store.update_progress(
job_id,
lease_owner=lease_owner,
status="failed",
error_code=code,
error_message=message,
)
with suppress(Exception): # noqa: BLE001
await self._session_store.update(
session_id,
status="failed",
error=f"{code}: {message}",
active_job_id=None,
)
with suppress(Exception): # noqa: BLE001
await self._emit_terminal(job_id, session_id, "job_failed")
async def _emit_terminal(self, job_id: str, session_id: str, event_type: str) -> None:
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink
sink = PersistingEventSink(
self._event_store,
session_id=session_id,
job_id=job_id,
on_event=self._live_publisher,
)
await sink.emit(event_type, phase="done", payload={"jobId": job_id})
# ── heartbeat (lease renewal) ───────────────────────────────────────────
async def _heartbeat(self, job_id: str, lease_owner: str, main_task: asyncio.Task) -> None:
try:
while not main_task.done():
await asyncio.sleep(_HEARTBEAT_INTERVAL)
if main_task.done():
return
try:
ok = await self._job_store.renew_lease(
job_id,
lease_owner=lease_owner,
lease_until=datetime.now(UTC) + timedelta(seconds=_LEASE_TTL_SECONDS),
)
except Exception: # noqa: BLE001
continue
if not ok:
logger.info("deep-research job %s: lease lost, cancelling", job_id)
if not main_task.done():
main_task.cancel()
return
except asyncio.CancelledError:
return
# ── cancel watcher ──────────────────────────────────────────────────────
async def _watch_cancellation(
self,
job_id: str,
main_task: asyncio.Task,
lease_owner: str | None,
cancel_token: CancellationToken,
) -> None:
"""Poll DB for ``cancel_requested``; cooperate + finalize when seen."""
try:
while not main_task.done():
await asyncio.sleep(_CANCEL_POLL_INTERVAL)
if main_task.done():
return
try:
row = await self._job_store.get_unscoped(job_id)
except Exception: # noqa: BLE001
continue
if row is None:
return
if row.get("cancel_requested") or row.get("status") == "cancelled":
logger.info("deep-research job %s: cancel requested, cooperating", job_id)
cancel_token.cancel()
# Finalize the cancel (lease-gated if we own the lease).
if row.get("status") not in ("cancelled", "completed", "failed"):
with suppress(Exception): # noqa: BLE001
await self._job_store.finalize_cancel(job_id, lease_owner=lease_owner)
if not main_task.done():
main_task.cancel()
return
except asyncio.CancelledError:
return
# ── runtime thread ──────────────────────────────────────────────────────
async def _ensure_runtime_thread(self, session_id: str, user_id: str) -> str:
"""Get or create the hidden thread id for artifact storage."""
existing = await self._session_store.get_runtime_thread_id(session_id)
if existing:
return existing
thread_id = f"dr_{_uuid.uuid4().hex[:20]}"
with suppress(Exception): # noqa: BLE001
await self._session_store.update(session_id, runtime_thread_id=thread_id)
return thread_id
__all__ = ["DeepResearchJobExecutor", "WORKER_ID", "_LEASE_TTL_SECONDS"]