代码提交
This commit is contained in:
1 parent
500dab29ab
commit
70da7c5649
59 files changed
+12108
-148
No files matched your search
@@ -0,0 +1,558 @@
|
||||
"""Deep-research background-job executor.
|
||||
|
||||
Mirrors the roundtable executor's proven pattern:
|
||||
|
||||
- ``start_job(snapshot, job_id, lease_owner)`` — idempotent ``asyncio.create_task``
|
||||
+ ``done_callback`` self-cleanup.
|
||||
- ``_run()`` — sets user context, builds the :class:`AdapterBundle`, spawns a
|
||||
heartbeat (lease renewal) + a cancel-watcher, then ``await engine.run()``.
|
||||
- Cooperative cancellation via :class:`CancellationToken` — the runner checks it
|
||||
at every phase boundary; the watcher sets it when ``cancel_requested`` appears
|
||||
in the DB (cross-process cancel from another worker / API call).
|
||||
|
||||
Key difference from roundtable: deep research persists an **append-only event
|
||||
log** (not whole-row polling) and persists **sources** incrementally via a
|
||||
:class:`RecordingEventSink` that wraps :class:`PersistingEventSink`.
|
||||
|
||||
The executor writes terminal state (completed / failed / cancelled) to **both**
|
||||
the job row and the session row so a single SSE stream + session GET carry the
|
||||
full result without extra joins.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import uuid as _uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.agents.deep_research.cancellation import (
|
||||
CancellationToken,
|
||||
ResearchAwaitingInput,
|
||||
ResearchCancelled,
|
||||
)
|
||||
from deerflow.agents.deep_research.config import DeepResearchConfig
|
||||
from deerflow.agents.deep_research.engine import DeepResearchEngine
|
||||
from deerflow.agents.deep_research.types import (
|
||||
AdapterBundle,
|
||||
DeepResearchRequest,
|
||||
DeepResearchResult,
|
||||
)
|
||||
from deerflow.runtime.user_context import reset_current_user, set_current_user
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.persistence.deep_research_events import DeepResearchEventRepository
|
||||
from deerflow.persistence.deep_research_jobs import DeepResearchJobRepository
|
||||
from deerflow.persistence.deep_research_sessions import DeepResearchSessionRepository
|
||||
from deerflow.persistence.deep_research_sources import DeepResearchSourceRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── constants ───────────────────────────────────────────────────────────────
|
||||
|
||||
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
|
||||
|
||||
_LEASE_TTL_SECONDS = 60
|
||||
_HEARTBEAT_INTERVAL = 20.0
|
||||
_CANCEL_POLL_INTERVAL = 5.0
|
||||
|
||||
# Phase → progress% mapping for the job timeline.
|
||||
_PHASE_PROGRESS: dict[str, int] = {
|
||||
"initializing": 0,
|
||||
"planning": 10,
|
||||
"collecting": 30,
|
||||
"curating": 60,
|
||||
"compressing": 70,
|
||||
"deepening": 50,
|
||||
"writing": 85,
|
||||
"reviewing": 92,
|
||||
"illustrating": 94,
|
||||
"exporting": 95,
|
||||
"done": 100,
|
||||
}
|
||||
|
||||
|
||||
class _JobUser:
|
||||
"""Minimal CurrentUser: satisfies user_context's ``.id`` protocol."""
|
||||
|
||||
def __init__(self, user_id: str) -> None:
|
||||
self.id = user_id
|
||||
|
||||
|
||||
# ── recording event sink (source persistence + progress fan-out) ─────────────
|
||||
|
||||
|
||||
class RecordingEventSink:
|
||||
"""Wraps :class:`PersistingEventSink`, adding DB side-effects on key events.
|
||||
|
||||
- ``source_added`` → upsert into the source store (incremental, live panel).
|
||||
- ``phase_changed`` → update the job row's ``phase`` + ``progress``.
|
||||
- ``sources_curated`` → mark ``selected`` on curated source ids.
|
||||
|
||||
All side-effects are best-effort (logged, not raised) so a DB hiccup never
|
||||
kills the research pipeline — the event log (persisted by the delegate) is
|
||||
the source of truth for the SSE stream; these writes are convenience
|
||||
projections for the REST endpoints.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
delegate,
|
||||
*,
|
||||
source_store: DeepResearchSourceRepository | None,
|
||||
job_store: DeepResearchJobRepository,
|
||||
job_id: str,
|
||||
session_id: str,
|
||||
user_id: str | None,
|
||||
lease_owner: str | None,
|
||||
) -> None:
|
||||
self._delegate = delegate
|
||||
self._source_store = source_store
|
||||
self._job_store = job_store
|
||||
self._job_id = job_id
|
||||
self._session_id = session_id
|
||||
self._user_id = user_id
|
||||
self._lease_owner = lease_owner
|
||||
|
||||
async def emit(self, type, *, phase="initializing", payload=None): # noqa: A002
|
||||
# Delegate first (persist-then-stream ordering).
|
||||
ev = await self._delegate.emit(type, phase=phase, payload=payload or {})
|
||||
|
||||
# Side-effects.
|
||||
if type == "source_added" and self._source_store is not None:
|
||||
src = (payload or {}).get("source")
|
||||
if isinstance(src, dict):
|
||||
try:
|
||||
await self._source_store.upsert(
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
user_id=self._user_id,
|
||||
**src,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("source upsert failed for %s", src.get("id"), exc_info=True)
|
||||
|
||||
elif type == "phase_changed":
|
||||
new_phase = (payload or {}).get("to", phase)
|
||||
progress = _PHASE_PROGRESS.get(new_phase, 0)
|
||||
try:
|
||||
await self._job_store.update_progress(
|
||||
self._job_id,
|
||||
lease_owner=self._lease_owner,
|
||||
phase=new_phase,
|
||||
progress=progress,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("progress update failed for job %s", self._job_id, exc_info=True)
|
||||
|
||||
elif type == "sources_curated" and self._source_store is not None:
|
||||
for sid in (payload or {}).get("selectedIds", []):
|
||||
try:
|
||||
await self._source_store.set_selected(
|
||||
sid, selected=True, reason="curated", user_id=self._user_id
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
elif type == "awaiting_input":
|
||||
payload = payload or {}
|
||||
try:
|
||||
await self._job_store.update_progress(
|
||||
self._job_id,
|
||||
lease_owner=self._lease_owner,
|
||||
status="awaiting_input",
|
||||
phase=phase,
|
||||
progress=_PHASE_PROGRESS.get(phase, 10),
|
||||
awaiting_interrupt_id=str(payload.get("interruptId") or ""),
|
||||
awaiting_payload=payload,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("failed to pause deep-research job %s", self._job_id, exc_info=True)
|
||||
|
||||
return ev
|
||||
|
||||
async def publish_live(self, type, *, phase="initializing", payload=None): # noqa: A002
|
||||
"""Forward non-durable model deltas without adding projection writes."""
|
||||
await self._delegate.publish_live(type, phase=phase, payload=payload or {})
|
||||
|
||||
|
||||
# ── executor ────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class DeepResearchJobExecutor:
|
||||
"""Runs deep-research jobs as background asyncio tasks (one per worker)."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
job_store: DeepResearchJobRepository,
|
||||
session_store: DeepResearchSessionRepository,
|
||||
event_store: DeepResearchEventRepository,
|
||||
source_store: DeepResearchSourceRepository | None = None,
|
||||
live_publisher: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
|
||||
) -> None:
|
||||
self._job_store = job_store
|
||||
self._session_store = session_store
|
||||
self._event_store = event_store
|
||||
self._source_store = source_store
|
||||
self._live_publisher = live_publisher
|
||||
self._tasks: dict[str, asyncio.Task] = {}
|
||||
|
||||
# ── public API ──────────────────────────────────────────────────────────
|
||||
|
||||
def start_job(
|
||||
self,
|
||||
snapshot: dict[str, Any],
|
||||
*,
|
||||
job_id: str,
|
||||
lease_owner: str | None = None,
|
||||
) -> bool:
|
||||
"""Launch a background research task. Idempotent (same job → ignore).
|
||||
|
||||
Returns ``True`` if a new task was created.
|
||||
"""
|
||||
if job_id in self._tasks and not self._tasks[job_id].done():
|
||||
return False
|
||||
task = asyncio.create_task(self._run(snapshot, job_id=job_id, lease_owner=lease_owner))
|
||||
self._tasks[job_id] = task
|
||||
task.add_done_callback(lambda t, jid=job_id: self._tasks.pop(jid, None))
|
||||
return True
|
||||
|
||||
def cancel(self, job_id: str) -> bool:
|
||||
"""Cancel a running task. Returns ``True`` if the task was live."""
|
||||
task = self._tasks.get(job_id)
|
||||
if task is None or task.done():
|
||||
return False
|
||||
task.cancel()
|
||||
return True
|
||||
|
||||
def is_running(self, job_id: str) -> bool:
|
||||
task = self._tasks.get(job_id)
|
||||
return task is not None and not task.done()
|
||||
|
||||
async def await_job(self, job_id: str) -> None:
|
||||
"""Wait for a job to finish (tests only)."""
|
||||
task = self._tasks.get(job_id)
|
||||
if task is not None:
|
||||
with suppress(Exception, asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
# ── the main run ────────────────────────────────────────────────────────
|
||||
|
||||
async def _run(self, snapshot: dict[str, Any], *, job_id: str, lease_owner: str | None) -> None:
|
||||
session_id = str(snapshot.get("session_id") or "")
|
||||
user_id = str(snapshot.get("user_id") or "")
|
||||
query = str(snapshot.get("query") or "")
|
||||
config_dict = snapshot.get("config") or {}
|
||||
title = str(snapshot.get("title") or "")
|
||||
|
||||
token = set_current_user(_JobUser(user_id)) if user_id else None
|
||||
cancel_token = CancellationToken()
|
||||
|
||||
try:
|
||||
config = DeepResearchConfig.model_validate(config_dict)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("job %s: invalid config snapshot, marking failed", job_id)
|
||||
await self._mark_failed(job_id, session_id, lease_owner, "CONFIG_INVALID", "配置无效")
|
||||
if token is not None:
|
||||
reset_current_user(token)
|
||||
return
|
||||
|
||||
main_task = asyncio.current_task()
|
||||
heartbeat = (
|
||||
asyncio.create_task(self._heartbeat(job_id, lease_owner, main_task))
|
||||
if lease_owner
|
||||
else None
|
||||
)
|
||||
watcher = asyncio.create_task(
|
||||
self._watch_cancellation(job_id, main_task, lease_owner, cancel_token)
|
||||
)
|
||||
|
||||
try:
|
||||
result = await self._execute(
|
||||
job_id=job_id,
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
title=title,
|
||||
config=config,
|
||||
lease_owner=lease_owner,
|
||||
cancel_token=cancel_token,
|
||||
)
|
||||
await self._mark_completed(job_id, session_id, lease_owner, result)
|
||||
|
||||
except ResearchAwaitingInput as pause:
|
||||
logger.info("deep-research job %s awaiting human plan review", job_id)
|
||||
await self._mark_awaiting(job_id, session_id, pause)
|
||||
|
||||
except ResearchCancelled:
|
||||
logger.info("deep-research job %s cancelled (cooperative)", job_id)
|
||||
# Terminal state written by the watcher / cancel API.
|
||||
|
||||
except asyncio.CancelledError:
|
||||
logger.info("deep-research job %s cancelled (task)", job_id)
|
||||
raise
|
||||
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("deep-research job %s crashed", job_id)
|
||||
await self._mark_failed(job_id, session_id, lease_owner, type(exc).__name__, str(exc))
|
||||
|
||||
finally:
|
||||
watcher.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await watcher
|
||||
if heartbeat is not None:
|
||||
heartbeat.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await heartbeat
|
||||
if token is not None:
|
||||
reset_current_user(token)
|
||||
|
||||
# ── adapter assembly + engine call ──────────────────────────────────────
|
||||
|
||||
async def _execute(
|
||||
self,
|
||||
*,
|
||||
job_id: str,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
query: str,
|
||||
title: str,
|
||||
config: DeepResearchConfig,
|
||||
lease_owner: str | None,
|
||||
cancel_token: CancellationToken,
|
||||
) -> DeepResearchResult:
|
||||
from deerflow.agents.deep_research.adapters.artifacts import ThreadOutputsArtifactWriter
|
||||
from deerflow.agents.deep_research.adapters.context import DeerFlowContextCompressor
|
||||
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink
|
||||
from deerflow.agents.deep_research.adapters.image import build_image_provider
|
||||
from deerflow.agents.deep_research.adapters.llm import DeerFlowCompletionBackend
|
||||
from deerflow.agents.deep_research.adapters.material_provider import DeerFlowMaterialProvider
|
||||
|
||||
# Resolve / create the hidden runtime thread for artifacts.
|
||||
runtime_thread_id = await self._ensure_runtime_thread(session_id, user_id)
|
||||
|
||||
# Build the event sink chain: Persisting → Recording (side-effects).
|
||||
base_sink = PersistingEventSink(
|
||||
self._event_store,
|
||||
session_id=session_id,
|
||||
job_id=job_id,
|
||||
on_event=self._live_publisher,
|
||||
)
|
||||
recording_sink = RecordingEventSink(
|
||||
base_sink,
|
||||
source_store=self._source_store,
|
||||
job_store=self._job_store,
|
||||
job_id=job_id,
|
||||
session_id=session_id,
|
||||
user_id=user_id or None,
|
||||
lease_owner=lease_owner,
|
||||
)
|
||||
|
||||
# LLM backend (single chokepoint).
|
||||
llm_backend = DeerFlowCompletionBackend(config, event_sink=recording_sink, cancellation=cancel_token)
|
||||
|
||||
# Material provider.
|
||||
material_provider = DeerFlowMaterialProvider(
|
||||
user_id=user_id,
|
||||
session_id=session_id,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
model_name=config.fast_model or config.smart_model or config.strategic_model,
|
||||
)
|
||||
|
||||
# Context compressor (shares the LLM for the summary path).
|
||||
context_compressor = DeerFlowContextCompressor(llm=llm_backend)
|
||||
|
||||
# Artifact writer.
|
||||
artifact_writer = ThreadOutputsArtifactWriter(runtime_thread_id, user_id=user_id or None)
|
||||
|
||||
bundle = AdapterBundle(
|
||||
llm=llm_backend,
|
||||
materials=material_provider,
|
||||
context=context_compressor,
|
||||
artifacts=artifact_writer,
|
||||
events=recording_sink,
|
||||
images=build_image_provider(),
|
||||
)
|
||||
|
||||
job = await self._job_store.get_unscoped(job_id)
|
||||
request = DeepResearchRequest(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
config=config,
|
||||
resume_payload=(job or {}).get("awaiting_payload") or {},
|
||||
interrupted_plan=await self._load_interrupted_plan(job_id),
|
||||
)
|
||||
|
||||
engine = DeepResearchEngine()
|
||||
result = await engine.run(request, adapters=bundle, cancellation=cancel_token)
|
||||
return result
|
||||
|
||||
async def _load_interrupted_plan(self, job_id: str) -> dict[str, Any] | None:
|
||||
"""Recover the original plan-review payload when a queued job resumes."""
|
||||
events = await self._event_store.list_after(job_id, after=0, limit=200)
|
||||
for event in reversed(events):
|
||||
if event.get("event_type") != "awaiting_input":
|
||||
continue
|
||||
plan = (event.get("payload") or {}).get("plan")
|
||||
if isinstance(plan, dict):
|
||||
return plan
|
||||
return None
|
||||
|
||||
# ── terminal-state writers ──────────────────────────────────────────────
|
||||
|
||||
async def _mark_completed(
|
||||
self, job_id: str, session_id: str, lease_owner: str | None, result: DeepResearchResult
|
||||
) -> None:
|
||||
# Job row → completed.
|
||||
await self._job_store.update_progress(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="completed",
|
||||
phase="done",
|
||||
progress=100,
|
||||
result_snapshot={
|
||||
"report_markdown": result.report_markdown[:200] if result.report_markdown else "",
|
||||
"source_ids": result.source_ids,
|
||||
"plan": result.plan,
|
||||
},
|
||||
)
|
||||
# Session row → completed + report.
|
||||
source_count = len(result.source_ids)
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._session_store.update(
|
||||
session_id,
|
||||
status="completed",
|
||||
report_markdown=result.report_markdown,
|
||||
report_html=result.report_html,
|
||||
plan_snapshot=result.plan,
|
||||
# Keep the terminal job id alongside model usage. The
|
||||
# frontend uses it only to replay the persisted research trace
|
||||
# after a refresh; it does not affect billing or job state.
|
||||
usage_snapshot={**(result.usage or {}), "lastJobId": job_id},
|
||||
source_count=source_count,
|
||||
active_job_id=None,
|
||||
)
|
||||
# Emit a terminal event for SSE consumers.
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._emit_terminal(job_id, session_id, "job_completed")
|
||||
|
||||
async def _mark_awaiting(
|
||||
self, job_id: str, session_id: str, pause: ResearchAwaitingInput
|
||||
) -> None:
|
||||
"""Project a persisted plan-review interrupt onto the owning session."""
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._session_store.update(
|
||||
session_id,
|
||||
status="awaiting_input",
|
||||
plan_snapshot=pause.plan,
|
||||
active_job_id=job_id,
|
||||
)
|
||||
|
||||
async def _mark_failed(
|
||||
self, job_id: str, session_id: str, lease_owner: str | None, code: str, message: str
|
||||
) -> None:
|
||||
await self._job_store.update_progress(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="failed",
|
||||
error_code=code,
|
||||
error_message=message,
|
||||
)
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._session_store.update(
|
||||
session_id,
|
||||
status="failed",
|
||||
error=f"{code}: {message}",
|
||||
active_job_id=None,
|
||||
)
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._emit_terminal(job_id, session_id, "job_failed")
|
||||
|
||||
async def _emit_terminal(self, job_id: str, session_id: str, event_type: str) -> None:
|
||||
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink
|
||||
|
||||
sink = PersistingEventSink(
|
||||
self._event_store,
|
||||
session_id=session_id,
|
||||
job_id=job_id,
|
||||
on_event=self._live_publisher,
|
||||
)
|
||||
await sink.emit(event_type, phase="done", payload={"jobId": job_id})
|
||||
|
||||
# ── heartbeat (lease renewal) ───────────────────────────────────────────
|
||||
|
||||
async def _heartbeat(self, job_id: str, lease_owner: str, main_task: asyncio.Task) -> None:
|
||||
try:
|
||||
while not main_task.done():
|
||||
await asyncio.sleep(_HEARTBEAT_INTERVAL)
|
||||
if main_task.done():
|
||||
return
|
||||
try:
|
||||
ok = await self._job_store.renew_lease(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
lease_until=datetime.now(UTC) + timedelta(seconds=_LEASE_TTL_SECONDS),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if not ok:
|
||||
logger.info("deep-research job %s: lease lost, cancelling", job_id)
|
||||
if not main_task.done():
|
||||
main_task.cancel()
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
|
||||
# ── cancel watcher ──────────────────────────────────────────────────────
|
||||
|
||||
async def _watch_cancellation(
|
||||
self,
|
||||
job_id: str,
|
||||
main_task: asyncio.Task,
|
||||
lease_owner: str | None,
|
||||
cancel_token: CancellationToken,
|
||||
) -> None:
|
||||
"""Poll DB for ``cancel_requested``; cooperate + finalize when seen."""
|
||||
try:
|
||||
while not main_task.done():
|
||||
await asyncio.sleep(_CANCEL_POLL_INTERVAL)
|
||||
if main_task.done():
|
||||
return
|
||||
try:
|
||||
row = await self._job_store.get_unscoped(job_id)
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
if row is None:
|
||||
return
|
||||
if row.get("cancel_requested") or row.get("status") == "cancelled":
|
||||
logger.info("deep-research job %s: cancel requested, cooperating", job_id)
|
||||
cancel_token.cancel()
|
||||
# Finalize the cancel (lease-gated if we own the lease).
|
||||
if row.get("status") not in ("cancelled", "completed", "failed"):
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._job_store.finalize_cancel(job_id, lease_owner=lease_owner)
|
||||
if not main_task.done():
|
||||
main_task.cancel()
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
return
|
||||
|
||||
# ── runtime thread ──────────────────────────────────────────────────────
|
||||
|
||||
async def _ensure_runtime_thread(self, session_id: str, user_id: str) -> str:
|
||||
"""Get or create the hidden thread id for artifact storage."""
|
||||
existing = await self._session_store.get_runtime_thread_id(session_id)
|
||||
if existing:
|
||||
return existing
|
||||
thread_id = f"dr_{_uuid.uuid4().hex[:20]}"
|
||||
with suppress(Exception): # noqa: BLE001
|
||||
await self._session_store.update(session_id, runtime_thread_id=thread_id)
|
||||
return thread_id
|
||||
|
||||
|
||||
__all__ = ["DeepResearchJobExecutor", "WORKER_ID", "_LEASE_TTL_SECONDS"]
|
||||
Reference in new issue
Block a user