初始化
This commit is contained in:
1 parent
34474bd5fc
commit
b638b541d0
3008 files changed
+837699
No files matched your search
+19
@@ -0,0 +1,19 @@
|
||||
"""Deep Research adapters — the four-boundary isolation layer.
|
||||
|
||||
Adapters are the *only* place the vendored GPT Researcher algorithms touch
|
||||
DeerFlow infrastructure. Each adapter wraps one concern:
|
||||
|
||||
================ ===========================================================
|
||||
``llm`` model calls → ``create_chat_model()`` (no upstream Provider)
|
||||
``material`` search/fetch → DeerFlow ``web_search`` + knowledge + MCP
|
||||
``context`` context compression (no legacy text-splitter deps)
|
||||
``event_sink`` progress events → ``deep_research_events`` + SSE
|
||||
``artifacts`` report/export writes → hidden runtime-thread outputs
|
||||
``image`` image generation (optional, Phase 5)
|
||||
================ ===========================================================
|
||||
|
||||
Protocols live in each module alongside (or ahead of) their concrete impl, so
|
||||
tests can inject fakes that satisfy the protocol without importing infra.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
+159
@@ -0,0 +1,159 @@
|
||||
"""Artifact writer — writes report/export files into the session's hidden thread.
|
||||
|
||||
Each deep-research session owns a *hidden* DeerFlow runtime thread
|
||||
(``metadata.thread_type=deep_research``, ``hidden=true``). Report artifacts
|
||||
(``report.md``, ``report.html``, ``sources.json``, …) are written into that
|
||||
thread's sandbox ``outputs/`` directory and downloaded through the existing
|
||||
``/api/threads/{thread_id}/artifacts`` endpoint (which already has path-safety
|
||||
guards + owner checks — §13.6, §18.3).
|
||||
|
||||
The writer is a thin protocol so the engine/runner never touches the filesystem
|
||||
layout directly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ArtifactWriter(Protocol):
|
||||
"""Protocol for writing research artifacts (§13.6 §18)."""
|
||||
|
||||
@property
|
||||
def thread_id(self) -> str:
|
||||
"""The hidden runtime thread that owns these artifacts."""
|
||||
...
|
||||
|
||||
async def write_text(self, name: str, content: str) -> str:
|
||||
"""Write a text artifact (e.g. ``report.md``) and return its virtual path."""
|
||||
...
|
||||
|
||||
async def write_bytes(self, name: str, content: bytes) -> str:
|
||||
"""Write a binary artifact (e.g. ``report.pdf``). Optional capability."""
|
||||
...
|
||||
|
||||
def allowed_names(self) -> tuple[str, ...]:
|
||||
"""The server-enumerated filename whitelist (§18.3 — no arbitrary paths)."""
|
||||
...
|
||||
|
||||
|
||||
# Server-enumerated artifact filename whitelist (§18.3, §13.6).
|
||||
# The writer rejects any name not in this set to prevent path traversal.
|
||||
ARTIFACT_NAMES: tuple[str, ...] = (
|
||||
"report.md",
|
||||
"report.html",
|
||||
"sources.json",
|
||||
"research-plan.json",
|
||||
"report.pdf", # optional capability
|
||||
"report.docx", # optional capability
|
||||
)
|
||||
|
||||
# Image artifact names are server-generated only. Keeping the numerical range
|
||||
# small makes the whitelist finite in practice while still letting the report
|
||||
# writer select the matching format returned by the configured provider.
|
||||
_RESEARCH_IMAGE_NAME_RE = re.compile(r"research-image-0[1-4]\.(?:png|jpe?g|webp)$")
|
||||
|
||||
|
||||
def is_allowed_artifact_name(name: str) -> bool:
|
||||
"""Return whether *name* is one of the server-controlled research outputs."""
|
||||
return name in ARTIFACT_NAMES or bool(_RESEARCH_IMAGE_NAME_RE.fullmatch(name))
|
||||
|
||||
|
||||
def resolve_research_artifact_path(
|
||||
thread_id: str,
|
||||
name: str,
|
||||
*,
|
||||
user_id: str | None = None,
|
||||
) -> Path:
|
||||
"""Resolve a whitelisted research artifact into its thread outputs dir.
|
||||
|
||||
``Paths.resolve_virtual_path`` intentionally accepts only paths that an
|
||||
in-sandbox agent could see. Research code deals in server-generated file
|
||||
names, so passing a bare name (for example ``report.md``) would fail that
|
||||
guard on every host, including native Windows deployments. Keep the name
|
||||
allowlist and construct the canonical sandbox path here, at the boundary.
|
||||
"""
|
||||
if not is_allowed_artifact_name(name):
|
||||
raise ValueError(f"Artifact name {name!r} is not in the allowed research-artifact whitelist")
|
||||
|
||||
from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths
|
||||
|
||||
return get_paths().resolve_virtual_path(
|
||||
thread_id,
|
||||
f"{VIRTUAL_PATH_PREFIX}/outputs/{name}",
|
||||
user_id=user_id,
|
||||
)
|
||||
|
||||
|
||||
class ThreadOutputsArtifactWriter:
|
||||
"""Concrete writer backed by a DeerFlow thread's sandbox ``outputs/`` dir.
|
||||
|
||||
The actual path resolution reuses
|
||||
:func:`deerflow.config.paths.get_paths().sandbox_outputs_dir` (the same
|
||||
guard ``app/gateway/routers/artifacts.py`` uses), so traversal/symlink
|
||||
protection is inherited. File IO is offloaded to a thread so the event
|
||||
loop is never blocked (§25.6).
|
||||
"""
|
||||
|
||||
def __init__(self, thread_id: str, *, user_id: str | None = None) -> None:
|
||||
self._thread_id = thread_id
|
||||
self._user_id = user_id
|
||||
|
||||
@property
|
||||
def thread_id(self) -> str:
|
||||
return self._thread_id
|
||||
|
||||
def allowed_names(self) -> tuple[str, ...]:
|
||||
return ARTIFACT_NAMES
|
||||
|
||||
async def write_text(self, name: str, content: str) -> str:
|
||||
self._validate_name(name)
|
||||
import asyncio
|
||||
|
||||
path = await asyncio.to_thread(self._resolve_path, name)
|
||||
await asyncio.to_thread(self._write_file, path, content.encode("utf-8"))
|
||||
return name
|
||||
|
||||
async def write_bytes(self, name: str, content: bytes) -> str:
|
||||
self._validate_name(name)
|
||||
import asyncio
|
||||
|
||||
path = await asyncio.to_thread(self._resolve_path, name)
|
||||
await asyncio.to_thread(self._write_file, path, content)
|
||||
return name
|
||||
|
||||
# ── internals ───────────────────────────────────────────────────────────
|
||||
|
||||
def _validate_name(self, name: str) -> None:
|
||||
if not is_allowed_artifact_name(name):
|
||||
raise ValueError(
|
||||
f"Artifact name {name!r} is not in the allowed research-artifact whitelist"
|
||||
)
|
||||
|
||||
def _resolve_path(self, name: str):
|
||||
# Ensure the outputs dir exists; the paths module rejects traversal and
|
||||
# the name is whitelisted above, so the canonical virtual path is safe.
|
||||
from deerflow.config.paths import get_paths
|
||||
|
||||
get_paths().sandbox_outputs_dir(self._thread_id, user_id=self._user_id)
|
||||
return resolve_research_artifact_path(
|
||||
self._thread_id,
|
||||
name,
|
||||
user_id=self._user_id,
|
||||
)
|
||||
|
||||
def _write_file(self, path, data: bytes) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ARTIFACT_NAMES",
|
||||
"ArtifactWriter",
|
||||
"ThreadOutputsArtifactWriter",
|
||||
"is_allowed_artifact_name",
|
||||
"resolve_research_artifact_path",
|
||||
]
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
"""Context compressor — DeerFlow replacement for upstream's legacy compressor.
|
||||
|
||||
Upstream's ``ContextManager`` / ``compression`` module depends on
|
||||
``langchain_classic`` and ``langchain_text_splitters``, which DeerFlow does not
|
||||
declare. Rather than pull in those legacy packages, we keep the *calling
|
||||
interface* (§10) and provide :class:`DeerFlowContextCompressor` with a
|
||||
self-contained algorithm:
|
||||
|
||||
1. Chunk by title/paragraph/char-limit without breaking Markdown code blocks.
|
||||
2. Each chunk retains its source id (no cross-source amalgamation).
|
||||
3. If embeddings are configured → similarity pre-filter; else keyword/BM25-style.
|
||||
4. Over budget → per-source summary via ``fast_model`` (Phase 2).
|
||||
5. Final ranking balances source diversity, recency, relevance.
|
||||
|
||||
Phase 1 ships the chunking + keyword pre-filter (no embedding dependency);
|
||||
the ``fast_model`` summarisation path is wired but only triggers when the
|
||||
context genuinely exceeds ``max_context_words``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.agents.deep_research.adapters.llm import DeerFlowCompletionBackend
|
||||
from deerflow.agents.deep_research.types import ResearchMaterial
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ContextCompressor(Protocol):
|
||||
"""The interface vendored ``ContextManager`` is rewired to call (§10)."""
|
||||
|
||||
async def async_get_context(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 10,
|
||||
cost_callback: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return the compressed context list (each item = one material's context dict).
|
||||
|
||||
Output shape matches :meth:`ResearchMaterial.to_context_dict` so the
|
||||
vendored writer sees what it expects.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class DeerFlowContextCompressor:
|
||||
"""Concrete compressor: chunk → keyword-filter → optional summary.
|
||||
|
||||
Holds a reference to the current material set (set by the runner before
|
||||
the vendored research loop calls ``async_get_context``) and the LLM backend
|
||||
(for the summary path).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
llm: DeerFlowCompletionBackend | None = None,
|
||||
max_context_words: int = 24_000,
|
||||
) -> None:
|
||||
self._llm = llm
|
||||
self._max_words = max_context_words
|
||||
self._materials: list[ResearchMaterial] = []
|
||||
|
||||
def set_materials(self, materials: list[ResearchMaterial]) -> None:
|
||||
"""Prime the compressor with the current source pool (called per research phase)."""
|
||||
self._materials = list(materials)
|
||||
|
||||
async def async_get_context(
|
||||
self,
|
||||
query: str,
|
||||
max_results: int = 10,
|
||||
cost_callback: Callable[..., Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Select + compress the best ``max_results`` materials for ``query``."""
|
||||
from deerflow.agents.deep_research.adapters.material import dedupe_materials
|
||||
|
||||
deduped = dedupe_materials(self._materials)
|
||||
# Keyword pre-filter (no embedding dep in Phase 1).
|
||||
scored = [(_keyword_score(query, m), m) for m in deduped]
|
||||
scored.sort(key=lambda pair: pair[0], reverse=True)
|
||||
selected = [m for _, m in scored[:max_results]]
|
||||
|
||||
# Budget guard: if total words exceed limit, summarise the tail.
|
||||
total_words = sum(len((m.raw_content or "").split()) for m in selected)
|
||||
if total_words > self._max_words and self._llm is not None:
|
||||
selected = await self._summarise_tail(selected, query)
|
||||
|
||||
if cost_callback is not None:
|
||||
try:
|
||||
cost_callback(1, 0) # upstream signature: (total, done)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return [m.to_context_dict() for m in selected]
|
||||
|
||||
async def _summarise_tail(
|
||||
self, materials: list[ResearchMaterial], query: str
|
||||
) -> list[ResearchMaterial]:
|
||||
"""Summarise lower-ranked materials to fit the word budget (Phase 2)."""
|
||||
# Phase 1: simply truncate the list rather than calling the model,
|
||||
# so a fresh deploy without model tuning still produces a report.
|
||||
# (The full summarisation path is enabled once the basic runner is green.)
|
||||
words = 0
|
||||
kept: list[ResearchMaterial] = []
|
||||
for m in materials:
|
||||
w = len((m.raw_content or "").split())
|
||||
if words + w > self._max_words:
|
||||
break
|
||||
words += w
|
||||
kept.append(m)
|
||||
return kept or materials[:1]
|
||||
|
||||
|
||||
def _keyword_score(query: str, material: ResearchMaterial) -> float:
|
||||
"""Cheap relevance heuristic: fraction of query terms present in the material."""
|
||||
q_terms = {t.lower() for t in query.split() if len(t) > 1}
|
||||
if not q_terms:
|
||||
return 0.0
|
||||
haystack = f"{material.title}\n{material.raw_content}".lower()
|
||||
hits = sum(1 for t in q_terms if t in haystack)
|
||||
base = hits / len(q_terms)
|
||||
# Tiny recency/relevance nudges when metadata is present.
|
||||
if material.relevance_score:
|
||||
base += 0.01 * float(material.relevance_score)
|
||||
return base
|
||||
|
||||
|
||||
__all__ = ["ContextCompressor", "DeerFlowContextCompressor"]
|
||||
+376
@@ -0,0 +1,376 @@
|
||||
"""Persisting event sink — bridges runners to the event-store repository.
|
||||
|
||||
Runners call :meth:`PersistingEventSink.emit` (which satisfies the
|
||||
:class:`~deerflow.agents.deep_research.events.EventSink` protocol); the sink
|
||||
delegates to :class:`~deerflow.persistence.deep_research_events.DeepResearchEventRepository`,
|
||||
which assigns the monotonic ``seq`` and persists the row **before** returning.
|
||||
This "persist-then-stream" ordering (§15.3) is what makes ``?after=<seq>``
|
||||
reconnect-replay correct.
|
||||
|
||||
The sink is bound to one ``(session_id, job_id)`` pair at construction; runners
|
||||
don't carry those ids around.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from datetime import date, datetime
|
||||
from enum import Enum
|
||||
from time import monotonic
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.agents.deep_research.events import (
|
||||
DeepResearchEvent,
|
||||
EventType,
|
||||
ResearchPhase,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.persistence.deep_research_events import DeepResearchEventRepository
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _json_default(value: Any) -> Any:
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.isoformat()
|
||||
if isinstance(value, Enum):
|
||||
return value.value
|
||||
if hasattr(value, "model_dump"):
|
||||
return value.model_dump(mode="json")
|
||||
return str(value)
|
||||
|
||||
|
||||
# Live-only frame types that have no durable counterpart at all. Report and
|
||||
# summary text are already checkpointed (``report_chunk`` / ``summary_completed``),
|
||||
# so only the reasoning channel needs a cross-worker mirror.
|
||||
_MIRRORED_LIVE_TYPES = frozenset({"report_thinking", "plan_reasoning", "summary_thinking"})
|
||||
# Coalescing bounds for that mirror: one bounded row per ~600 characters or
|
||||
# ~1.5 seconds, never one row per provider token.
|
||||
_MIRROR_MIN_CHARS = 600
|
||||
_MIRROR_MAX_DELAY_S = 1.5
|
||||
# Floor for the time-based flush: a slowly trickling reasoning stream must not
|
||||
# degrade into one durable row per token. Anything shorter is carried until the
|
||||
# next durable milestone flushes it.
|
||||
_MIRROR_MIN_FLUSH_CHARS = 40
|
||||
|
||||
|
||||
def _json_safe_payload(payload: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize event payloads before they reach any event repository.
|
||||
|
||||
This protects deployments that still use a strict ``json.dumps`` inside
|
||||
the SQL repository and keeps native datetime values in source metadata
|
||||
from degrading every ``source_added`` event.
|
||||
"""
|
||||
return json.loads(json.dumps(payload, ensure_ascii=False, default=_json_default))
|
||||
|
||||
|
||||
class PersistingEventSink:
|
||||
"""EventSink implementation backed by the deep-research event repository.
|
||||
|
||||
Satisfies :class:`~deerflow.agents.deep_research.events.EventSink` (the
|
||||
runtime-checkable protocol) via structural typing.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
repo: DeepResearchEventRepository,
|
||||
*,
|
||||
session_id: str,
|
||||
job_id: str,
|
||||
on_event: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
|
||||
live_subscriber_probe: Callable[[str], bool] | None = None,
|
||||
) -> None:
|
||||
self._repo = repo
|
||||
self._session_id = session_id
|
||||
self._job_id = job_id
|
||||
self._on_event = on_event
|
||||
# Reports whether this worker holds an SSE connection for the job.
|
||||
# Absent probe → assume the live path reaches the viewer (single-worker
|
||||
# deployments and tests keep today's zero-extra-write behaviour).
|
||||
self._live_subscriber_probe = live_subscriber_probe
|
||||
self._mirror_buffers: dict[str, dict[str, Any]] = {}
|
||||
self._mirroring = False
|
||||
self.diagnostics: list[dict[str, Any]] = []
|
||||
# A durable event row normally owns the sequence. When the event
|
||||
# database is temporarily unavailable, use distinct negative sequence
|
||||
# numbers for the live-only path so clients can render every milestone
|
||||
# without advancing their durable ``after`` cursor.
|
||||
self._fallback_seq = 0
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
type: EventType, # noqa: A002
|
||||
*,
|
||||
phase: ResearchPhase = "initializing",
|
||||
payload: dict | None = None,
|
||||
) -> DeepResearchEvent:
|
||||
event_payload = _json_safe_payload(payload or {})
|
||||
persistence_error: Exception | None = None
|
||||
if not self._mirroring:
|
||||
# Order any coalesced reasoning tail before the next milestone
|
||||
# instead of leaving it buffered until the job ends.
|
||||
await self._flush_live_mirrors()
|
||||
try:
|
||||
row = await self._repo.append(
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
event_type=type,
|
||||
phase=phase,
|
||||
payload=event_payload,
|
||||
)
|
||||
# Reconstruct the envelope from the persisted row (seq is now assigned).
|
||||
event = DeepResearchEvent(
|
||||
seq=int(row.get("seq", 0)),
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
type=row.get("event_type", type),
|
||||
phase=row.get("phase", phase),
|
||||
payload=row.get("payload") or event_payload,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Progress persistence must not turn a recoverable database blip
|
||||
# into a failed report. The job/status stores still record terminal
|
||||
# state; this live-only frame lets an attached UI stay responsive
|
||||
# until durable replay becomes available again after reconnect.
|
||||
self._fallback_seq -= 1
|
||||
persistence_error = exc
|
||||
self.diagnostics.append(
|
||||
{
|
||||
"code": "event_persistence_degraded",
|
||||
"stage": phase,
|
||||
"eventType": type,
|
||||
"errorType": exc.__class__.__name__,
|
||||
"detail": str(exc).split("[SQL:", 1)[0].strip()[:600] or exc.__class__.__name__,
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
"deep-research event persistence degraded; continuing job with a live-only event "
|
||||
"(job=%s, type=%s, phase=%s, error=%s)",
|
||||
self._job_id,
|
||||
type,
|
||||
phase,
|
||||
persistence_error.__class__.__name__,
|
||||
)
|
||||
event = DeepResearchEvent(
|
||||
seq=self._fallback_seq,
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
type=type,
|
||||
phase=phase,
|
||||
payload={**event_payload, "eventPersistenceDegraded": True},
|
||||
)
|
||||
await self._publish_event(event)
|
||||
if persistence_error is not None:
|
||||
# Keep the original source/progress event live, and additionally
|
||||
# expose one copyable diagnostic frame. This never re-enters the
|
||||
# failing repository and therefore cannot interrupt the job.
|
||||
self._fallback_seq -= 1
|
||||
detail = str(persistence_error).split("[SQL:", 1)[0].strip()[:600]
|
||||
await self._publish_event(
|
||||
DeepResearchEvent(
|
||||
seq=self._fallback_seq,
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
type="warning",
|
||||
phase=phase,
|
||||
payload={
|
||||
"code": "event_persistence_degraded",
|
||||
"message": "研究过程事件暂未写入数据库,报告任务已继续运行",
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": phase,
|
||||
"eventType": type,
|
||||
"errorType": persistence_error.__class__.__name__,
|
||||
"detail": detail or persistence_error.__class__.__name__,
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
return event
|
||||
|
||||
async def _publish_event(self, event: DeepResearchEvent) -> None:
|
||||
"""Push a persisted or live-only event without risking the worker."""
|
||||
if self._on_event is None:
|
||||
return
|
||||
wire = event.to_sse_dict()
|
||||
if event.seq < 0:
|
||||
wire["liveId"] = f"event-persistence-fallback:{self._job_id}:{abs(event.seq)}"
|
||||
try:
|
||||
await self._on_event(wire)
|
||||
except Exception: # noqa: BLE001
|
||||
# A browser disconnect or a live-hub issue must not abort report
|
||||
# generation; persisted rows will still be replayable on reconnect.
|
||||
logger.warning(
|
||||
"deep-research live event publish failed; job continues (job=%s, seq=%s)",
|
||||
self._job_id,
|
||||
event.seq,
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def publish_live(
|
||||
self,
|
||||
type: str, # noqa: A002
|
||||
*,
|
||||
phase: ResearchPhase = "initializing",
|
||||
payload: dict | None = None,
|
||||
) -> None:
|
||||
if self._on_event is None:
|
||||
return
|
||||
try:
|
||||
await self._on_event(
|
||||
{
|
||||
"seq": 0,
|
||||
"sessionId": self._session_id,
|
||||
"jobId": self._job_id,
|
||||
"type": type,
|
||||
"phase": phase,
|
||||
"payload": payload or {},
|
||||
}
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.warning(
|
||||
"deep-research live frame publish failed; job continues (job=%s, type=%s)",
|
||||
self._job_id,
|
||||
type,
|
||||
exc_info=True,
|
||||
)
|
||||
with suppress(Exception):
|
||||
await self._mirror_live_frame(type, phase=phase, payload=payload or {})
|
||||
|
||||
# ── cross-worker mirror for live-only reasoning frames ──────────────────
|
||||
|
||||
async def _mirror_live_frame(
|
||||
self,
|
||||
type: str, # noqa: A002
|
||||
*,
|
||||
phase: ResearchPhase,
|
||||
payload: dict[str, Any],
|
||||
) -> None:
|
||||
"""Buffer a live-only reasoning delta when no local viewer can see it.
|
||||
|
||||
The live hub is in-process. A job runs on the worker that won its
|
||||
durable lease, while the SSE connection is served by whichever worker
|
||||
accepted it, so on a multi-worker deployment the model's thinking is
|
||||
usually published into a hub with no subscribers and is then lost —
|
||||
there is no durable row to replay it from. When that is the case, the
|
||||
reasoning is coalesced here and persisted as a bounded durable event
|
||||
so the polling stream on the other worker still renders it.
|
||||
"""
|
||||
if type not in _MIRRORED_LIVE_TYPES or self._live_subscriber_probe is None:
|
||||
return
|
||||
delta = payload.get("delta")
|
||||
if not isinstance(delta, str) or not delta:
|
||||
return
|
||||
if self._live_subscriber_probe(self._job_id):
|
||||
# A viewer is attached here and has already received every frame
|
||||
# live. Drop the buffer rather than flushing it, so that viewer
|
||||
# does not also receive the same text as a durable replay.
|
||||
self._mirror_buffers.pop(self._mirror_key(type, payload), None)
|
||||
return
|
||||
key = self._mirror_key(type, payload)
|
||||
buffered = self._mirror_buffers.get(key)
|
||||
if buffered is None:
|
||||
buffered = {
|
||||
"type": type,
|
||||
"phase": phase,
|
||||
"target": payload.get("target"),
|
||||
"reportVersion": payload.get("reportVersion"),
|
||||
"text": "",
|
||||
"since": monotonic(),
|
||||
}
|
||||
self._mirror_buffers[key] = buffered
|
||||
buffered["text"] += delta
|
||||
buffered["phase"] = phase
|
||||
buffered_chars = len(buffered["text"])
|
||||
aged_out = (
|
||||
buffered_chars >= _MIRROR_MIN_FLUSH_CHARS
|
||||
and monotonic() - float(buffered["since"]) >= _MIRROR_MAX_DELAY_S
|
||||
)
|
||||
if buffered_chars >= _MIRROR_MIN_CHARS or aged_out:
|
||||
await self._emit_mirror(key)
|
||||
|
||||
@staticmethod
|
||||
def _mirror_key(type: str, payload: dict[str, Any]) -> str: # noqa: A002
|
||||
return f"{type}::{payload.get('target') or 'report'}"
|
||||
|
||||
async def _flush_live_mirrors(self) -> None:
|
||||
for key in tuple(self._mirror_buffers):
|
||||
with suppress(Exception):
|
||||
await self._emit_mirror(key)
|
||||
|
||||
async def _emit_mirror(self, key: str) -> None:
|
||||
buffered = self._mirror_buffers.pop(key, None)
|
||||
if buffered is None or not buffered["text"]:
|
||||
return
|
||||
payload: dict[str, Any] = {
|
||||
"delta": buffered["text"],
|
||||
"liveMirror": True,
|
||||
}
|
||||
if buffered["target"] is not None:
|
||||
payload["target"] = buffered["target"]
|
||||
if buffered["reportVersion"] is not None:
|
||||
payload["reportVersion"] = buffered["reportVersion"]
|
||||
self._mirroring = True
|
||||
try:
|
||||
await self.emit(buffered["type"], phase=buffered["phase"], payload=payload)
|
||||
finally:
|
||||
self._mirroring = False
|
||||
|
||||
|
||||
class NullEventSink:
|
||||
"""No-op sink for unit tests / dry runs that don't have a database.
|
||||
|
||||
Satisfies :class:`EventSink` structurally; assigns a monotonic in-memory seq
|
||||
so fake tests can still assert event ordering.
|
||||
"""
|
||||
|
||||
def __init__(self, *, session_id: str = "test", job_id: str = "test") -> None:
|
||||
self._session_id = session_id
|
||||
self._job_id = job_id
|
||||
self._seq = 0
|
||||
self.events: list[DeepResearchEvent] = []
|
||||
self.live_events: list[dict[str, Any]] = []
|
||||
|
||||
async def emit(
|
||||
self,
|
||||
type: EventType, # noqa: A002
|
||||
*,
|
||||
phase: ResearchPhase = "initializing",
|
||||
payload: dict | None = None,
|
||||
) -> DeepResearchEvent:
|
||||
self._seq += 1
|
||||
ev = DeepResearchEvent(
|
||||
seq=self._seq,
|
||||
session_id=self._session_id,
|
||||
job_id=self._job_id,
|
||||
type=type,
|
||||
phase=phase,
|
||||
payload=payload or {},
|
||||
)
|
||||
self.events.append(ev)
|
||||
return ev
|
||||
|
||||
async def publish_live(
|
||||
self,
|
||||
type: str, # noqa: A002
|
||||
*,
|
||||
phase: ResearchPhase = "initializing",
|
||||
payload: dict | None = None,
|
||||
) -> None:
|
||||
self.live_events.append(
|
||||
{
|
||||
"seq": 0,
|
||||
"sessionId": self._session_id,
|
||||
"jobId": self._job_id,
|
||||
"type": type,
|
||||
"phase": phase,
|
||||
"payload": payload or {},
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["NullEventSink", "PersistingEventSink"]
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
"""Optional, DeerFlow-configured report image generation.
|
||||
|
||||
The upstream GPT Researcher image skill owns its provider configuration, so it
|
||||
cannot be safely reused in DeerFlow. This adapter supports both a deployment
|
||||
configured OpenAI-compatible ``/images/generations`` endpoint and the native
|
||||
DashScope multimodal-generation endpoint used by Qwen Image 3.0 and Wan 2.7.
|
||||
It is never configured from a browser request: model credentials stay in
|
||||
``config.yaml`` and the per-session snapshot contains only the boolean
|
||||
``generate_images``.
|
||||
|
||||
OpenAI-compatible images are requested as base64; DashScope image URLs are
|
||||
downloaded immediately. The runner therefore always puts exact bytes into the
|
||||
session's protected artifact directory, and reports never depend on an expiring
|
||||
public image URL.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from typing import Any, Literal, Protocol, runtime_checkable
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
_MAX_IMAGE_BYTES = 20 * 1024 * 1024
|
||||
_DASHSCOPE_GENERATION_PATH = "/api/v1/services/aigc/multimodal-generation/generation"
|
||||
|
||||
|
||||
class ImageGenerationError(RuntimeError):
|
||||
"""A recoverable image-provider failure (the report still completes)."""
|
||||
|
||||
|
||||
class ImageGenerationSettings(BaseModel):
|
||||
"""Server-only configuration for a report image endpoint.
|
||||
|
||||
Example ``config.yaml`` section::
|
||||
|
||||
deep_research:
|
||||
image_generation:
|
||||
enabled: true
|
||||
provider: dashscope_native
|
||||
base_url: https://token-plan.cn-beijing.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation
|
||||
api_key: $DEEP_RESEARCH_IMAGE_API_KEY
|
||||
model: qwen-image-3.0-pro
|
||||
|
||||
``api_key`` is resolved by :class:`AppConfig` while loading YAML, but this
|
||||
object is deliberately not placed in ``DeepResearchConfig`` and is never
|
||||
persisted to a session/job row.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
provider: Literal["openai_compatible", "dashscope_native"] = "openai_compatible"
|
||||
base_url: str = ""
|
||||
api_key: str = ""
|
||||
model: str = ""
|
||||
size: str = "1024x1024"
|
||||
quality: str | None = None
|
||||
watermark: bool = False
|
||||
timeout_seconds: float = Field(default=120, ge=1, le=300)
|
||||
|
||||
model_config = {"extra": "ignore"}
|
||||
|
||||
@field_validator("base_url")
|
||||
@classmethod
|
||||
def _normalise_base_url(cls, value: str) -> str:
|
||||
return value.strip().rstrip("/")
|
||||
|
||||
@field_validator("size")
|
||||
@classmethod
|
||||
def _validate_size(cls, value: str) -> str:
|
||||
text = value.strip()
|
||||
if not text or len(text) > 32:
|
||||
raise ValueError("image_generation.size must be a short non-empty value")
|
||||
return text
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
return (
|
||||
self.enabled
|
||||
and self.base_url.startswith(("https://", "http://"))
|
||||
and bool(self.api_key.strip())
|
||||
and bool(self.model.strip())
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_app_config(cls) -> ImageGenerationSettings:
|
||||
"""Read the optional ``deep_research.image_generation`` section.
|
||||
|
||||
The core ``AppConfig`` allows product-specific extra sections, which
|
||||
keeps this optional capability backward-compatible with old installs.
|
||||
Any malformed optional configuration disables image generation rather
|
||||
than preventing the gateway from serving normal research requests.
|
||||
"""
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
root = (get_app_config().model_extra or {}).get("deep_research")
|
||||
raw = root.get("image_generation") if isinstance(root, dict) else None
|
||||
return cls.model_validate(raw if isinstance(raw, dict) else {})
|
||||
except Exception: # noqa: BLE001
|
||||
return cls()
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ImageProvider(Protocol):
|
||||
"""Generate an image for a research section.
|
||||
|
||||
Successful providers return a mapping with binary ``data``, ``mime_type``,
|
||||
``alt`` and ``prompt`` fields. The runner, not the provider, writes the
|
||||
result into a protected research artifact.
|
||||
"""
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
section_id: str = "",
|
||||
user_id: str = "",
|
||||
) -> dict[str, Any] | None:
|
||||
"""Return image bytes metadata, or ``None`` when generation is disabled."""
|
||||
...
|
||||
|
||||
|
||||
class NullImageProvider:
|
||||
"""No-op image provider — ``generate_images=false`` default."""
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
section_id: str = "",
|
||||
user_id: str = "",
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class OpenAICompatibleImageProvider:
|
||||
"""Generate one image through a configured OpenAI-compatible endpoint.
|
||||
|
||||
``transport`` is injection-only for adapter tests. Production instances
|
||||
use a normal ``httpx.AsyncClient`` and make no assumptions about a specific
|
||||
vendor beyond the standard ``/images/generations`` response shape.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: ImageGenerationSettings,
|
||||
*,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
if not settings.is_configured:
|
||||
raise ValueError("Image generation provider is not configured")
|
||||
self._settings = settings
|
||||
self._transport = transport
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
section_id: str = "",
|
||||
user_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
del user_id # Credentials are deployment-scoped; users never supply them.
|
||||
body: dict[str, Any] = {
|
||||
"model": self._settings.model,
|
||||
"prompt": prompt,
|
||||
"n": 1,
|
||||
"size": self._settings.size,
|
||||
# Base64 is mandatory: it yields a stable, protected local artifact.
|
||||
"response_format": "b64_json",
|
||||
}
|
||||
if self._settings.quality:
|
||||
body["quality"] = self._settings.quality
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._settings.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
endpoint = f"{self._settings.base_url}/images/generations"
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self._settings.timeout_seconds,
|
||||
transport=self._transport,
|
||||
) as client:
|
||||
response = await client.post(endpoint, json=body, headers=headers)
|
||||
response.raise_for_status()
|
||||
except httpx.TimeoutException as exc:
|
||||
raise ImageGenerationError("图片生成服务响应超时") from exc
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise ImageGenerationError(f"图片生成服务返回 HTTP {exc.response.status_code}") from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise ImageGenerationError("图片生成服务暂时不可用") from exc
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise ImageGenerationError("图片生成服务返回了无效响应") from exc
|
||||
items = payload.get("data") if isinstance(payload, dict) else None
|
||||
item = items[0] if isinstance(items, list) and items and isinstance(items[0], dict) else None
|
||||
encoded = str((item or {}).get("b64_json") or "")
|
||||
if not encoded:
|
||||
raise ImageGenerationError("图片生成服务未返回可保存的图片数据")
|
||||
|
||||
# A few compatible endpoints prepend a data-URL header. Strip it
|
||||
# without accepting arbitrary content as a URL or another fetch target.
|
||||
if encoded.startswith("data:") and "," in encoded:
|
||||
encoded = encoded.split(",", 1)[1]
|
||||
try:
|
||||
data = base64.b64decode(encoded, validate=True)
|
||||
except (ValueError, binascii.Error) as exc:
|
||||
raise ImageGenerationError("图片生成服务返回的图片编码无效") from exc
|
||||
if not data or len(data) > _MAX_IMAGE_BYTES:
|
||||
raise ImageGenerationError("生成图片为空或超过大小限制")
|
||||
|
||||
mime_type = str((item or {}).get("mime_type") or "image/png").lower()
|
||||
if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
|
||||
mime_type = "image/png"
|
||||
alt = f"{section_id or '研究报告'}配图"
|
||||
return {"data": data, "mime_type": mime_type, "alt": alt, "prompt": prompt}
|
||||
|
||||
|
||||
class DashScopeNativeImageProvider:
|
||||
"""Generate report illustrations through DashScope's native image API.
|
||||
|
||||
Qwen Image 3.0 and Wan 2.7 do not implement the OpenAI
|
||||
``/images/generations`` request/response contract. They receive a
|
||||
multimodal ``input.messages`` payload and normally return a short-lived
|
||||
image URL. The URL is downloaded immediately so the research report keeps
|
||||
a protected local artifact, just like the OpenAI-compatible provider.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: ImageGenerationSettings,
|
||||
*,
|
||||
transport: httpx.AsyncBaseTransport | None = None,
|
||||
) -> None:
|
||||
if not settings.is_configured:
|
||||
raise ValueError("Image generation provider is not configured")
|
||||
self._settings = settings
|
||||
self._transport = transport
|
||||
|
||||
async def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
section_id: str = "",
|
||||
user_id: str = "",
|
||||
) -> dict[str, Any]:
|
||||
del user_id # Credentials are deployment-scoped; users never supply them.
|
||||
body = _dashscope_request_body(self._settings, prompt)
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self._settings.api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self._settings.timeout_seconds,
|
||||
transport=self._transport,
|
||||
follow_redirects=True,
|
||||
) as client:
|
||||
response = await client.post(
|
||||
_dashscope_generation_endpoint(self._settings.base_url),
|
||||
json=body,
|
||||
headers=headers,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
encoded, image_url, mime_type = _dashscope_image_result(payload)
|
||||
if encoded:
|
||||
data = _decode_image_base64(encoded)
|
||||
elif image_url:
|
||||
data, mime_type = await _download_dashscope_image(client, image_url, mime_type)
|
||||
else:
|
||||
raise ImageGenerationError("千问图片生成服务未返回图片地址或图片数据")
|
||||
except ImageGenerationError:
|
||||
raise
|
||||
except httpx.TimeoutException as exc:
|
||||
raise ImageGenerationError("图片生成服务响应超时") from exc
|
||||
except httpx.HTTPStatusError as exc:
|
||||
raise ImageGenerationError(_http_error_message(exc.response)) from exc
|
||||
except httpx.HTTPError as exc:
|
||||
raise ImageGenerationError("图片生成服务暂时不可用") from exc
|
||||
except ValueError as exc:
|
||||
raise ImageGenerationError("图片生成服务返回了无效响应") from exc
|
||||
|
||||
if not data or len(data) > _MAX_IMAGE_BYTES:
|
||||
raise ImageGenerationError("生成图片为空或超过大小限制")
|
||||
alt = f"{section_id or '研究报告'}配图"
|
||||
return {"data": data, "mime_type": _normalise_mime_type(mime_type), "alt": alt, "prompt": prompt}
|
||||
|
||||
|
||||
def _dashscope_generation_endpoint(base_url: str) -> str:
|
||||
"""Resolve a native DashScope endpoint without relying on a workspace id.
|
||||
|
||||
Token Plan deployments commonly expose the full native endpoint, while
|
||||
regular DashScope configuration often uses just an API origin. Supporting
|
||||
both avoids an accidental ``.../generation/images/generations`` path.
|
||||
"""
|
||||
|
||||
base = base_url.rstrip("/")
|
||||
if base.endswith(_DASHSCOPE_GENERATION_PATH):
|
||||
return base
|
||||
parsed = urlparse(base)
|
||||
if not parsed.scheme or not parsed.netloc:
|
||||
return base
|
||||
return f"{parsed.scheme}://{parsed.netloc}{_DASHSCOPE_GENERATION_PATH}"
|
||||
|
||||
|
||||
def _dashscope_request_body(settings: ImageGenerationSettings, prompt: str) -> dict[str, Any]:
|
||||
parameters: dict[str, Any] = {
|
||||
"n": 1,
|
||||
"size": settings.size.replace("x", "*").replace("X", "*"),
|
||||
"watermark": settings.watermark,
|
||||
}
|
||||
# ``prompt_extend`` is a Qwen Image-only parameter. Wan 2.7 rejects
|
||||
# unrelated generation options, so do not send it there.
|
||||
if settings.model.strip().lower().startswith("qwen-image-"):
|
||||
parameters["prompt_extend"] = True
|
||||
return {
|
||||
"model": settings.model,
|
||||
"input": {
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"text": prompt}],
|
||||
}
|
||||
]
|
||||
},
|
||||
"parameters": parameters,
|
||||
}
|
||||
|
||||
|
||||
def _dashscope_image_result(payload: object) -> tuple[str, str, str]:
|
||||
"""Return optional base64, image URL and MIME type from native output."""
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return "", "", ""
|
||||
output = payload.get("output")
|
||||
choices = output.get("choices") if isinstance(output, dict) else None
|
||||
if not isinstance(choices, list):
|
||||
return "", "", ""
|
||||
for choice in choices:
|
||||
message = choice.get("message") if isinstance(choice, dict) else None
|
||||
content = message.get("content") if isinstance(message, dict) else None
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for part in content:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
encoded = str(part.get("b64_json") or "")
|
||||
raw_image_url = part.get("image") or part.get("image_url")
|
||||
if isinstance(raw_image_url, dict):
|
||||
raw_image_url = raw_image_url.get("url")
|
||||
image_url = str(raw_image_url or "")
|
||||
if encoded or image_url:
|
||||
return encoded, image_url, str(part.get("mime_type") or "")
|
||||
return "", "", ""
|
||||
|
||||
|
||||
async def _download_dashscope_image(
|
||||
client: httpx.AsyncClient,
|
||||
image_url: str,
|
||||
mime_type: str,
|
||||
) -> tuple[bytes, str]:
|
||||
"""Download only an HTTPS model result, never forwarding the API key."""
|
||||
|
||||
parsed = urlparse(image_url)
|
||||
if parsed.scheme != "https" or not parsed.netloc:
|
||||
raise ImageGenerationError("千问图片生成服务返回了不安全的图片地址")
|
||||
response = await client.get(image_url, headers={"Accept": "image/png,image/jpeg,image/webp"})
|
||||
response.raise_for_status()
|
||||
final_url = urlparse(str(response.url))
|
||||
if final_url.scheme != "https" or not final_url.netloc:
|
||||
raise ImageGenerationError("千问图片下载地址不安全")
|
||||
response_mime_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
|
||||
return response.content, response_mime_type or mime_type
|
||||
|
||||
|
||||
def _decode_image_base64(encoded: str) -> bytes:
|
||||
if encoded.startswith("data:") and "," in encoded:
|
||||
encoded = encoded.split(",", 1)[1]
|
||||
try:
|
||||
return base64.b64decode(encoded, validate=True)
|
||||
except (ValueError, binascii.Error) as exc:
|
||||
raise ImageGenerationError("图片生成服务返回的图片编码无效") from exc
|
||||
|
||||
|
||||
def _normalise_mime_type(value: str) -> str:
|
||||
mime_type = value.lower()
|
||||
return mime_type if mime_type in {"image/png", "image/jpeg", "image/webp"} else "image/png"
|
||||
|
||||
|
||||
def _http_error_message(response: httpx.Response) -> str:
|
||||
"""Expose a short provider message without leaking credentials or payloads."""
|
||||
|
||||
detail = ""
|
||||
try:
|
||||
payload = response.json()
|
||||
if isinstance(payload, dict):
|
||||
error = payload.get("error")
|
||||
source = error if isinstance(error, dict) else payload
|
||||
message = source.get("message") if isinstance(source, dict) else None
|
||||
if isinstance(message, str) and message.strip():
|
||||
detail = f":{message.strip()[:300]}"
|
||||
except ValueError:
|
||||
pass
|
||||
return f"图片生成服务返回 HTTP {response.status_code}{detail}"
|
||||
|
||||
|
||||
def build_image_provider() -> ImageProvider:
|
||||
"""Build the optional provider from server config, or a safe no-op."""
|
||||
settings = ImageGenerationSettings.from_app_config()
|
||||
if not settings.is_configured:
|
||||
return NullImageProvider()
|
||||
if settings.provider == "dashscope_native":
|
||||
return DashScopeNativeImageProvider(settings)
|
||||
return OpenAICompatibleImageProvider(settings)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"ImageGenerationError",
|
||||
"ImageGenerationSettings",
|
||||
"ImageProvider",
|
||||
"NullImageProvider",
|
||||
"OpenAICompatibleImageProvider",
|
||||
"DashScopeNativeImageProvider",
|
||||
"build_image_provider",
|
||||
]
|
||||
+639
@@ -0,0 +1,639 @@
|
||||
"""LLM adapter — the single chokepoint between vendored code and DeerFlow models.
|
||||
|
||||
The vendored GPT Researcher code expects ``create_chat_completion()`` /
|
||||
``get_llm()`` style helpers (``gpt_researcher/utils/llm.py``). Rather than
|
||||
letting it dynamically import providers, we keep those function *signatures* in
|
||||
the vendored copy but route them through :class:`DeerFlowCompletionBackend`,
|
||||
which delegates to DeerFlow's :func:`create_chat_model`.
|
||||
|
||||
Responsibilities (§9):
|
||||
|
||||
- Resolve the model name from the per-task ``DeepResearchConfig`` by role
|
||||
(``fast`` / ``smart`` / ``strategic``); ``None`` → system default.
|
||||
- Convert ``[{role, content}, ...]`` dicts to LangChain messages.
|
||||
- Call ``model.ainvoke()`` and normalise text + ``usage_metadata``.
|
||||
- Support structured output via a Pydantic ``response_schema`` (one JSON-repair
|
||||
retry, no ``json-repair`` hard dep — §9.5).
|
||||
- Emit a ``model_usage`` event per call.
|
||||
- Honour the :class:`CancellationToken`.
|
||||
|
||||
Retry policy (§9.5): exactly **one** retry layer here — a structured-output
|
||||
parse failure triggers one "fix the JSON" re-prompt. Cancel / permission /
|
||||
config errors are never retried.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from deerflow.agents.deep_research.cancellation import CancellationToken
|
||||
from deerflow.agents.deep_research.config import DeepResearchConfig, ModelRole
|
||||
from deerflow.agents.deep_research.events import EventSink
|
||||
from deerflow.agents.deep_research.types import CompletionResult
|
||||
from deerflow.utils.stream_text import InlineThinkTagFilter, strip_inline_think_tags
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langchain.chat_models import BaseChatModel
|
||||
from langchain_core.messages import BaseMessage
|
||||
from pydantic import BaseModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _parent_isolated_run_config(run_name: str) -> dict[str, Any]:
|
||||
"""Keep LangGraph runtime, drop only inherited LLM streaming callbacks.
|
||||
|
||||
``StreamWriter`` reads ``get_config()[configurable]`` at emit time. Replacing
|
||||
the child config with a blank dict makes ``report_delta`` / ``report_thinking``
|
||||
raise and get swallowed, so the sandbox only appears after the file lands.
|
||||
"""
|
||||
parent: dict[str, Any] = {}
|
||||
try:
|
||||
from langchain_core.runnables.config import var_child_runnable_config
|
||||
|
||||
current = var_child_runnable_config.get()
|
||||
if isinstance(current, dict):
|
||||
parent = dict(current)
|
||||
except Exception: # noqa: BLE001 — isolation must never fail the completion
|
||||
parent = {}
|
||||
tags = [tag for tag in (parent.get("tags") or []) if tag != "nostream"]
|
||||
tags.append("nostream")
|
||||
parent["callbacks"] = []
|
||||
parent["tags"] = tags
|
||||
parent["run_name"] = run_name
|
||||
return parent
|
||||
|
||||
|
||||
def _clear_inherited_runnable_config(isolated: dict[str, Any]) -> Callable[[], None]:
|
||||
"""Install the isolated config for the duration of ``astream``."""
|
||||
try:
|
||||
from langchain_core.runnables.config import var_child_runnable_config
|
||||
except ImportError:
|
||||
return lambda: None
|
||||
|
||||
token = var_child_runnable_config.set(isolated)
|
||||
return lambda: var_child_runnable_config.reset(token)
|
||||
|
||||
|
||||
# ── message conversion ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def to_langchain_messages(messages: list[dict[str, str]]) -> list[BaseMessage]:
|
||||
"""Convert ``[{role, content}, ...]`` dicts to LangChain message objects.
|
||||
|
||||
``role`` accepts the usual OpenAI-ish names: ``system`` / ``user`` /
|
||||
``assistant`` / ``human`` / ``ai``. Unknown roles fall back to ``HumanMessage``
|
||||
so a bad prompt never crashes the whole research run.
|
||||
"""
|
||||
from langchain_core.messages import (
|
||||
AIMessage,
|
||||
HumanMessage,
|
||||
SystemMessage,
|
||||
)
|
||||
|
||||
out: list[BaseMessage] = []
|
||||
for m in messages:
|
||||
role = (m.get("role") or "user").lower()
|
||||
content = m.get("content") or ""
|
||||
if role in ("system", "developer"):
|
||||
out.append(SystemMessage(content=content))
|
||||
elif role in ("assistant", "ai"):
|
||||
out.append(AIMessage(content=content))
|
||||
else: # user / human / unknown → human (safe default)
|
||||
out.append(HumanMessage(content=content))
|
||||
return out
|
||||
|
||||
|
||||
def _content_to_text(resp: Any) -> str:
|
||||
"""Best-effort extraction of text from a LangChain AIMessage-like response."""
|
||||
content = getattr(resp, "content", None)
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, list):
|
||||
# Some providers return content blocks; concatenate text parts.
|
||||
parts: list[str] = []
|
||||
for block in content:
|
||||
if isinstance(block, str):
|
||||
parts.append(block)
|
||||
elif isinstance(block, dict):
|
||||
# Never fall through to ``content`` for structured reasoning
|
||||
# blocks. Providers differ here, but those blocks are not
|
||||
# report prose and must not reach ReportChunkEmitter.
|
||||
if block.get("type") in {
|
||||
"thinking",
|
||||
"thinking_delta",
|
||||
"reasoning",
|
||||
"reasoning_delta",
|
||||
}:
|
||||
continue
|
||||
parts.append(str(block.get("text") or block.get("content") or ""))
|
||||
return "".join(parts)
|
||||
return str(content or "")
|
||||
|
||||
|
||||
def _reasoning_delta(chunk: Any) -> str:
|
||||
"""Extract the incremental reasoning/thinking text from a streamed chunk.
|
||||
|
||||
Reasoning models (deepseek-reasoner, GLM thinking, QwQ, ...) stream their
|
||||
thinking in ``additional_kwargs.reasoning_content`` while ``content`` stays
|
||||
empty until the final answer starts. A stream loop that only reads
|
||||
``content`` skips these chunks entirely, so callers get a dedicated
|
||||
callback for them.
|
||||
"""
|
||||
ak = getattr(chunk, "additional_kwargs", None) or {}
|
||||
val = ak.get("reasoning_content") or ak.get("reasoning") or ""
|
||||
if isinstance(val, str) and val:
|
||||
return val
|
||||
# Anthropic/OpenAI-compatible adapters may put the same increment in an
|
||||
# explicitly typed content block instead of ``additional_kwargs``.
|
||||
content = getattr(chunk, "content", None)
|
||||
if isinstance(content, list):
|
||||
parts = [
|
||||
str(block.get("text") or block.get("content") or "")
|
||||
for block in content
|
||||
if isinstance(block, dict)
|
||||
and block.get("type") in {"thinking", "thinking_delta", "reasoning", "reasoning_delta"}
|
||||
]
|
||||
return "".join(parts)
|
||||
return ""
|
||||
|
||||
|
||||
def _extract_usage(resp: Any) -> dict[str, Any]:
|
||||
"""Read ``AIMessage.usage_metadata`` (§9.4) — model-agnostic token accounting."""
|
||||
um = getattr(resp, "usage_metadata", None)
|
||||
if isinstance(um, dict):
|
||||
return {
|
||||
"prompt_tokens": um.get("input_tokens", 0),
|
||||
"completion_tokens": um.get("output_tokens", 0),
|
||||
"total_tokens": um.get("total_tokens", 0),
|
||||
}
|
||||
# Fallback: some providers populate response_metadata.
|
||||
rm = getattr(resp, "response_metadata", None) or {}
|
||||
token_usage = rm.get("token_usage") or rm.get("usage") or {}
|
||||
if isinstance(token_usage, dict) and token_usage:
|
||||
return {
|
||||
"prompt_tokens": token_usage.get("prompt_tokens", 0),
|
||||
"completion_tokens": token_usage.get("completion_tokens", 0),
|
||||
"total_tokens": token_usage.get("total_tokens", 0),
|
||||
}
|
||||
return {}
|
||||
|
||||
|
||||
# ── errors ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class CompletionError(Exception):
|
||||
"""Non-retryable completion failure (config / permission / budget)."""
|
||||
|
||||
def __init__(self, message: str, *, code: str = "completion_error", retryable: bool = False) -> None:
|
||||
super().__init__(message)
|
||||
self.code = code
|
||||
self.retryable = retryable
|
||||
|
||||
|
||||
# ── the backend ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class DeerFlowCompletionBackend:
|
||||
"""The LLM adapter every vendored call funnels through.
|
||||
|
||||
Constructed once per job (with the frozen config) and handed to runners via
|
||||
:class:`AdapterBundle`. Model instances are cached by role so repeated
|
||||
calls don't rebuild the provider.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: DeepResearchConfig,
|
||||
event_sink: EventSink | None = None,
|
||||
cancellation: CancellationToken | None = None,
|
||||
force_disable_thinking: bool = False,
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._events = event_sink
|
||||
self._cancel = cancellation
|
||||
# 与 lead agent 同语义的「思考硬关」开关(内网 vLLM/Qwen 只声明
|
||||
# supports_thinking 时,thinking_enabled=False 不会注入 enable_thinking=false)。
|
||||
self._force_disable_thinking = force_disable_thinking
|
||||
self._models: dict[str, BaseChatModel] = {}
|
||||
# Running token totals (keyed by model name) for the final usage report.
|
||||
self._usage_totals: dict[str, dict[str, int]] = {}
|
||||
|
||||
# ── model resolution ────────────────────────────────────────────────────
|
||||
|
||||
def _resolve_model_name(self, role: ModelRole) -> str | None:
|
||||
return self._config.resolve_model_name(role)
|
||||
|
||||
def _get_model(self, role: ModelRole) -> BaseChatModel:
|
||||
name = self._resolve_model_name(role)
|
||||
cache_key = name or "__default__"
|
||||
cached = self._models.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
from deerflow.models import create_chat_model
|
||||
|
||||
model = create_chat_model(
|
||||
name=name,
|
||||
thinking_enabled=self._config.thinking_enabled,
|
||||
force_disable_thinking=self._force_disable_thinking,
|
||||
)
|
||||
self._models[cache_key] = model
|
||||
return model
|
||||
|
||||
# ── the main entry point ────────────────────────────────────────────────
|
||||
|
||||
async def complete(
|
||||
self,
|
||||
*,
|
||||
model_role: ModelRole = "smart",
|
||||
messages: list[dict[str, str]],
|
||||
temperature: float | None = None,
|
||||
response_schema: type[BaseModel] | None = None,
|
||||
operation: str = "unknown",
|
||||
) -> CompletionResult:
|
||||
"""Run one completion, normalise output, bill usage.
|
||||
|
||||
Args:
|
||||
model_role: which config role to use (fast/cheap research queries →
|
||||
``fast``; report writing → ``smart``; strategic planning → ``strategic``).
|
||||
messages: OpenAI-style ``[{role, content}]`` dicts.
|
||||
temperature: optional override (most vendored calls don't set one).
|
||||
response_schema: if given, parse + validate the output against this
|
||||
Pydantic model (structured output). One JSON-repair retry on
|
||||
parse failure (§9.5).
|
||||
operation: a short label for the ``model_usage`` event
|
||||
(e.g. ``"generate_queries"``, ``"write_report"``).
|
||||
"""
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
|
||||
model = self._get_model(model_role)
|
||||
model_name = self._resolve_model_name(model_role) or "default"
|
||||
lc_messages = to_langchain_messages(messages)
|
||||
|
||||
kwargs: dict[str, Any] = {}
|
||||
if temperature is not None:
|
||||
kwargs["temperature"] = temperature
|
||||
|
||||
# ── plain completion ──
|
||||
if response_schema is None:
|
||||
resp = await self._invoke_with_timeout(model, lc_messages, kwargs, operation)
|
||||
text = _content_to_text(resp)
|
||||
usage = _extract_usage(resp)
|
||||
await self._record_usage(model_name, operation, usage)
|
||||
return CompletionResult(text=text, model=model_name, usage=usage)
|
||||
|
||||
# ── structured completion (with one repair retry) ──
|
||||
schema_instructions = _schema_instructions(response_schema)
|
||||
structured_messages = list(lc_messages)
|
||||
# Append the schema instruction to the last system/user message block.
|
||||
structured_messages = _inject_schema_hint(structured_messages, schema_instructions)
|
||||
|
||||
for attempt in range(2):
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
resp = await self._invoke_with_timeout(model, structured_messages, kwargs, operation)
|
||||
text = _content_to_text(resp)
|
||||
usage = _extract_usage(resp)
|
||||
parsed = _try_parse_structured(text, response_schema)
|
||||
if parsed is not None:
|
||||
await self._record_usage(model_name, operation, usage)
|
||||
return CompletionResult(text=text, model=model_name, usage=usage, structured=parsed)
|
||||
if attempt == 0:
|
||||
# One repair re-prompt (§9.5).
|
||||
logger.warning("deep_research llm: JSON parse failed for %s, retrying", operation)
|
||||
structured_messages = _inject_repair_hint(structured_messages, text, response_schema)
|
||||
await self._record_usage(model_name, operation, usage)
|
||||
|
||||
raise CompletionError(
|
||||
f"Structured output for '{operation}' could not be parsed after retry",
|
||||
code="structured_parse_failed",
|
||||
retryable=True,
|
||||
)
|
||||
|
||||
async def _invoke_with_timeout(
|
||||
self,
|
||||
model: BaseChatModel,
|
||||
messages: list[BaseMessage],
|
||||
kwargs: dict[str, Any],
|
||||
operation: str,
|
||||
) -> Any:
|
||||
"""Bound a non-streaming control-plane call without capping report prose.
|
||||
|
||||
Model configurations may have a much longer HTTP ``request_timeout``.
|
||||
A research job cannot leave its visible planning/review stage frozen for
|
||||
that whole interval, so these short structured calls get their own
|
||||
deadline. ``wait_for`` cancels the pending provider coroutine before a
|
||||
durable runner either uses its safe fallback or records a terminal error.
|
||||
"""
|
||||
timeout = self._config.control_plane_timeout_seconds
|
||||
try:
|
||||
return await asyncio.wait_for(model.ainvoke(messages, **kwargs), timeout=timeout)
|
||||
except TimeoutError as exc:
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
raise CompletionError(
|
||||
f"Model did not complete '{operation}' within {timeout:g} seconds",
|
||||
code="completion_timeout",
|
||||
retryable=True,
|
||||
) from exc
|
||||
|
||||
async def stream_complete(
|
||||
self,
|
||||
*,
|
||||
model_role: ModelRole = "smart",
|
||||
messages: list[dict[str, str]],
|
||||
temperature: float | None = None,
|
||||
max_tokens: int | None = None,
|
||||
operation: str = "unknown",
|
||||
on_delta: Callable[[str], Awaitable[None]] | None = None,
|
||||
on_reasoning: Callable[[str], Awaitable[None]] | None = None,
|
||||
) -> CompletionResult:
|
||||
"""Run a prose completion through LangChain's incremental stream API.
|
||||
|
||||
Structured planning/review calls intentionally stay on :meth:`complete`.
|
||||
This method is reserved for report prose so the final returned text is
|
||||
exactly the concatenation of the deltas the user has already seen.
|
||||
|
||||
``on_reasoning`` receives the thinking/reasoning increments reasoning
|
||||
models emit before the answer; they are not part of the returned text.
|
||||
"""
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
|
||||
model = self._get_model(model_role)
|
||||
model_name = self._resolve_model_name(model_role) or "default"
|
||||
kwargs: dict[str, Any] = {}
|
||||
if temperature is not None:
|
||||
kwargs["temperature"] = temperature
|
||||
if max_tokens is not None:
|
||||
# Report-only callers may need a larger completion than a model's
|
||||
# conservative deployment default. Codex Responses endpoints
|
||||
# reject both token-limit parameters, so preserve the factory's
|
||||
# provider-specific exclusion for that adapter.
|
||||
from deerflow.models.openai_codex_provider import CodexChatModel
|
||||
|
||||
if not isinstance(model, CodexChatModel):
|
||||
kwargs["max_tokens"] = max_tokens
|
||||
|
||||
text_parts: list[str] = []
|
||||
usage: dict[str, Any] = {}
|
||||
inline_think_filter = InlineThinkTagFilter()
|
||||
# Isolate from the parent LangGraph run: inherited callbacks would
|
||||
# replay every report token into the chat ``messages`` stream, so the
|
||||
# left-hand bubble and the sandbox would write the same prose at once.
|
||||
# Custom ``report_delta`` events remain the only live preview channel.
|
||||
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
|
||||
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
|
||||
attempts = 2 if operation.startswith("write_report") else 1
|
||||
last_error: Exception | None = None
|
||||
try:
|
||||
for attempt in range(attempts):
|
||||
text_parts.clear()
|
||||
inline_think_filter = InlineThinkTagFilter()
|
||||
try:
|
||||
async for chunk in model.astream(
|
||||
to_langchain_messages(messages),
|
||||
config=isolated_config,
|
||||
**kwargs,
|
||||
):
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
reasoning = _reasoning_delta(chunk)
|
||||
if reasoning and on_reasoning is not None:
|
||||
await on_reasoning(reasoning)
|
||||
# A few OpenAI-compatible models put their reasoning in the
|
||||
# ordinary content channel instead of ``reasoning_content``.
|
||||
# Split it from Markdown even when tags span tokens: callers may
|
||||
# display a live thought trace, but it never joins ``text``.
|
||||
delta, inline_reasoning = inline_think_filter.push_parts(_content_to_text(chunk))
|
||||
if inline_reasoning and on_reasoning is not None:
|
||||
await on_reasoning(inline_reasoning)
|
||||
if delta:
|
||||
text_parts.append(delta)
|
||||
if on_delta is not None:
|
||||
await on_delta(delta)
|
||||
chunk_usage = _extract_usage(chunk)
|
||||
if any(chunk_usage.values()):
|
||||
usage = chunk_usage
|
||||
last_error = None
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001 - report writing waits like ordinary chat
|
||||
from deerflow.agents.deep_research.cancellation import ResearchCancelled
|
||||
|
||||
if isinstance(exc, (ResearchCancelled, asyncio.CancelledError)):
|
||||
raise
|
||||
last_error = exc
|
||||
# Only replay a clean connection; a half-written stream is
|
||||
# reset by the runner, not by appending a second attempt.
|
||||
if text_parts or attempt + 1 >= attempts:
|
||||
raise
|
||||
logger.warning(
|
||||
"deep_research llm: %s stream interrupted before the first token (%s); retrying once",
|
||||
operation,
|
||||
type(exc).__name__,
|
||||
)
|
||||
await asyncio.sleep(1.5)
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
finally:
|
||||
reset_parent_config()
|
||||
|
||||
tail, inline_reasoning_tail = inline_think_filter.finish_parts()
|
||||
if inline_reasoning_tail and on_reasoning is not None:
|
||||
await on_reasoning(inline_reasoning_tail)
|
||||
if tail:
|
||||
text_parts.append(tail)
|
||||
if on_delta is not None:
|
||||
await on_delta(tail)
|
||||
|
||||
await self._record_usage(model_name, operation, usage)
|
||||
return CompletionResult(text="".join(text_parts), model=model_name, usage=usage)
|
||||
|
||||
async def complete_prose(
|
||||
self,
|
||||
*,
|
||||
model_role: ModelRole = "smart",
|
||||
messages: list[dict[str, str]],
|
||||
temperature: float | None = None,
|
||||
max_tokens: int | None = None,
|
||||
operation: str = "unknown",
|
||||
) -> CompletionResult:
|
||||
"""Generate long prose without the provider's inter-chunk timeout.
|
||||
|
||||
This is the report retry path. Some reasoning models pause for longer
|
||||
than the streaming client's 120-second chunk deadline after already
|
||||
producing several chunks. A normal complete response can still finish
|
||||
successfully, so the retry deliberately waits for that full response.
|
||||
"""
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
|
||||
model = self._get_model(model_role)
|
||||
model_name = self._resolve_model_name(model_role) or "default"
|
||||
kwargs: dict[str, Any] = {}
|
||||
if temperature is not None:
|
||||
kwargs["temperature"] = temperature
|
||||
if max_tokens is not None:
|
||||
from deerflow.models.openai_codex_provider import CodexChatModel
|
||||
|
||||
if not isinstance(model, CodexChatModel):
|
||||
kwargs["max_tokens"] = max_tokens
|
||||
|
||||
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
|
||||
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
|
||||
try:
|
||||
response = await model.ainvoke(
|
||||
to_langchain_messages(messages),
|
||||
config=isolated_config,
|
||||
**kwargs,
|
||||
)
|
||||
finally:
|
||||
reset_parent_config()
|
||||
|
||||
if self._cancel is not None:
|
||||
self._cancel.raise_if_cancelled()
|
||||
# ``ainvoke`` returns raw content, including the inline ``<think>``
|
||||
# block some OpenAI-compatible deployments write there. The streamed
|
||||
# path filters it out, so this retry path must not reintroduce it.
|
||||
text = strip_inline_think_tags(_content_to_text(response))
|
||||
usage = _extract_usage(response)
|
||||
await self._record_usage(model_name, operation, usage)
|
||||
return CompletionResult(text=text, model=model_name, usage=usage)
|
||||
|
||||
# ── usage accounting ────────────────────────────────────────────────────
|
||||
|
||||
async def _record_usage(self, model_name: str, operation: str, usage: dict[str, Any]) -> None:
|
||||
"""Accumulate per-model totals + emit a ``model_usage`` event."""
|
||||
bucket = self._usage_totals.setdefault(model_name, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0})
|
||||
for k in bucket:
|
||||
bucket[k] += int(usage.get(k, 0) or 0)
|
||||
|
||||
if self._events is not None:
|
||||
await self._events.emit(
|
||||
"model_usage",
|
||||
payload={
|
||||
"model": model_name,
|
||||
"operation": operation,
|
||||
"usage": usage,
|
||||
},
|
||||
)
|
||||
|
||||
@property
|
||||
def usage_totals(self) -> dict[str, dict[str, int]]:
|
||||
"""Snapshot of cumulative token usage by model (for the final result)."""
|
||||
return {k: dict(v) for k, v in self._usage_totals.items()}
|
||||
|
||||
# ── convenience: the upstream-compatible function family ────────────────
|
||||
#
|
||||
# These mirror ``gpt_researcher/utils/llm.py`` signatures so the vendored
|
||||
# action/skill code needs minimal changes — it calls a context-local bound
|
||||
# method instead of a module global.
|
||||
|
||||
async def create_chat_completion(
|
||||
self,
|
||||
messages: list[dict[str, str]],
|
||||
*,
|
||||
model_role: ModelRole = "smart",
|
||||
temperature: float | None = None,
|
||||
response_format: type[BaseModel] | None = None,
|
||||
stream: bool = False,
|
||||
) -> str:
|
||||
"""Upstream-compatible ``create_chat_completion`` → returns raw text."""
|
||||
if stream and response_format is None:
|
||||
result = await self.stream_complete(
|
||||
model_role=model_role,
|
||||
messages=messages,
|
||||
temperature=temperature,
|
||||
operation="chat_completion",
|
||||
)
|
||||
else:
|
||||
result = await self.complete(
|
||||
model_role=model_role,
|
||||
messages=messages,
|
||||
temperature=temperature,
|
||||
response_schema=response_format,
|
||||
operation="chat_completion",
|
||||
)
|
||||
return result.text
|
||||
|
||||
|
||||
# ── structured-output helpers (no json-repair dependency) ───────────────────
|
||||
|
||||
|
||||
def _schema_instructions(schema: type[BaseModel]) -> str:
|
||||
"""Build a compact "respond as JSON matching this schema" instruction."""
|
||||
props = schema.model_json_schema().get("properties", {})
|
||||
fields = ", ".join(f'"{k}"' for k in props)
|
||||
return f"Respond with ONLY a valid JSON object (no markdown, no prose) containing these keys: {fields}."
|
||||
|
||||
|
||||
def _inject_schema_hint(messages: list[BaseMessage], hint: str) -> list[BaseMessage]:
|
||||
"""Append the schema instruction as a final system message."""
|
||||
from langchain_core.messages import SystemMessage
|
||||
|
||||
return [*messages, SystemMessage(content=hint)]
|
||||
|
||||
|
||||
def _inject_repair_hint(messages: list[BaseMessage], bad_text: str, schema: type[BaseModel]) -> list[BaseMessage]:
|
||||
"""Re-prompt: 'your last reply was invalid JSON, fix it'."""
|
||||
from langchain_core.messages import HumanMessage
|
||||
|
||||
return [
|
||||
*messages,
|
||||
HumanMessage(content=(f"Your previous reply was not valid JSON and could not be parsed:\n{bad_text[:500]}\n\nPlease respond again with ONLY a valid JSON object.")),
|
||||
]
|
||||
|
||||
|
||||
def parse_structured_text(text: str, schema: type[BaseModel]) -> BaseModel | None:
|
||||
"""Parse a completed (possibly streamed) model output against a schema.
|
||||
|
||||
Public wrapper over the same fence-stripping, blob-extracting parse used
|
||||
by :meth:`DeerFlowCompletionBackend.complete`, so runners that collect the
|
||||
text through :meth:`stream_complete` can validate it afterwards.
|
||||
"""
|
||||
return _try_parse_structured(text, schema)
|
||||
|
||||
|
||||
def _try_parse_structured(text: str, schema: type[BaseModel]) -> BaseModel | None:
|
||||
"""Strip markdown fences + attempt ``schema.model_validate_json``."""
|
||||
cleaned = _strip_code_fences(text).strip()
|
||||
if not cleaned:
|
||||
return None
|
||||
try:
|
||||
return schema.model_validate_json(cleaned)
|
||||
except Exception:
|
||||
# Try to locate the first {...} blob as a last resort.
|
||||
start = cleaned.find("{")
|
||||
end = cleaned.rfind("}")
|
||||
if start != -1 and end > start:
|
||||
try:
|
||||
return schema.model_validate_json(cleaned[start : end + 1])
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _strip_code_fences(text: str) -> str:
|
||||
"""Remove ```json ... ``` / ``` ... ``` wrapping if present."""
|
||||
stripped = text.strip()
|
||||
if stripped.startswith("```"):
|
||||
# Drop first line (the fence + optional language tag).
|
||||
lines = stripped.splitlines()
|
||||
if len(lines) >= 2:
|
||||
lines = lines[1:]
|
||||
if lines and lines[-1].strip().startswith("```"):
|
||||
lines = lines[:-1]
|
||||
return "\n".join(lines)
|
||||
return stripped
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CompletionError",
|
||||
"DeerFlowCompletionBackend",
|
||||
"parse_structured_text",
|
||||
"to_langchain_messages",
|
||||
]
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
"""Material provider — DeerFlow's unified information-collection boundary.
|
||||
|
||||
The vendored GPT Researcher code normally fires off its own retrievers + scrapers
|
||||
per source URL. We replace that entirely: the :class:`MaterialProvider` returns
|
||||
**fully-fetched** :class:`ResearchMaterial` objects (``raw_content`` already
|
||||
populated), so the vendored scraper/BrowserManager is never invoked (§6.4).
|
||||
|
||||
First version (Phase 1) wraps :func:`deerflow.community.configurable_search.execute_search`:
|
||||
|
||||
1. Run the configured ``web_search`` tool for the query.
|
||||
2. Map normalised results (``recUuid / content / title / url / source / time``)
|
||||
→ :class:`ResearchMaterial`.
|
||||
3. If ``content`` is already full text → use as ``raw_content`` (no fetch needed).
|
||||
4. If only a snippet → a DeerFlow fetch tool backfills (Phase 2).
|
||||
5. Dedupe by ``rec_uuid`` → canonical URL → ``content_hash``.
|
||||
|
||||
The provider never imports ``app.*`` and never does raw ``requests.get``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING, Protocol, runtime_checkable
|
||||
|
||||
from deerflow.agents.deep_research.types import ResearchMaterial
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from deerflow.agents.deep_research.cancellation import CancellationToken
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class MaterialProvider(Protocol):
|
||||
"""Protocol every material channel implements (§8.3)."""
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
domains: list[str] | None = None,
|
||||
limit: int = 8,
|
||||
user_id: str = "",
|
||||
session_id: str = "",
|
||||
cancellation: CancellationToken | None = None,
|
||||
) -> list[ResearchMaterial]:
|
||||
"""Return fully-fetched materials for ``query``.
|
||||
|
||||
Implementations guarantee ``raw_content`` is non-empty for every item
|
||||
that has fetchable text (the vendored writer relies on this).
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
def dedupe_materials(materials: list[ResearchMaterial]) -> list[ResearchMaterial]:
|
||||
"""Stable dedup by rec_uuid → canonical url → content_hash (§8.3 §6).
|
||||
|
||||
Keeps the first occurrence; later duplicates are dropped (but their queries
|
||||
are noted on the survivor's metadata so multi-query hits aren't lost).
|
||||
"""
|
||||
seen: dict[str, int] = {} # key → index in result
|
||||
out: list[ResearchMaterial] = []
|
||||
for m in materials:
|
||||
key = (m.rec_uuid and f"rec:{m.rec_uuid}") or (m.url and f"url:{m.url}") or m.compute_hash()
|
||||
if key in seen:
|
||||
idx = seen[key]
|
||||
# Record that another query also surfaced this material.
|
||||
surviving = out[idx]
|
||||
extra_queries = surviving.metadata.setdefault("_also_matched_queries", [])
|
||||
if m.query and m.query not in extra_queries:
|
||||
extra_queries.append(m.query)
|
||||
continue
|
||||
seen[key] = len(out)
|
||||
out.append(m)
|
||||
return out
|
||||
|
||||
|
||||
__all__ = ["MaterialProvider", "dedupe_materials"]
|
||||
+601
@@ -0,0 +1,601 @@
|
||||
"""Deep Research material provider backed by DeerFlow's Q&A retrieval stack.
|
||||
|
||||
This is the only place adapted GPT Researcher runners retrieve materials. It
|
||||
uses the same hierarchy as DeerFlow Q&A instead of treating the legacy
|
||||
configurable ``web_search`` endpoint as the only possible source:
|
||||
|
||||
1. Run enabled Q&A research skills through the existing ``SubagentExecutor``
|
||||
runtime when their required search tool is registered.
|
||||
2. Use a configured ``web_search`` provider when the deployment has one.
|
||||
3. Fall back to DeerFlow's existing keyless DuckDuckGo search provider.
|
||||
|
||||
The fallback is important for this offline deployment: its configurable search
|
||||
entry is deliberately disabled while it still contains a placeholder intranet
|
||||
endpoint. A missing endpoint must never turn every Deep Research report into
|
||||
"no materials collected".
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from deerflow.agents.deep_research.adapters.material import dedupe_materials
|
||||
from deerflow.agents.deep_research.types import ResearchMaterial
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Treat shorter content as a snippet and try a full-text fetch when available.
|
||||
SNIPPET_THRESHOLD = 300
|
||||
ONLINE_SEARCH_TIMEOUT_SECONDS = 45
|
||||
SECONDARY_ONLINE_SEARCH_TIMEOUT_SECONDS = 20
|
||||
|
||||
# Same skills that guide the Q&A assistant's public-web / knowledge retrieval.
|
||||
# The order deliberately prefers the multi-round skill for a Deep Research job.
|
||||
QUESTION_ANSWER_RESEARCH_SKILLS: tuple[str, ...] = (
|
||||
"deep-search",
|
||||
"web-research",
|
||||
"weknora-knowledge-search",
|
||||
)
|
||||
|
||||
# A skill is run only when the Q&A runtime exposes the search tool it needs.
|
||||
# This prevents a disabled placeholder web_search config from launching an
|
||||
# agent which is incapable of searching and can only wait for its timeout.
|
||||
_SKILL_REQUIRED_TOOLS: dict[str, set[str]] = {
|
||||
"deep-search": {"web_search"},
|
||||
"web-research": {"web_search"},
|
||||
"weknora-knowledge-search": {"weknora_search"},
|
||||
}
|
||||
|
||||
|
||||
class DeerFlowMaterialProvider:
|
||||
"""MaterialProvider backed by the Q&A skills and online retrieval stack.
|
||||
|
||||
A configurable ``web_search`` endpoint remains supported, but it is no
|
||||
longer a single point of failure. All returned materials preserve the
|
||||
original source, URL and retrieval channel for citations and auditing.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
user_id: str = "",
|
||||
session_id: str = "",
|
||||
runtime_thread_id: str = "",
|
||||
model_name: str | None = None,
|
||||
fetch_enabled: bool = True,
|
||||
retrieval_skills: list[str] | None = None,
|
||||
) -> None:
|
||||
self._user_id = user_id
|
||||
self._session_id = session_id
|
||||
self._runtime_thread_id = runtime_thread_id
|
||||
self._model_name = model_name
|
||||
self._fetch_enabled = fetch_enabled
|
||||
self._retrieval_skills = _normalize_named_skills(retrieval_skills)
|
||||
self._settings = None
|
||||
self._settings_error: str | None = None
|
||||
self._question_answer_tool_names = _load_question_answer_tool_names(model_name)
|
||||
self._research_skills = _load_question_answer_research_skills()
|
||||
self._online_search_available = importlib.util.find_spec("ddgs") is not None
|
||||
self._online_search_semaphore: asyncio.Semaphore | None = None
|
||||
try:
|
||||
from deerflow.community.configurable_search.tools import ConfigurableSearchSettings
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
cfg = get_app_config().get_tool_config("web_search")
|
||||
self._settings = ConfigurableSearchSettings.from_tool_config(cfg)
|
||||
if not self._settings.enabled:
|
||||
self._settings_error = "web_search tool is disabled in config"
|
||||
self._settings = None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self._settings_error = f"web_search not configured: {exc}"
|
||||
logger.warning("Deep Research configurable web_search unavailable: %s", self._settings_error)
|
||||
|
||||
async def search(
|
||||
self,
|
||||
query: str,
|
||||
*,
|
||||
domains: list[str] | None = None,
|
||||
limit: int = 8,
|
||||
user_id: str = "",
|
||||
session_id: str = "",
|
||||
cancellation=None,
|
||||
) -> list[ResearchMaterial]:
|
||||
"""Collect actual Q&A-compatible materials for one research query."""
|
||||
if cancellation is not None:
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
materials: list[ResearchMaterial] = []
|
||||
seen_keys: set[str] = set()
|
||||
directed_skills = bool(self._retrieval_skills)
|
||||
|
||||
# 1. The genuine Q&A skill runtime. Skills may call web, MCP or another
|
||||
# deployment-specific source according to their SKILL.md instructions.
|
||||
# A non-empty 检索来源 list replaces the default research-skill trio.
|
||||
skill_results = await self._search_via_question_answer_skills(query, limit=limit)
|
||||
materials.extend(
|
||||
self._normalise_search_results(
|
||||
skill_results,
|
||||
query=query,
|
||||
channel="skill",
|
||||
domains=domains,
|
||||
seen_keys=seen_keys,
|
||||
)
|
||||
)
|
||||
if cancellation is not None:
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
# Specified skills are the retrieval source: do not also dump generic
|
||||
# web_search / DDG on top. Empty 检索来源 keeps the original fallbacks.
|
||||
if directed_skills:
|
||||
hydrated: list[ResearchMaterial] = []
|
||||
for material in materials[:limit]:
|
||||
if cancellation is not None:
|
||||
cancellation.raise_if_cancelled()
|
||||
if material.url and len(material.raw_content) < SNIPPET_THRESHOLD and self._fetch_enabled:
|
||||
fetched = await asyncio.to_thread(_safe_fetch, material.url)
|
||||
if fetched:
|
||||
material.raw_content = fetched
|
||||
if material.raw_content:
|
||||
hydrated.append(material)
|
||||
return dedupe_materials(hydrated)[:limit]
|
||||
|
||||
# 2. Retain compatibility with deployed configurable search providers.
|
||||
if len(materials) < limit:
|
||||
configured_results = await self._search_configured_web(query)
|
||||
materials.extend(
|
||||
self._normalise_configured_results(
|
||||
configured_results,
|
||||
query=query,
|
||||
domains=domains,
|
||||
seen_keys=seen_keys,
|
||||
)
|
||||
)
|
||||
if cancellation is not None:
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
# 3. Reuse DeerFlow's maintained online provider if the Q&A search tool
|
||||
# is absent or the configured provider returned too few materials.
|
||||
if len(materials) < limit:
|
||||
online_results = await self._search_via_online_fallback(query, limit=limit - len(materials))
|
||||
materials.extend(
|
||||
self._normalise_search_results(
|
||||
online_results,
|
||||
query=query,
|
||||
channel="web_search",
|
||||
domains=domains,
|
||||
seen_keys=seen_keys,
|
||||
)
|
||||
)
|
||||
|
||||
hydrated: list[ResearchMaterial] = []
|
||||
for material in materials[:limit]:
|
||||
if cancellation is not None:
|
||||
cancellation.raise_if_cancelled()
|
||||
if material.url and len(material.raw_content) < SNIPPET_THRESHOLD and self._fetch_enabled:
|
||||
fetched = await asyncio.to_thread(_safe_fetch, material.url)
|
||||
if fetched:
|
||||
material.raw_content = fetched
|
||||
if material.raw_content:
|
||||
hydrated.append(material)
|
||||
return dedupe_materials(hydrated)[:limit]
|
||||
|
||||
async def _search_via_question_answer_skills(self, query: str, *, limit: int) -> list[Any]:
|
||||
"""Run eligible research skills through the same runtime as Q&A."""
|
||||
runnable = self._runnable_research_skills()
|
||||
if not runnable:
|
||||
logger.info(
|
||||
"Deep Research Q&A skills are enabled but their required search tools are unavailable; "
|
||||
"using the online fallback"
|
||||
)
|
||||
return []
|
||||
|
||||
try:
|
||||
from deerflow.agents.ai_writing.nodes._skill_agent import run_skill_via_real_agent
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research could not load the Q&A skill runtime: %s", exc)
|
||||
return []
|
||||
|
||||
run_config = {
|
||||
"configurable": {
|
||||
"thread_id": self._runtime_thread_id,
|
||||
"model_name": self._model_name or "",
|
||||
}
|
||||
}
|
||||
state = {"model_name": self._model_name or ""}
|
||||
collected: list[Any] = []
|
||||
directed = bool(self._retrieval_skills)
|
||||
for skill in runnable:
|
||||
try:
|
||||
results, error = await run_skill_via_real_agent(
|
||||
skill=skill,
|
||||
intent=f"Collect verifiable research materials for: {query}",
|
||||
keywords=[query],
|
||||
user_query=query,
|
||||
config=run_config,
|
||||
state=state,
|
||||
model_name_fallback=self._model_name,
|
||||
max_results=limit,
|
||||
timeout_s=ONLINE_SEARCH_TIMEOUT_SECONDS,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research Q&A skill %s failed: %s", skill.name, exc)
|
||||
continue
|
||||
if results:
|
||||
collected.extend(list(results)[:limit])
|
||||
if not directed or len(collected) >= limit:
|
||||
return collected[:limit]
|
||||
if error:
|
||||
logger.info("Deep Research Q&A skill %s returned no materials: %s", skill.name, error)
|
||||
return collected[:limit]
|
||||
|
||||
def _runnable_research_skills(self) -> list[Any]:
|
||||
"""Default Q&A trio, or the report-structure 检索来源 when named."""
|
||||
skills = (
|
||||
_load_named_research_skills(self._retrieval_skills)
|
||||
if self._retrieval_skills
|
||||
else self._research_skills
|
||||
)
|
||||
if self._retrieval_skills:
|
||||
# User-named skills may use any tool; skip the trio's required-tool gate.
|
||||
return list(skills)
|
||||
return [
|
||||
skill
|
||||
for skill in skills
|
||||
if _SKILL_REQUIRED_TOOLS.get(skill.name, set()) <= self._question_answer_tool_names
|
||||
]
|
||||
|
||||
async def _search_configured_web(self, query: str) -> list[dict[str, Any]]:
|
||||
"""Run the existing configurable ``web_search`` adapter when enabled."""
|
||||
if self._settings is None:
|
||||
return []
|
||||
try:
|
||||
from deerflow.community.configurable_search.tools import execute_search
|
||||
|
||||
raw = await execute_search(query, self._settings)
|
||||
envelope = json.loads(raw)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research configured web_search failed: %s", exc)
|
||||
return []
|
||||
if not isinstance(envelope, dict) or envelope.get("error"):
|
||||
logger.info("Deep Research configured web_search returned no materials: %s", envelope)
|
||||
return []
|
||||
return [item for item in envelope.get("results") or [] if isinstance(item, dict)]
|
||||
|
||||
async def _search_via_online_fallback(self, query: str, *, limit: int) -> list[Any]:
|
||||
"""Use maintained DDG retrieval, then a relevant-result web fallback."""
|
||||
if not self._online_search_available or limit < 1:
|
||||
return []
|
||||
if self._online_search_semaphore is None:
|
||||
self._online_search_semaphore = asyncio.Semaphore(2)
|
||||
try:
|
||||
from deerflow.agents.ai_writing.search.duckduckgo_search import DuckDuckGoSearchProvider
|
||||
|
||||
async with self._online_search_semaphore:
|
||||
provider = DuckDuckGoSearchProvider(max_timeout=ONLINE_SEARCH_TIMEOUT_SECONDS)
|
||||
ddg_results = await asyncio.wait_for(
|
||||
provider.search(query, max_results=limit),
|
||||
timeout=ONLINE_SEARCH_TIMEOUT_SECONDS + 5,
|
||||
)
|
||||
relevant_results = _filter_relevant_search_results(ddg_results, query)
|
||||
if relevant_results:
|
||||
return relevant_results[:limit]
|
||||
# DDGS can occasionally return generic login/tracking pages for
|
||||
# Chinese queries. Do not turn those into report citations;
|
||||
# collect a second set from the search HTML instead.
|
||||
return await asyncio.to_thread(_search_360_web, query, limit)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research online fallback search failed for %r: %s", query, exc)
|
||||
return []
|
||||
|
||||
def _normalise_configured_results(
|
||||
self,
|
||||
results: list[dict[str, Any]],
|
||||
*,
|
||||
query: str,
|
||||
domains: list[str] | None,
|
||||
seen_keys: set[str],
|
||||
) -> list[ResearchMaterial]:
|
||||
materials: list[ResearchMaterial] = []
|
||||
for item in results:
|
||||
material = result_to_material(item, query=query)
|
||||
if not _material_matches_domains(material, domains) or not _accept_material(material, seen_keys):
|
||||
continue
|
||||
material.metadata["retrieval_channel"] = "configured_web_search"
|
||||
materials.append(material)
|
||||
return materials
|
||||
|
||||
def _normalise_search_results(
|
||||
self,
|
||||
results: list[Any],
|
||||
*,
|
||||
query: str,
|
||||
channel: str,
|
||||
domains: list[str] | None,
|
||||
seen_keys: set[str],
|
||||
) -> list[ResearchMaterial]:
|
||||
materials: list[ResearchMaterial] = []
|
||||
for result in results:
|
||||
material = _search_result_to_material(result, query=query, channel=channel)
|
||||
if material is None or not _material_matches_domains(material, domains) or not _accept_material(material, seen_keys):
|
||||
continue
|
||||
materials.append(material)
|
||||
return materials
|
||||
|
||||
@property
|
||||
def is_configured(self) -> bool:
|
||||
"""Whether at least one material channel can run."""
|
||||
return bool(self.material_channels)
|
||||
|
||||
@property
|
||||
def material_channels(self) -> list[str]:
|
||||
"""Advertise active Q&A-compatible retrieval channels to the frontend."""
|
||||
channels: list[str] = []
|
||||
if any(
|
||||
_SKILL_REQUIRED_TOOLS.get(skill.name, set()) <= self._question_answer_tool_names
|
||||
for skill in self._research_skills
|
||||
):
|
||||
channels.append("skill")
|
||||
if self._settings is not None or self._online_search_available:
|
||||
channels.append("web_search")
|
||||
return channels
|
||||
|
||||
@property
|
||||
def settings_error(self) -> str | None:
|
||||
return self._settings_error
|
||||
|
||||
|
||||
def result_to_material(item: dict[str, Any], *, query: str) -> ResearchMaterial:
|
||||
"""Map a configurable-search result item into a research material.
|
||||
|
||||
Public so the chat-collection thread harvester can reuse the exact same
|
||||
field mapping for ``web_search`` tool envelopes harvested off a thread.
|
||||
"""
|
||||
rec_uuid = str(item.get("recUuid") or "").strip() or None
|
||||
url = str(item.get("url") or "").strip() or None
|
||||
title = str(item.get("title") or "").strip()
|
||||
content = str(item.get("content") or "").strip()
|
||||
source = str(item.get("source") or "").strip() or "web"
|
||||
raw_hash = hashlib.sha256((url or rec_uuid or content).encode("utf-8")).hexdigest()[:32]
|
||||
published_at = _try_parse_datetime(str(item.get("publish_time") or item.get("time") or "").strip())
|
||||
return ResearchMaterial(
|
||||
id=f"src_{raw_hash[:16]}",
|
||||
query=query,
|
||||
title=title,
|
||||
url=url,
|
||||
raw_content=content,
|
||||
snippet=content[:200] if content else None,
|
||||
source=source,
|
||||
source_type="web",
|
||||
published_at=published_at,
|
||||
rec_uuid=rec_uuid,
|
||||
content_hash=raw_hash,
|
||||
)
|
||||
|
||||
|
||||
def _search_result_to_material(result: Any, *, query: str, channel: str) -> ResearchMaterial | None:
|
||||
"""Map a Q&A/AI-writing SearchResult-like object into ResearchMaterial."""
|
||||
title = str(getattr(result, "title", "") or "").strip()
|
||||
content = str(getattr(result, "content", "") or "").strip()
|
||||
if not title and not content:
|
||||
return None
|
||||
url = str(getattr(result, "url", "") or "").strip() or None
|
||||
rec_uuid = str(getattr(result, "recUuid", "") or "").strip() or None
|
||||
source = str(getattr(result, "source", "") or "").strip() or channel
|
||||
source_type = str(getattr(result, "source_type", "") or "").strip()
|
||||
if source_type not in {"web", "knowledge", "skill", "mcp", "file", "other"}:
|
||||
source_type = "skill" if channel == "skill" else "web"
|
||||
raw_hash = hashlib.sha256((url or rec_uuid or title or content).encode("utf-8")).hexdigest()[:32]
|
||||
return ResearchMaterial(
|
||||
id=f"src_{raw_hash[:16]}",
|
||||
query=query,
|
||||
title=title,
|
||||
url=url,
|
||||
raw_content=content,
|
||||
snippet=content[:200] if content else None,
|
||||
source=source,
|
||||
source_type=source_type,
|
||||
published_at=_try_parse_datetime(str(getattr(result, "published_at", "") or "").strip()),
|
||||
relevance_score=_safe_float(getattr(result, "relevance_score", None)),
|
||||
rec_uuid=rec_uuid,
|
||||
content_hash=raw_hash,
|
||||
metadata={"retrieval_channel": channel},
|
||||
)
|
||||
|
||||
|
||||
def _load_question_answer_tool_names(model_name: str | None) -> set[str]:
|
||||
"""Return exactly the tool names the Q&A agent can currently call."""
|
||||
try:
|
||||
from deerflow.tools import get_available_tools
|
||||
|
||||
return {str(tool.name) for tool in get_available_tools(model_name=model_name, include_mcp=True)}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research failed to inspect Q&A tools: %s", exc)
|
||||
return set()
|
||||
|
||||
|
||||
def _load_question_answer_research_skills() -> list[Any]:
|
||||
"""Load only enabled Q&A research skills, preserving normal skill state."""
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
by_name = {skill.name: skill for skill in get_or_new_skill_storage().load_skills(enabled_only=True)}
|
||||
return [by_name[name] for name in QUESTION_ANSWER_RESEARCH_SKILLS if name in by_name]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research failed to load Q&A research skills: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
def _load_named_research_skills(names: list[str]) -> list[Any]:
|
||||
"""Load enabled skills by name, preserving the requested order."""
|
||||
if not names:
|
||||
return []
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
|
||||
by_name = {skill.name: skill for skill in get_or_new_skill_storage().load_skills(enabled_only=True)}
|
||||
return [by_name[name] for name in names if name in by_name]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research failed to load named retrieval skills: %s", exc)
|
||||
return []
|
||||
|
||||
|
||||
def _normalize_named_skills(value: list[str] | None) -> list[str]:
|
||||
from deerflow.persistence.report_structures.skills import normalize_retrieval_skills
|
||||
|
||||
return normalize_retrieval_skills(value)
|
||||
|
||||
|
||||
def _material_matches_domains(material: ResearchMaterial, domains: list[str] | None) -> bool:
|
||||
"""Apply the requested domain allow-list without dropping URL-less KB hits."""
|
||||
if not domains or not material.url:
|
||||
return True
|
||||
host = (urlparse(material.url).hostname or "").lower()
|
||||
allowed = {str(domain).lower().strip().lstrip(".") for domain in domains if str(domain).strip()}
|
||||
return not allowed or any(host == domain or host.endswith(f".{domain}") for domain in allowed)
|
||||
|
||||
|
||||
def _accept_material(material: ResearchMaterial, seen_keys: set[str]) -> bool:
|
||||
"""Deduplicate cross-channel hits before optional fetch backfill."""
|
||||
key = material.url or material.rec_uuid or material.content_hash
|
||||
if not key or key in seen_keys:
|
||||
return False
|
||||
seen_keys.add(key)
|
||||
return True
|
||||
|
||||
|
||||
def _safe_float(value: Any) -> float | None:
|
||||
try:
|
||||
return float(value) if value is not None else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _filter_relevant_search_results(results: list[Any], query: str) -> list[Any]:
|
||||
"""Discard obvious search-engine noise before it reaches report citations."""
|
||||
tokens = _query_relevance_tokens(query)
|
||||
if not tokens:
|
||||
return list(results)
|
||||
kept: list[Any] = []
|
||||
for result in results:
|
||||
haystack = " ".join(
|
||||
[
|
||||
str(getattr(result, "title", "") or ""),
|
||||
str(getattr(result, "content", "") or ""),
|
||||
str(getattr(result, "url", "") or ""),
|
||||
]
|
||||
).lower()
|
||||
if any(token in haystack for token in tokens):
|
||||
kept.append(result)
|
||||
return kept
|
||||
|
||||
|
||||
def _query_relevance_tokens(query: str) -> set[str]:
|
||||
"""Build compact English/CJK tokens suitable for rejecting blank noise."""
|
||||
tokens = {token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", query)}
|
||||
for chunk in re.findall(r"[\u4e00-\u9fff]+", query):
|
||||
for width in (4, 3, 2):
|
||||
tokens.update(chunk[index : index + width] for index in range(max(0, len(chunk) - width + 1)))
|
||||
return tokens
|
||||
|
||||
|
||||
def _search_360_web(query: str, limit: int) -> list[Any]:
|
||||
"""Return parsed, real 360 Search snippets when DDG returns no relevant hit."""
|
||||
try:
|
||||
import httpx
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from deerflow.agents.ai_writing.search.base import SearchResult
|
||||
|
||||
response = httpx.get(
|
||||
"https://www.so.com/s",
|
||||
params={"q": query},
|
||||
headers={"User-Agent": "Mozilla/5.0 (compatible; DeerFlow/1.0)"},
|
||||
timeout=SECONDARY_ONLINE_SEARCH_TIMEOUT_SECONDS,
|
||||
follow_redirects=True,
|
||||
)
|
||||
response.raise_for_status()
|
||||
soup = BeautifulSoup(response.text, "html.parser")
|
||||
rows: list[Any] = []
|
||||
seen_urls: set[str] = set()
|
||||
for card in soup.select("li.res-list"):
|
||||
anchor = card.select_one("h3 a") or card.select_one("a.res-link")
|
||||
if anchor is None:
|
||||
continue
|
||||
title = anchor.get_text(" ", strip=True)
|
||||
url = str(anchor.get("href") or "").strip()
|
||||
content_node = card.select_one(".res-desc") or card.select_one(".res-rich") or card
|
||||
content = content_node.get_text(" ", strip=True)
|
||||
if not title or not content or not url or url in seen_urls:
|
||||
continue
|
||||
seen_urls.add(url)
|
||||
rows.append(
|
||||
SearchResult(
|
||||
title=title,
|
||||
content=content[:500],
|
||||
source=urlparse(url).hostname or "360 Search",
|
||||
published_at="",
|
||||
url=url,
|
||||
relevance_score=0.7,
|
||||
source_type="internet",
|
||||
)
|
||||
)
|
||||
if len(rows) >= limit:
|
||||
break
|
||||
return _filter_relevant_search_results(rows, query)[:limit]
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("Deep Research 360 fallback search failed for %r: %s", query, exc)
|
||||
return []
|
||||
|
||||
|
||||
def _try_parse_datetime(value: str) -> datetime | None:
|
||||
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%dT%H:%M:%S"):
|
||||
try:
|
||||
return datetime.strptime(value[:19], fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _safe_fetch(url: str) -> str:
|
||||
"""Best-effort full-text fetch via the configured DeerFlow web_fetch tool."""
|
||||
try:
|
||||
from deerflow.config import get_app_config
|
||||
|
||||
fetch_cfg = get_app_config().get_tool_config("web_fetch")
|
||||
if fetch_cfg is None or not fetch_cfg.enabled:
|
||||
return ""
|
||||
use = fetch_cfg.use or ""
|
||||
html = ""
|
||||
if "jina_ai" in use:
|
||||
from deerflow.community.jina_ai.jina_client import JinaClient
|
||||
|
||||
html = JinaClient().crawl(url, return_format="html", timeout=10) or ""
|
||||
elif "infoquest" in use:
|
||||
from deerflow.community.infoquest.infoquest_client import InfoQuestClient
|
||||
|
||||
html = InfoQuestClient().fetch(url, return_format="html") or ""
|
||||
if not html or html.startswith("Error"):
|
||||
return ""
|
||||
from deerflow.utils.readability import ReadabilityExtractor
|
||||
|
||||
return ReadabilityExtractor().extract_article(html).to_markdown(including_title=True) or ""
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.debug("Deep Research material fetch failed for %s: %s", url, exc)
|
||||
return ""
|
||||
|
||||
|
||||
__all__ = [
|
||||
"DeerFlowMaterialProvider",
|
||||
"ONLINE_SEARCH_TIMEOUT_SECONDS",
|
||||
"QUESTION_ANSWER_RESEARCH_SKILLS",
|
||||
"SNIPPET_THRESHOLD",
|
||||
"result_to_material",
|
||||
]
|
||||
Reference in new issue
Block a user