初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,19 @@
"""Deep Research adapters — the four-boundary isolation layer.
Adapters are the *only* place the vendored GPT Researcher algorithms touch
DeerFlow infrastructure. Each adapter wraps one concern:
================ ===========================================================
``llm`` model calls → ``create_chat_model()`` (no upstream Provider)
``material`` search/fetch → DeerFlow ``web_search`` + knowledge + MCP
``context`` context compression (no legacy text-splitter deps)
``event_sink`` progress events → ``deep_research_events`` + SSE
``artifacts`` report/export writes → hidden runtime-thread outputs
``image`` image generation (optional, Phase 5)
================ ===========================================================
Protocols live in each module alongside (or ahead of) their concrete impl, so
tests can inject fakes that satisfy the protocol without importing infra.
"""
from __future__ import annotations
@@ -0,0 +1,159 @@
"""Artifact writer — writes report/export files into the session's hidden thread.
Each deep-research session owns a *hidden* DeerFlow runtime thread
(``metadata.thread_type=deep_research``, ``hidden=true``). Report artifacts
(``report.md``, ``report.html``, ``sources.json``, …) are written into that
thread's sandbox ``outputs/`` directory and downloaded through the existing
``/api/threads/{thread_id}/artifacts`` endpoint (which already has path-safety
guards + owner checks — §13.6, §18.3).
The writer is a thin protocol so the engine/runner never touches the filesystem
layout directly.
"""
from __future__ import annotations
import re
from pathlib import Path
from typing import Protocol, runtime_checkable
@runtime_checkable
class ArtifactWriter(Protocol):
"""Protocol for writing research artifacts (§13.6 §18)."""
@property
def thread_id(self) -> str:
"""The hidden runtime thread that owns these artifacts."""
...
async def write_text(self, name: str, content: str) -> str:
"""Write a text artifact (e.g. ``report.md``) and return its virtual path."""
...
async def write_bytes(self, name: str, content: bytes) -> str:
"""Write a binary artifact (e.g. ``report.pdf``). Optional capability."""
...
def allowed_names(self) -> tuple[str, ...]:
"""The server-enumerated filename whitelist (§18.3 — no arbitrary paths)."""
...
# Server-enumerated artifact filename whitelist (§18.3, §13.6).
# The writer rejects any name not in this set to prevent path traversal.
ARTIFACT_NAMES: tuple[str, ...] = (
"report.md",
"report.html",
"sources.json",
"research-plan.json",
"report.pdf", # optional capability
"report.docx", # optional capability
)
# Image artifact names are server-generated only. Keeping the numerical range
# small makes the whitelist finite in practice while still letting the report
# writer select the matching format returned by the configured provider.
_RESEARCH_IMAGE_NAME_RE = re.compile(r"research-image-0[1-4]\.(?:png|jpe?g|webp)$")
def is_allowed_artifact_name(name: str) -> bool:
"""Return whether *name* is one of the server-controlled research outputs."""
return name in ARTIFACT_NAMES or bool(_RESEARCH_IMAGE_NAME_RE.fullmatch(name))
def resolve_research_artifact_path(
thread_id: str,
name: str,
*,
user_id: str | None = None,
) -> Path:
"""Resolve a whitelisted research artifact into its thread outputs dir.
``Paths.resolve_virtual_path`` intentionally accepts only paths that an
in-sandbox agent could see. Research code deals in server-generated file
names, so passing a bare name (for example ``report.md``) would fail that
guard on every host, including native Windows deployments. Keep the name
allowlist and construct the canonical sandbox path here, at the boundary.
"""
if not is_allowed_artifact_name(name):
raise ValueError(f"Artifact name {name!r} is not in the allowed research-artifact whitelist")
from deerflow.config.paths import VIRTUAL_PATH_PREFIX, get_paths
return get_paths().resolve_virtual_path(
thread_id,
f"{VIRTUAL_PATH_PREFIX}/outputs/{name}",
user_id=user_id,
)
class ThreadOutputsArtifactWriter:
"""Concrete writer backed by a DeerFlow thread's sandbox ``outputs/`` dir.
The actual path resolution reuses
:func:`deerflow.config.paths.get_paths().sandbox_outputs_dir` (the same
guard ``app/gateway/routers/artifacts.py`` uses), so traversal/symlink
protection is inherited. File IO is offloaded to a thread so the event
loop is never blocked (§25.6).
"""
def __init__(self, thread_id: str, *, user_id: str | None = None) -> None:
self._thread_id = thread_id
self._user_id = user_id
@property
def thread_id(self) -> str:
return self._thread_id
def allowed_names(self) -> tuple[str, ...]:
return ARTIFACT_NAMES
async def write_text(self, name: str, content: str) -> str:
self._validate_name(name)
import asyncio
path = await asyncio.to_thread(self._resolve_path, name)
await asyncio.to_thread(self._write_file, path, content.encode("utf-8"))
return name
async def write_bytes(self, name: str, content: bytes) -> str:
self._validate_name(name)
import asyncio
path = await asyncio.to_thread(self._resolve_path, name)
await asyncio.to_thread(self._write_file, path, content)
return name
# ── internals ───────────────────────────────────────────────────────────
def _validate_name(self, name: str) -> None:
if not is_allowed_artifact_name(name):
raise ValueError(
f"Artifact name {name!r} is not in the allowed research-artifact whitelist"
)
def _resolve_path(self, name: str):
# Ensure the outputs dir exists; the paths module rejects traversal and
# the name is whitelisted above, so the canonical virtual path is safe.
from deerflow.config.paths import get_paths
get_paths().sandbox_outputs_dir(self._thread_id, user_id=self._user_id)
return resolve_research_artifact_path(
self._thread_id,
name,
user_id=self._user_id,
)
def _write_file(self, path, data: bytes) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
__all__ = [
"ARTIFACT_NAMES",
"ArtifactWriter",
"ThreadOutputsArtifactWriter",
"is_allowed_artifact_name",
"resolve_research_artifact_path",
]
@@ -0,0 +1,130 @@
"""Context compressor — DeerFlow replacement for upstream's legacy compressor.
Upstream's ``ContextManager`` / ``compression`` module depends on
``langchain_classic`` and ``langchain_text_splitters``, which DeerFlow does not
declare. Rather than pull in those legacy packages, we keep the *calling
interface* (§10) and provide :class:`DeerFlowContextCompressor` with a
self-contained algorithm:
1. Chunk by title/paragraph/char-limit without breaking Markdown code blocks.
2. Each chunk retains its source id (no cross-source amalgamation).
3. If embeddings are configured → similarity pre-filter; else keyword/BM25-style.
4. Over budget → per-source summary via ``fast_model`` (Phase 2).
5. Final ranking balances source diversity, recency, relevance.
Phase 1 ships the chunking + keyword pre-filter (no embedding dependency);
the ``fast_model`` summarisation path is wired but only triggers when the
context genuinely exceeds ``max_context_words``.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
if TYPE_CHECKING:
from deerflow.agents.deep_research.adapters.llm import DeerFlowCompletionBackend
from deerflow.agents.deep_research.types import ResearchMaterial
@runtime_checkable
class ContextCompressor(Protocol):
"""The interface vendored ``ContextManager`` is rewired to call (§10)."""
async def async_get_context(
self,
query: str,
max_results: int = 10,
cost_callback: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Return the compressed context list (each item = one material's context dict).
Output shape matches :meth:`ResearchMaterial.to_context_dict` so the
vendored writer sees what it expects.
"""
...
class DeerFlowContextCompressor:
"""Concrete compressor: chunk → keyword-filter → optional summary.
Holds a reference to the current material set (set by the runner before
the vendored research loop calls ``async_get_context``) and the LLM backend
(for the summary path).
"""
def __init__(
self,
*,
llm: DeerFlowCompletionBackend | None = None,
max_context_words: int = 24_000,
) -> None:
self._llm = llm
self._max_words = max_context_words
self._materials: list[ResearchMaterial] = []
def set_materials(self, materials: list[ResearchMaterial]) -> None:
"""Prime the compressor with the current source pool (called per research phase)."""
self._materials = list(materials)
async def async_get_context(
self,
query: str,
max_results: int = 10,
cost_callback: Callable[..., Any] | None = None,
) -> list[dict[str, Any]]:
"""Select + compress the best ``max_results`` materials for ``query``."""
from deerflow.agents.deep_research.adapters.material import dedupe_materials
deduped = dedupe_materials(self._materials)
# Keyword pre-filter (no embedding dep in Phase 1).
scored = [(_keyword_score(query, m), m) for m in deduped]
scored.sort(key=lambda pair: pair[0], reverse=True)
selected = [m for _, m in scored[:max_results]]
# Budget guard: if total words exceed limit, summarise the tail.
total_words = sum(len((m.raw_content or "").split()) for m in selected)
if total_words > self._max_words and self._llm is not None:
selected = await self._summarise_tail(selected, query)
if cost_callback is not None:
try:
cost_callback(1, 0) # upstream signature: (total, done)
except Exception: # noqa: BLE001
pass
return [m.to_context_dict() for m in selected]
async def _summarise_tail(
self, materials: list[ResearchMaterial], query: str
) -> list[ResearchMaterial]:
"""Summarise lower-ranked materials to fit the word budget (Phase 2)."""
# Phase 1: simply truncate the list rather than calling the model,
# so a fresh deploy without model tuning still produces a report.
# (The full summarisation path is enabled once the basic runner is green.)
words = 0
kept: list[ResearchMaterial] = []
for m in materials:
w = len((m.raw_content or "").split())
if words + w > self._max_words:
break
words += w
kept.append(m)
return kept or materials[:1]
def _keyword_score(query: str, material: ResearchMaterial) -> float:
"""Cheap relevance heuristic: fraction of query terms present in the material."""
q_terms = {t.lower() for t in query.split() if len(t) > 1}
if not q_terms:
return 0.0
haystack = f"{material.title}\n{material.raw_content}".lower()
hits = sum(1 for t in q_terms if t in haystack)
base = hits / len(q_terms)
# Tiny recency/relevance nudges when metadata is present.
if material.relevance_score:
base += 0.01 * float(material.relevance_score)
return base
__all__ = ["ContextCompressor", "DeerFlowContextCompressor"]
@@ -0,0 +1,376 @@
"""Persisting event sink — bridges runners to the event-store repository.
Runners call :meth:`PersistingEventSink.emit` (which satisfies the
:class:`~deerflow.agents.deep_research.events.EventSink` protocol); the sink
delegates to :class:`~deerflow.persistence.deep_research_events.DeepResearchEventRepository`,
which assigns the monotonic ``seq`` and persists the row **before** returning.
This "persist-then-stream" ordering (§15.3) is what makes ``?after=<seq>``
reconnect-replay correct.
The sink is bound to one ``(session_id, job_id)`` pair at construction; runners
don't carry those ids around.
"""
from __future__ import annotations
import json
import logging
from collections.abc import Awaitable, Callable
from contextlib import suppress
from datetime import date, datetime
from enum import Enum
from time import monotonic
from typing import TYPE_CHECKING, Any
from deerflow.agents.deep_research.events import (
DeepResearchEvent,
EventType,
ResearchPhase,
)
if TYPE_CHECKING:
from deerflow.persistence.deep_research_events import DeepResearchEventRepository
logger = logging.getLogger(__name__)
def _json_default(value: Any) -> Any:
if isinstance(value, (datetime, date)):
return value.isoformat()
if isinstance(value, Enum):
return value.value
if hasattr(value, "model_dump"):
return value.model_dump(mode="json")
return str(value)
# Live-only frame types that have no durable counterpart at all. Report and
# summary text are already checkpointed (``report_chunk`` / ``summary_completed``),
# so only the reasoning channel needs a cross-worker mirror.
_MIRRORED_LIVE_TYPES = frozenset({"report_thinking", "plan_reasoning", "summary_thinking"})
# Coalescing bounds for that mirror: one bounded row per ~600 characters or
# ~1.5 seconds, never one row per provider token.
_MIRROR_MIN_CHARS = 600
_MIRROR_MAX_DELAY_S = 1.5
# Floor for the time-based flush: a slowly trickling reasoning stream must not
# degrade into one durable row per token. Anything shorter is carried until the
# next durable milestone flushes it.
_MIRROR_MIN_FLUSH_CHARS = 40
def _json_safe_payload(payload: dict[str, Any]) -> dict[str, Any]:
"""Normalize event payloads before they reach any event repository.
This protects deployments that still use a strict ``json.dumps`` inside
the SQL repository and keeps native datetime values in source metadata
from degrading every ``source_added`` event.
"""
return json.loads(json.dumps(payload, ensure_ascii=False, default=_json_default))
class PersistingEventSink:
"""EventSink implementation backed by the deep-research event repository.
Satisfies :class:`~deerflow.agents.deep_research.events.EventSink` (the
runtime-checkable protocol) via structural typing.
"""
def __init__(
self,
repo: DeepResearchEventRepository,
*,
session_id: str,
job_id: str,
on_event: Callable[[dict[str, Any]], Awaitable[None]] | None = None,
live_subscriber_probe: Callable[[str], bool] | None = None,
) -> None:
self._repo = repo
self._session_id = session_id
self._job_id = job_id
self._on_event = on_event
# Reports whether this worker holds an SSE connection for the job.
# Absent probe → assume the live path reaches the viewer (single-worker
# deployments and tests keep today's zero-extra-write behaviour).
self._live_subscriber_probe = live_subscriber_probe
self._mirror_buffers: dict[str, dict[str, Any]] = {}
self._mirroring = False
self.diagnostics: list[dict[str, Any]] = []
# A durable event row normally owns the sequence. When the event
# database is temporarily unavailable, use distinct negative sequence
# numbers for the live-only path so clients can render every milestone
# without advancing their durable ``after`` cursor.
self._fallback_seq = 0
async def emit(
self,
type: EventType, # noqa: A002
*,
phase: ResearchPhase = "initializing",
payload: dict | None = None,
) -> DeepResearchEvent:
event_payload = _json_safe_payload(payload or {})
persistence_error: Exception | None = None
if not self._mirroring:
# Order any coalesced reasoning tail before the next milestone
# instead of leaving it buffered until the job ends.
await self._flush_live_mirrors()
try:
row = await self._repo.append(
session_id=self._session_id,
job_id=self._job_id,
event_type=type,
phase=phase,
payload=event_payload,
)
# Reconstruct the envelope from the persisted row (seq is now assigned).
event = DeepResearchEvent(
seq=int(row.get("seq", 0)),
session_id=self._session_id,
job_id=self._job_id,
type=row.get("event_type", type),
phase=row.get("phase", phase),
payload=row.get("payload") or event_payload,
)
except Exception as exc: # noqa: BLE001
# Progress persistence must not turn a recoverable database blip
# into a failed report. The job/status stores still record terminal
# state; this live-only frame lets an attached UI stay responsive
# until durable replay becomes available again after reconnect.
self._fallback_seq -= 1
persistence_error = exc
self.diagnostics.append(
{
"code": "event_persistence_degraded",
"stage": phase,
"eventType": type,
"errorType": exc.__class__.__name__,
"detail": str(exc).split("[SQL:", 1)[0].strip()[:600] or exc.__class__.__name__,
}
)
logger.warning(
"deep-research event persistence degraded; continuing job with a live-only event "
"(job=%s, type=%s, phase=%s, error=%s)",
self._job_id,
type,
phase,
persistence_error.__class__.__name__,
)
event = DeepResearchEvent(
seq=self._fallback_seq,
session_id=self._session_id,
job_id=self._job_id,
type=type,
phase=phase,
payload={**event_payload, "eventPersistenceDegraded": True},
)
await self._publish_event(event)
if persistence_error is not None:
# Keep the original source/progress event live, and additionally
# expose one copyable diagnostic frame. This never re-enters the
# failing repository and therefore cannot interrupt the job.
self._fallback_seq -= 1
detail = str(persistence_error).split("[SQL:", 1)[0].strip()[:600]
await self._publish_event(
DeepResearchEvent(
seq=self._fallback_seq,
session_id=self._session_id,
job_id=self._job_id,
type="warning",
phase=phase,
payload={
"code": "event_persistence_degraded",
"message": "研究过程事件暂未写入数据库,报告任务已继续运行",
"recoverable": True,
"diagnostic": {
"stage": phase,
"eventType": type,
"errorType": persistence_error.__class__.__name__,
"detail": detail or persistence_error.__class__.__name__,
},
},
)
)
return event
async def _publish_event(self, event: DeepResearchEvent) -> None:
"""Push a persisted or live-only event without risking the worker."""
if self._on_event is None:
return
wire = event.to_sse_dict()
if event.seq < 0:
wire["liveId"] = f"event-persistence-fallback:{self._job_id}:{abs(event.seq)}"
try:
await self._on_event(wire)
except Exception: # noqa: BLE001
# A browser disconnect or a live-hub issue must not abort report
# generation; persisted rows will still be replayable on reconnect.
logger.warning(
"deep-research live event publish failed; job continues (job=%s, seq=%s)",
self._job_id,
event.seq,
exc_info=True,
)
async def publish_live(
self,
type: str, # noqa: A002
*,
phase: ResearchPhase = "initializing",
payload: dict | None = None,
) -> None:
if self._on_event is None:
return
try:
await self._on_event(
{
"seq": 0,
"sessionId": self._session_id,
"jobId": self._job_id,
"type": type,
"phase": phase,
"payload": payload or {},
}
)
except Exception: # noqa: BLE001
logger.warning(
"deep-research live frame publish failed; job continues (job=%s, type=%s)",
self._job_id,
type,
exc_info=True,
)
with suppress(Exception):
await self._mirror_live_frame(type, phase=phase, payload=payload or {})
# ── cross-worker mirror for live-only reasoning frames ──────────────────
async def _mirror_live_frame(
self,
type: str, # noqa: A002
*,
phase: ResearchPhase,
payload: dict[str, Any],
) -> None:
"""Buffer a live-only reasoning delta when no local viewer can see it.
The live hub is in-process. A job runs on the worker that won its
durable lease, while the SSE connection is served by whichever worker
accepted it, so on a multi-worker deployment the model's thinking is
usually published into a hub with no subscribers and is then lost —
there is no durable row to replay it from. When that is the case, the
reasoning is coalesced here and persisted as a bounded durable event
so the polling stream on the other worker still renders it.
"""
if type not in _MIRRORED_LIVE_TYPES or self._live_subscriber_probe is None:
return
delta = payload.get("delta")
if not isinstance(delta, str) or not delta:
return
if self._live_subscriber_probe(self._job_id):
# A viewer is attached here and has already received every frame
# live. Drop the buffer rather than flushing it, so that viewer
# does not also receive the same text as a durable replay.
self._mirror_buffers.pop(self._mirror_key(type, payload), None)
return
key = self._mirror_key(type, payload)
buffered = self._mirror_buffers.get(key)
if buffered is None:
buffered = {
"type": type,
"phase": phase,
"target": payload.get("target"),
"reportVersion": payload.get("reportVersion"),
"text": "",
"since": monotonic(),
}
self._mirror_buffers[key] = buffered
buffered["text"] += delta
buffered["phase"] = phase
buffered_chars = len(buffered["text"])
aged_out = (
buffered_chars >= _MIRROR_MIN_FLUSH_CHARS
and monotonic() - float(buffered["since"]) >= _MIRROR_MAX_DELAY_S
)
if buffered_chars >= _MIRROR_MIN_CHARS or aged_out:
await self._emit_mirror(key)
@staticmethod
def _mirror_key(type: str, payload: dict[str, Any]) -> str: # noqa: A002
return f"{type}::{payload.get('target') or 'report'}"
async def _flush_live_mirrors(self) -> None:
for key in tuple(self._mirror_buffers):
with suppress(Exception):
await self._emit_mirror(key)
async def _emit_mirror(self, key: str) -> None:
buffered = self._mirror_buffers.pop(key, None)
if buffered is None or not buffered["text"]:
return
payload: dict[str, Any] = {
"delta": buffered["text"],
"liveMirror": True,
}
if buffered["target"] is not None:
payload["target"] = buffered["target"]
if buffered["reportVersion"] is not None:
payload["reportVersion"] = buffered["reportVersion"]
self._mirroring = True
try:
await self.emit(buffered["type"], phase=buffered["phase"], payload=payload)
finally:
self._mirroring = False
class NullEventSink:
"""No-op sink for unit tests / dry runs that don't have a database.
Satisfies :class:`EventSink` structurally; assigns a monotonic in-memory seq
so fake tests can still assert event ordering.
"""
def __init__(self, *, session_id: str = "test", job_id: str = "test") -> None:
self._session_id = session_id
self._job_id = job_id
self._seq = 0
self.events: list[DeepResearchEvent] = []
self.live_events: list[dict[str, Any]] = []
async def emit(
self,
type: EventType, # noqa: A002
*,
phase: ResearchPhase = "initializing",
payload: dict | None = None,
) -> DeepResearchEvent:
self._seq += 1
ev = DeepResearchEvent(
seq=self._seq,
session_id=self._session_id,
job_id=self._job_id,
type=type,
phase=phase,
payload=payload or {},
)
self.events.append(ev)
return ev
async def publish_live(
self,
type: str, # noqa: A002
*,
phase: ResearchPhase = "initializing",
payload: dict | None = None,
) -> None:
self.live_events.append(
{
"seq": 0,
"sessionId": self._session_id,
"jobId": self._job_id,
"type": type,
"phase": phase,
"payload": payload or {},
}
)
__all__ = ["NullEventSink", "PersistingEventSink"]
@@ -0,0 +1,432 @@
"""Optional, DeerFlow-configured report image generation.
The upstream GPT Researcher image skill owns its provider configuration, so it
cannot be safely reused in DeerFlow. This adapter supports both a deployment
configured OpenAI-compatible ``/images/generations`` endpoint and the native
DashScope multimodal-generation endpoint used by Qwen Image 3.0 and Wan 2.7.
It is never configured from a browser request: model credentials stay in
``config.yaml`` and the per-session snapshot contains only the boolean
``generate_images``.
OpenAI-compatible images are requested as base64; DashScope image URLs are
downloaded immediately. The runner therefore always puts exact bytes into the
session's protected artifact directory, and reports never depend on an expiring
public image URL.
"""
from __future__ import annotations
import base64
import binascii
from typing import Any, Literal, Protocol, runtime_checkable
from urllib.parse import urlparse
import httpx
from pydantic import BaseModel, Field, field_validator
_MAX_IMAGE_BYTES = 20 * 1024 * 1024
_DASHSCOPE_GENERATION_PATH = "/api/v1/services/aigc/multimodal-generation/generation"
class ImageGenerationError(RuntimeError):
"""A recoverable image-provider failure (the report still completes)."""
class ImageGenerationSettings(BaseModel):
"""Server-only configuration for a report image endpoint.
Example ``config.yaml`` section::
deep_research:
image_generation:
enabled: true
provider: dashscope_native
base_url: https://token-plan.cn-beijing.maas.aliyuncs.com/api/v1/services/aigc/multimodal-generation/generation
api_key: $DEEP_RESEARCH_IMAGE_API_KEY
model: qwen-image-3.0-pro
``api_key`` is resolved by :class:`AppConfig` while loading YAML, but this
object is deliberately not placed in ``DeepResearchConfig`` and is never
persisted to a session/job row.
"""
enabled: bool = False
provider: Literal["openai_compatible", "dashscope_native"] = "openai_compatible"
base_url: str = ""
api_key: str = ""
model: str = ""
size: str = "1024x1024"
quality: str | None = None
watermark: bool = False
timeout_seconds: float = Field(default=120, ge=1, le=300)
model_config = {"extra": "ignore"}
@field_validator("base_url")
@classmethod
def _normalise_base_url(cls, value: str) -> str:
return value.strip().rstrip("/")
@field_validator("size")
@classmethod
def _validate_size(cls, value: str) -> str:
text = value.strip()
if not text or len(text) > 32:
raise ValueError("image_generation.size must be a short non-empty value")
return text
@property
def is_configured(self) -> bool:
return (
self.enabled
and self.base_url.startswith(("https://", "http://"))
and bool(self.api_key.strip())
and bool(self.model.strip())
)
@classmethod
def from_app_config(cls) -> ImageGenerationSettings:
"""Read the optional ``deep_research.image_generation`` section.
The core ``AppConfig`` allows product-specific extra sections, which
keeps this optional capability backward-compatible with old installs.
Any malformed optional configuration disables image generation rather
than preventing the gateway from serving normal research requests.
"""
try:
from deerflow.config import get_app_config
root = (get_app_config().model_extra or {}).get("deep_research")
raw = root.get("image_generation") if isinstance(root, dict) else None
return cls.model_validate(raw if isinstance(raw, dict) else {})
except Exception: # noqa: BLE001
return cls()
@runtime_checkable
class ImageProvider(Protocol):
"""Generate an image for a research section.
Successful providers return a mapping with binary ``data``, ``mime_type``,
``alt`` and ``prompt`` fields. The runner, not the provider, writes the
result into a protected research artifact.
"""
async def generate(
self,
prompt: str,
*,
section_id: str = "",
user_id: str = "",
) -> dict[str, Any] | None:
"""Return image bytes metadata, or ``None`` when generation is disabled."""
...
class NullImageProvider:
"""No-op image provider — ``generate_images=false`` default."""
async def generate(
self,
prompt: str,
*,
section_id: str = "",
user_id: str = "",
) -> None:
return None
class OpenAICompatibleImageProvider:
"""Generate one image through a configured OpenAI-compatible endpoint.
``transport`` is injection-only for adapter tests. Production instances
use a normal ``httpx.AsyncClient`` and make no assumptions about a specific
vendor beyond the standard ``/images/generations`` response shape.
"""
def __init__(
self,
settings: ImageGenerationSettings,
*,
transport: httpx.AsyncBaseTransport | None = None,
) -> None:
if not settings.is_configured:
raise ValueError("Image generation provider is not configured")
self._settings = settings
self._transport = transport
async def generate(
self,
prompt: str,
*,
section_id: str = "",
user_id: str = "",
) -> dict[str, Any]:
del user_id # Credentials are deployment-scoped; users never supply them.
body: dict[str, Any] = {
"model": self._settings.model,
"prompt": prompt,
"n": 1,
"size": self._settings.size,
# Base64 is mandatory: it yields a stable, protected local artifact.
"response_format": "b64_json",
}
if self._settings.quality:
body["quality"] = self._settings.quality
headers = {
"Authorization": f"Bearer {self._settings.api_key}",
"Content-Type": "application/json",
}
endpoint = f"{self._settings.base_url}/images/generations"
try:
async with httpx.AsyncClient(
timeout=self._settings.timeout_seconds,
transport=self._transport,
) as client:
response = await client.post(endpoint, json=body, headers=headers)
response.raise_for_status()
except httpx.TimeoutException as exc:
raise ImageGenerationError("图片生成服务响应超时") from exc
except httpx.HTTPStatusError as exc:
raise ImageGenerationError(f"图片生成服务返回 HTTP {exc.response.status_code}") from exc
except httpx.HTTPError as exc:
raise ImageGenerationError("图片生成服务暂时不可用") from exc
try:
payload = response.json()
except ValueError as exc:
raise ImageGenerationError("图片生成服务返回了无效响应") from exc
items = payload.get("data") if isinstance(payload, dict) else None
item = items[0] if isinstance(items, list) and items and isinstance(items[0], dict) else None
encoded = str((item or {}).get("b64_json") or "")
if not encoded:
raise ImageGenerationError("图片生成服务未返回可保存的图片数据")
# A few compatible endpoints prepend a data-URL header. Strip it
# without accepting arbitrary content as a URL or another fetch target.
if encoded.startswith("data:") and "," in encoded:
encoded = encoded.split(",", 1)[1]
try:
data = base64.b64decode(encoded, validate=True)
except (ValueError, binascii.Error) as exc:
raise ImageGenerationError("图片生成服务返回的图片编码无效") from exc
if not data or len(data) > _MAX_IMAGE_BYTES:
raise ImageGenerationError("生成图片为空或超过大小限制")
mime_type = str((item or {}).get("mime_type") or "image/png").lower()
if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
mime_type = "image/png"
alt = f"{section_id or '研究报告'}配图"
return {"data": data, "mime_type": mime_type, "alt": alt, "prompt": prompt}
class DashScopeNativeImageProvider:
"""Generate report illustrations through DashScope's native image API.
Qwen Image 3.0 and Wan 2.7 do not implement the OpenAI
``/images/generations`` request/response contract. They receive a
multimodal ``input.messages`` payload and normally return a short-lived
image URL. The URL is downloaded immediately so the research report keeps
a protected local artifact, just like the OpenAI-compatible provider.
"""
def __init__(
self,
settings: ImageGenerationSettings,
*,
transport: httpx.AsyncBaseTransport | None = None,
) -> None:
if not settings.is_configured:
raise ValueError("Image generation provider is not configured")
self._settings = settings
self._transport = transport
async def generate(
self,
prompt: str,
*,
section_id: str = "",
user_id: str = "",
) -> dict[str, Any]:
del user_id # Credentials are deployment-scoped; users never supply them.
body = _dashscope_request_body(self._settings, prompt)
headers = {
"Authorization": f"Bearer {self._settings.api_key}",
"Content-Type": "application/json",
}
try:
async with httpx.AsyncClient(
timeout=self._settings.timeout_seconds,
transport=self._transport,
follow_redirects=True,
) as client:
response = await client.post(
_dashscope_generation_endpoint(self._settings.base_url),
json=body,
headers=headers,
)
response.raise_for_status()
payload = response.json()
encoded, image_url, mime_type = _dashscope_image_result(payload)
if encoded:
data = _decode_image_base64(encoded)
elif image_url:
data, mime_type = await _download_dashscope_image(client, image_url, mime_type)
else:
raise ImageGenerationError("千问图片生成服务未返回图片地址或图片数据")
except ImageGenerationError:
raise
except httpx.TimeoutException as exc:
raise ImageGenerationError("图片生成服务响应超时") from exc
except httpx.HTTPStatusError as exc:
raise ImageGenerationError(_http_error_message(exc.response)) from exc
except httpx.HTTPError as exc:
raise ImageGenerationError("图片生成服务暂时不可用") from exc
except ValueError as exc:
raise ImageGenerationError("图片生成服务返回了无效响应") from exc
if not data or len(data) > _MAX_IMAGE_BYTES:
raise ImageGenerationError("生成图片为空或超过大小限制")
alt = f"{section_id or '研究报告'}配图"
return {"data": data, "mime_type": _normalise_mime_type(mime_type), "alt": alt, "prompt": prompt}
def _dashscope_generation_endpoint(base_url: str) -> str:
"""Resolve a native DashScope endpoint without relying on a workspace id.
Token Plan deployments commonly expose the full native endpoint, while
regular DashScope configuration often uses just an API origin. Supporting
both avoids an accidental ``.../generation/images/generations`` path.
"""
base = base_url.rstrip("/")
if base.endswith(_DASHSCOPE_GENERATION_PATH):
return base
parsed = urlparse(base)
if not parsed.scheme or not parsed.netloc:
return base
return f"{parsed.scheme}://{parsed.netloc}{_DASHSCOPE_GENERATION_PATH}"
def _dashscope_request_body(settings: ImageGenerationSettings, prompt: str) -> dict[str, Any]:
parameters: dict[str, Any] = {
"n": 1,
"size": settings.size.replace("x", "*").replace("X", "*"),
"watermark": settings.watermark,
}
# ``prompt_extend`` is a Qwen Image-only parameter. Wan 2.7 rejects
# unrelated generation options, so do not send it there.
if settings.model.strip().lower().startswith("qwen-image-"):
parameters["prompt_extend"] = True
return {
"model": settings.model,
"input": {
"messages": [
{
"role": "user",
"content": [{"text": prompt}],
}
]
},
"parameters": parameters,
}
def _dashscope_image_result(payload: object) -> tuple[str, str, str]:
"""Return optional base64, image URL and MIME type from native output."""
if not isinstance(payload, dict):
return "", "", ""
output = payload.get("output")
choices = output.get("choices") if isinstance(output, dict) else None
if not isinstance(choices, list):
return "", "", ""
for choice in choices:
message = choice.get("message") if isinstance(choice, dict) else None
content = message.get("content") if isinstance(message, dict) else None
if not isinstance(content, list):
continue
for part in content:
if not isinstance(part, dict):
continue
encoded = str(part.get("b64_json") or "")
raw_image_url = part.get("image") or part.get("image_url")
if isinstance(raw_image_url, dict):
raw_image_url = raw_image_url.get("url")
image_url = str(raw_image_url or "")
if encoded or image_url:
return encoded, image_url, str(part.get("mime_type") or "")
return "", "", ""
async def _download_dashscope_image(
client: httpx.AsyncClient,
image_url: str,
mime_type: str,
) -> tuple[bytes, str]:
"""Download only an HTTPS model result, never forwarding the API key."""
parsed = urlparse(image_url)
if parsed.scheme != "https" or not parsed.netloc:
raise ImageGenerationError("千问图片生成服务返回了不安全的图片地址")
response = await client.get(image_url, headers={"Accept": "image/png,image/jpeg,image/webp"})
response.raise_for_status()
final_url = urlparse(str(response.url))
if final_url.scheme != "https" or not final_url.netloc:
raise ImageGenerationError("千问图片下载地址不安全")
response_mime_type = response.headers.get("Content-Type", "").split(";", 1)[0].strip().lower()
return response.content, response_mime_type or mime_type
def _decode_image_base64(encoded: str) -> bytes:
if encoded.startswith("data:") and "," in encoded:
encoded = encoded.split(",", 1)[1]
try:
return base64.b64decode(encoded, validate=True)
except (ValueError, binascii.Error) as exc:
raise ImageGenerationError("图片生成服务返回的图片编码无效") from exc
def _normalise_mime_type(value: str) -> str:
mime_type = value.lower()
return mime_type if mime_type in {"image/png", "image/jpeg", "image/webp"} else "image/png"
def _http_error_message(response: httpx.Response) -> str:
"""Expose a short provider message without leaking credentials or payloads."""
detail = ""
try:
payload = response.json()
if isinstance(payload, dict):
error = payload.get("error")
source = error if isinstance(error, dict) else payload
message = source.get("message") if isinstance(source, dict) else None
if isinstance(message, str) and message.strip():
detail = f":{message.strip()[:300]}"
except ValueError:
pass
return f"图片生成服务返回 HTTP {response.status_code}{detail}"
def build_image_provider() -> ImageProvider:
"""Build the optional provider from server config, or a safe no-op."""
settings = ImageGenerationSettings.from_app_config()
if not settings.is_configured:
return NullImageProvider()
if settings.provider == "dashscope_native":
return DashScopeNativeImageProvider(settings)
return OpenAICompatibleImageProvider(settings)
__all__ = [
"ImageGenerationError",
"ImageGenerationSettings",
"ImageProvider",
"NullImageProvider",
"OpenAICompatibleImageProvider",
"DashScopeNativeImageProvider",
"build_image_provider",
]
@@ -0,0 +1,639 @@
"""LLM adapter — the single chokepoint between vendored code and DeerFlow models.
The vendored GPT Researcher code expects ``create_chat_completion()`` /
``get_llm()`` style helpers (``gpt_researcher/utils/llm.py``). Rather than
letting it dynamically import providers, we keep those function *signatures* in
the vendored copy but route them through :class:`DeerFlowCompletionBackend`,
which delegates to DeerFlow's :func:`create_chat_model`.
Responsibilities (§9):
- Resolve the model name from the per-task ``DeepResearchConfig`` by role
(``fast`` / ``smart`` / ``strategic``); ``None`` → system default.
- Convert ``[{role, content}, ...]`` dicts to LangChain messages.
- Call ``model.ainvoke()`` and normalise text + ``usage_metadata``.
- Support structured output via a Pydantic ``response_schema`` (one JSON-repair
retry, no ``json-repair`` hard dep — §9.5).
- Emit a ``model_usage`` event per call.
- Honour the :class:`CancellationToken`.
Retry policy (§9.5): exactly **one** retry layer here — a structured-output
parse failure triggers one "fix the JSON" re-prompt. Cancel / permission /
config errors are never retried.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from typing import TYPE_CHECKING, Any
from deerflow.agents.deep_research.cancellation import CancellationToken
from deerflow.agents.deep_research.config import DeepResearchConfig, ModelRole
from deerflow.agents.deep_research.events import EventSink
from deerflow.agents.deep_research.types import CompletionResult
from deerflow.utils.stream_text import InlineThinkTagFilter, strip_inline_think_tags
if TYPE_CHECKING:
from langchain.chat_models import BaseChatModel
from langchain_core.messages import BaseMessage
from pydantic import BaseModel
logger = logging.getLogger(__name__)
def _parent_isolated_run_config(run_name: str) -> dict[str, Any]:
"""Keep LangGraph runtime, drop only inherited LLM streaming callbacks.
``StreamWriter`` reads ``get_config()[configurable]`` at emit time. Replacing
the child config with a blank dict makes ``report_delta`` / ``report_thinking``
raise and get swallowed, so the sandbox only appears after the file lands.
"""
parent: dict[str, Any] = {}
try:
from langchain_core.runnables.config import var_child_runnable_config
current = var_child_runnable_config.get()
if isinstance(current, dict):
parent = dict(current)
except Exception: # noqa: BLE001 — isolation must never fail the completion
parent = {}
tags = [tag for tag in (parent.get("tags") or []) if tag != "nostream"]
tags.append("nostream")
parent["callbacks"] = []
parent["tags"] = tags
parent["run_name"] = run_name
return parent
def _clear_inherited_runnable_config(isolated: dict[str, Any]) -> Callable[[], None]:
"""Install the isolated config for the duration of ``astream``."""
try:
from langchain_core.runnables.config import var_child_runnable_config
except ImportError:
return lambda: None
token = var_child_runnable_config.set(isolated)
return lambda: var_child_runnable_config.reset(token)
# ── message conversion ──────────────────────────────────────────────────────
def to_langchain_messages(messages: list[dict[str, str]]) -> list[BaseMessage]:
"""Convert ``[{role, content}, ...]`` dicts to LangChain message objects.
``role`` accepts the usual OpenAI-ish names: ``system`` / ``user`` /
``assistant`` / ``human`` / ``ai``. Unknown roles fall back to ``HumanMessage``
so a bad prompt never crashes the whole research run.
"""
from langchain_core.messages import (
AIMessage,
HumanMessage,
SystemMessage,
)
out: list[BaseMessage] = []
for m in messages:
role = (m.get("role") or "user").lower()
content = m.get("content") or ""
if role in ("system", "developer"):
out.append(SystemMessage(content=content))
elif role in ("assistant", "ai"):
out.append(AIMessage(content=content))
else: # user / human / unknown → human (safe default)
out.append(HumanMessage(content=content))
return out
def _content_to_text(resp: Any) -> str:
"""Best-effort extraction of text from a LangChain AIMessage-like response."""
content = getattr(resp, "content", None)
if isinstance(content, str):
return content
if isinstance(content, list):
# Some providers return content blocks; concatenate text parts.
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
# Never fall through to ``content`` for structured reasoning
# blocks. Providers differ here, but those blocks are not
# report prose and must not reach ReportChunkEmitter.
if block.get("type") in {
"thinking",
"thinking_delta",
"reasoning",
"reasoning_delta",
}:
continue
parts.append(str(block.get("text") or block.get("content") or ""))
return "".join(parts)
return str(content or "")
def _reasoning_delta(chunk: Any) -> str:
"""Extract the incremental reasoning/thinking text from a streamed chunk.
Reasoning models (deepseek-reasoner, GLM thinking, QwQ, ...) stream their
thinking in ``additional_kwargs.reasoning_content`` while ``content`` stays
empty until the final answer starts. A stream loop that only reads
``content`` skips these chunks entirely, so callers get a dedicated
callback for them.
"""
ak = getattr(chunk, "additional_kwargs", None) or {}
val = ak.get("reasoning_content") or ak.get("reasoning") or ""
if isinstance(val, str) and val:
return val
# Anthropic/OpenAI-compatible adapters may put the same increment in an
# explicitly typed content block instead of ``additional_kwargs``.
content = getattr(chunk, "content", None)
if isinstance(content, list):
parts = [
str(block.get("text") or block.get("content") or "")
for block in content
if isinstance(block, dict)
and block.get("type") in {"thinking", "thinking_delta", "reasoning", "reasoning_delta"}
]
return "".join(parts)
return ""
def _extract_usage(resp: Any) -> dict[str, Any]:
"""Read ``AIMessage.usage_metadata`` (§9.4) — model-agnostic token accounting."""
um = getattr(resp, "usage_metadata", None)
if isinstance(um, dict):
return {
"prompt_tokens": um.get("input_tokens", 0),
"completion_tokens": um.get("output_tokens", 0),
"total_tokens": um.get("total_tokens", 0),
}
# Fallback: some providers populate response_metadata.
rm = getattr(resp, "response_metadata", None) or {}
token_usage = rm.get("token_usage") or rm.get("usage") or {}
if isinstance(token_usage, dict) and token_usage:
return {
"prompt_tokens": token_usage.get("prompt_tokens", 0),
"completion_tokens": token_usage.get("completion_tokens", 0),
"total_tokens": token_usage.get("total_tokens", 0),
}
return {}
# ── errors ──────────────────────────────────────────────────────────────────
class CompletionError(Exception):
"""Non-retryable completion failure (config / permission / budget)."""
def __init__(self, message: str, *, code: str = "completion_error", retryable: bool = False) -> None:
super().__init__(message)
self.code = code
self.retryable = retryable
# ── the backend ─────────────────────────────────────────────────────────────
class DeerFlowCompletionBackend:
"""The LLM adapter every vendored call funnels through.
Constructed once per job (with the frozen config) and handed to runners via
:class:`AdapterBundle`. Model instances are cached by role so repeated
calls don't rebuild the provider.
"""
def __init__(
self,
config: DeepResearchConfig,
event_sink: EventSink | None = None,
cancellation: CancellationToken | None = None,
force_disable_thinking: bool = False,
) -> None:
self._config = config
self._events = event_sink
self._cancel = cancellation
# 与 lead agent 同语义的「思考硬关」开关(内网 vLLM/Qwen 只声明
# supports_thinking 时,thinking_enabled=False 不会注入 enable_thinking=false)。
self._force_disable_thinking = force_disable_thinking
self._models: dict[str, BaseChatModel] = {}
# Running token totals (keyed by model name) for the final usage report.
self._usage_totals: dict[str, dict[str, int]] = {}
# ── model resolution ────────────────────────────────────────────────────
def _resolve_model_name(self, role: ModelRole) -> str | None:
return self._config.resolve_model_name(role)
def _get_model(self, role: ModelRole) -> BaseChatModel:
name = self._resolve_model_name(role)
cache_key = name or "__default__"
cached = self._models.get(cache_key)
if cached is not None:
return cached
from deerflow.models import create_chat_model
model = create_chat_model(
name=name,
thinking_enabled=self._config.thinking_enabled,
force_disable_thinking=self._force_disable_thinking,
)
self._models[cache_key] = model
return model
# ── the main entry point ────────────────────────────────────────────────
async def complete(
self,
*,
model_role: ModelRole = "smart",
messages: list[dict[str, str]],
temperature: float | None = None,
response_schema: type[BaseModel] | None = None,
operation: str = "unknown",
) -> CompletionResult:
"""Run one completion, normalise output, bill usage.
Args:
model_role: which config role to use (fast/cheap research queries →
``fast``; report writing → ``smart``; strategic planning → ``strategic``).
messages: OpenAI-style ``[{role, content}]`` dicts.
temperature: optional override (most vendored calls don't set one).
response_schema: if given, parse + validate the output against this
Pydantic model (structured output). One JSON-repair retry on
parse failure (§9.5).
operation: a short label for the ``model_usage`` event
(e.g. ``"generate_queries"``, ``"write_report"``).
"""
if self._cancel is not None:
self._cancel.raise_if_cancelled()
model = self._get_model(model_role)
model_name = self._resolve_model_name(model_role) or "default"
lc_messages = to_langchain_messages(messages)
kwargs: dict[str, Any] = {}
if temperature is not None:
kwargs["temperature"] = temperature
# ── plain completion ──
if response_schema is None:
resp = await self._invoke_with_timeout(model, lc_messages, kwargs, operation)
text = _content_to_text(resp)
usage = _extract_usage(resp)
await self._record_usage(model_name, operation, usage)
return CompletionResult(text=text, model=model_name, usage=usage)
# ── structured completion (with one repair retry) ──
schema_instructions = _schema_instructions(response_schema)
structured_messages = list(lc_messages)
# Append the schema instruction to the last system/user message block.
structured_messages = _inject_schema_hint(structured_messages, schema_instructions)
for attempt in range(2):
if self._cancel is not None:
self._cancel.raise_if_cancelled()
resp = await self._invoke_with_timeout(model, structured_messages, kwargs, operation)
text = _content_to_text(resp)
usage = _extract_usage(resp)
parsed = _try_parse_structured(text, response_schema)
if parsed is not None:
await self._record_usage(model_name, operation, usage)
return CompletionResult(text=text, model=model_name, usage=usage, structured=parsed)
if attempt == 0:
# One repair re-prompt (§9.5).
logger.warning("deep_research llm: JSON parse failed for %s, retrying", operation)
structured_messages = _inject_repair_hint(structured_messages, text, response_schema)
await self._record_usage(model_name, operation, usage)
raise CompletionError(
f"Structured output for '{operation}' could not be parsed after retry",
code="structured_parse_failed",
retryable=True,
)
async def _invoke_with_timeout(
self,
model: BaseChatModel,
messages: list[BaseMessage],
kwargs: dict[str, Any],
operation: str,
) -> Any:
"""Bound a non-streaming control-plane call without capping report prose.
Model configurations may have a much longer HTTP ``request_timeout``.
A research job cannot leave its visible planning/review stage frozen for
that whole interval, so these short structured calls get their own
deadline. ``wait_for`` cancels the pending provider coroutine before a
durable runner either uses its safe fallback or records a terminal error.
"""
timeout = self._config.control_plane_timeout_seconds
try:
return await asyncio.wait_for(model.ainvoke(messages, **kwargs), timeout=timeout)
except TimeoutError as exc:
if self._cancel is not None:
self._cancel.raise_if_cancelled()
raise CompletionError(
f"Model did not complete '{operation}' within {timeout:g} seconds",
code="completion_timeout",
retryable=True,
) from exc
async def stream_complete(
self,
*,
model_role: ModelRole = "smart",
messages: list[dict[str, str]],
temperature: float | None = None,
max_tokens: int | None = None,
operation: str = "unknown",
on_delta: Callable[[str], Awaitable[None]] | None = None,
on_reasoning: Callable[[str], Awaitable[None]] | None = None,
) -> CompletionResult:
"""Run a prose completion through LangChain's incremental stream API.
Structured planning/review calls intentionally stay on :meth:`complete`.
This method is reserved for report prose so the final returned text is
exactly the concatenation of the deltas the user has already seen.
``on_reasoning`` receives the thinking/reasoning increments reasoning
models emit before the answer; they are not part of the returned text.
"""
if self._cancel is not None:
self._cancel.raise_if_cancelled()
model = self._get_model(model_role)
model_name = self._resolve_model_name(model_role) or "default"
kwargs: dict[str, Any] = {}
if temperature is not None:
kwargs["temperature"] = temperature
if max_tokens is not None:
# Report-only callers may need a larger completion than a model's
# conservative deployment default. Codex Responses endpoints
# reject both token-limit parameters, so preserve the factory's
# provider-specific exclusion for that adapter.
from deerflow.models.openai_codex_provider import CodexChatModel
if not isinstance(model, CodexChatModel):
kwargs["max_tokens"] = max_tokens
text_parts: list[str] = []
usage: dict[str, Any] = {}
inline_think_filter = InlineThinkTagFilter()
# Isolate from the parent LangGraph run: inherited callbacks would
# replay every report token into the chat ``messages`` stream, so the
# left-hand bubble and the sandbox would write the same prose at once.
# Custom ``report_delta`` events remain the only live preview channel.
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
attempts = 2 if operation.startswith("write_report") else 1
last_error: Exception | None = None
try:
for attempt in range(attempts):
text_parts.clear()
inline_think_filter = InlineThinkTagFilter()
try:
async for chunk in model.astream(
to_langchain_messages(messages),
config=isolated_config,
**kwargs,
):
if self._cancel is not None:
self._cancel.raise_if_cancelled()
reasoning = _reasoning_delta(chunk)
if reasoning and on_reasoning is not None:
await on_reasoning(reasoning)
# A few OpenAI-compatible models put their reasoning in the
# ordinary content channel instead of ``reasoning_content``.
# Split it from Markdown even when tags span tokens: callers may
# display a live thought trace, but it never joins ``text``.
delta, inline_reasoning = inline_think_filter.push_parts(_content_to_text(chunk))
if inline_reasoning and on_reasoning is not None:
await on_reasoning(inline_reasoning)
if delta:
text_parts.append(delta)
if on_delta is not None:
await on_delta(delta)
chunk_usage = _extract_usage(chunk)
if any(chunk_usage.values()):
usage = chunk_usage
last_error = None
break
except Exception as exc: # noqa: BLE001 - report writing waits like ordinary chat
from deerflow.agents.deep_research.cancellation import ResearchCancelled
if isinstance(exc, (ResearchCancelled, asyncio.CancelledError)):
raise
last_error = exc
# Only replay a clean connection; a half-written stream is
# reset by the runner, not by appending a second attempt.
if text_parts or attempt + 1 >= attempts:
raise
logger.warning(
"deep_research llm: %s stream interrupted before the first token (%s); retrying once",
operation,
type(exc).__name__,
)
await asyncio.sleep(1.5)
if last_error is not None:
raise last_error
finally:
reset_parent_config()
tail, inline_reasoning_tail = inline_think_filter.finish_parts()
if inline_reasoning_tail and on_reasoning is not None:
await on_reasoning(inline_reasoning_tail)
if tail:
text_parts.append(tail)
if on_delta is not None:
await on_delta(tail)
await self._record_usage(model_name, operation, usage)
return CompletionResult(text="".join(text_parts), model=model_name, usage=usage)
async def complete_prose(
self,
*,
model_role: ModelRole = "smart",
messages: list[dict[str, str]],
temperature: float | None = None,
max_tokens: int | None = None,
operation: str = "unknown",
) -> CompletionResult:
"""Generate long prose without the provider's inter-chunk timeout.
This is the report retry path. Some reasoning models pause for longer
than the streaming client's 120-second chunk deadline after already
producing several chunks. A normal complete response can still finish
successfully, so the retry deliberately waits for that full response.
"""
if self._cancel is not None:
self._cancel.raise_if_cancelled()
model = self._get_model(model_role)
model_name = self._resolve_model_name(model_role) or "default"
kwargs: dict[str, Any] = {}
if temperature is not None:
kwargs["temperature"] = temperature
if max_tokens is not None:
from deerflow.models.openai_codex_provider import CodexChatModel
if not isinstance(model, CodexChatModel):
kwargs["max_tokens"] = max_tokens
isolated_config = _parent_isolated_run_config(f"deep_research:{operation}")
reset_parent_config = _clear_inherited_runnable_config(isolated_config)
try:
response = await model.ainvoke(
to_langchain_messages(messages),
config=isolated_config,
**kwargs,
)
finally:
reset_parent_config()
if self._cancel is not None:
self._cancel.raise_if_cancelled()
# ``ainvoke`` returns raw content, including the inline ``<think>``
# block some OpenAI-compatible deployments write there. The streamed
# path filters it out, so this retry path must not reintroduce it.
text = strip_inline_think_tags(_content_to_text(response))
usage = _extract_usage(response)
await self._record_usage(model_name, operation, usage)
return CompletionResult(text=text, model=model_name, usage=usage)
# ── usage accounting ────────────────────────────────────────────────────
async def _record_usage(self, model_name: str, operation: str, usage: dict[str, Any]) -> None:
"""Accumulate per-model totals + emit a ``model_usage`` event."""
bucket = self._usage_totals.setdefault(model_name, {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0})
for k in bucket:
bucket[k] += int(usage.get(k, 0) or 0)
if self._events is not None:
await self._events.emit(
"model_usage",
payload={
"model": model_name,
"operation": operation,
"usage": usage,
},
)
@property
def usage_totals(self) -> dict[str, dict[str, int]]:
"""Snapshot of cumulative token usage by model (for the final result)."""
return {k: dict(v) for k, v in self._usage_totals.items()}
# ── convenience: the upstream-compatible function family ────────────────
#
# These mirror ``gpt_researcher/utils/llm.py`` signatures so the vendored
# action/skill code needs minimal changes — it calls a context-local bound
# method instead of a module global.
async def create_chat_completion(
self,
messages: list[dict[str, str]],
*,
model_role: ModelRole = "smart",
temperature: float | None = None,
response_format: type[BaseModel] | None = None,
stream: bool = False,
) -> str:
"""Upstream-compatible ``create_chat_completion`` → returns raw text."""
if stream and response_format is None:
result = await self.stream_complete(
model_role=model_role,
messages=messages,
temperature=temperature,
operation="chat_completion",
)
else:
result = await self.complete(
model_role=model_role,
messages=messages,
temperature=temperature,
response_schema=response_format,
operation="chat_completion",
)
return result.text
# ── structured-output helpers (no json-repair dependency) ───────────────────
def _schema_instructions(schema: type[BaseModel]) -> str:
"""Build a compact "respond as JSON matching this schema" instruction."""
props = schema.model_json_schema().get("properties", {})
fields = ", ".join(f'"{k}"' for k in props)
return f"Respond with ONLY a valid JSON object (no markdown, no prose) containing these keys: {fields}."
def _inject_schema_hint(messages: list[BaseMessage], hint: str) -> list[BaseMessage]:
"""Append the schema instruction as a final system message."""
from langchain_core.messages import SystemMessage
return [*messages, SystemMessage(content=hint)]
def _inject_repair_hint(messages: list[BaseMessage], bad_text: str, schema: type[BaseModel]) -> list[BaseMessage]:
"""Re-prompt: 'your last reply was invalid JSON, fix it'."""
from langchain_core.messages import HumanMessage
return [
*messages,
HumanMessage(content=(f"Your previous reply was not valid JSON and could not be parsed:\n{bad_text[:500]}\n\nPlease respond again with ONLY a valid JSON object.")),
]
def parse_structured_text(text: str, schema: type[BaseModel]) -> BaseModel | None:
"""Parse a completed (possibly streamed) model output against a schema.
Public wrapper over the same fence-stripping, blob-extracting parse used
by :meth:`DeerFlowCompletionBackend.complete`, so runners that collect the
text through :meth:`stream_complete` can validate it afterwards.
"""
return _try_parse_structured(text, schema)
def _try_parse_structured(text: str, schema: type[BaseModel]) -> BaseModel | None:
"""Strip markdown fences + attempt ``schema.model_validate_json``."""
cleaned = _strip_code_fences(text).strip()
if not cleaned:
return None
try:
return schema.model_validate_json(cleaned)
except Exception:
# Try to locate the first {...} blob as a last resort.
start = cleaned.find("{")
end = cleaned.rfind("}")
if start != -1 and end > start:
try:
return schema.model_validate_json(cleaned[start : end + 1])
except Exception:
return None
return None
def _strip_code_fences(text: str) -> str:
"""Remove ```json ... ``` / ``` ... ``` wrapping if present."""
stripped = text.strip()
if stripped.startswith("```"):
# Drop first line (the fence + optional language tag).
lines = stripped.splitlines()
if len(lines) >= 2:
lines = lines[1:]
if lines and lines[-1].strip().startswith("```"):
lines = lines[:-1]
return "\n".join(lines)
return stripped
__all__ = [
"CompletionError",
"DeerFlowCompletionBackend",
"parse_structured_text",
"to_langchain_messages",
]
@@ -0,0 +1,75 @@
"""Material provider — DeerFlow's unified information-collection boundary.
The vendored GPT Researcher code normally fires off its own retrievers + scrapers
per source URL. We replace that entirely: the :class:`MaterialProvider` returns
**fully-fetched** :class:`ResearchMaterial` objects (``raw_content`` already
populated), so the vendored scraper/BrowserManager is never invoked (§6.4).
First version (Phase 1) wraps :func:`deerflow.community.configurable_search.execute_search`:
1. Run the configured ``web_search`` tool for the query.
2. Map normalised results (``recUuid / content / title / url / source / time``)
→ :class:`ResearchMaterial`.
3. If ``content`` is already full text → use as ``raw_content`` (no fetch needed).
4. If only a snippet → a DeerFlow fetch tool backfills (Phase 2).
5. Dedupe by ``rec_uuid`` → canonical URL → ``content_hash``.
The provider never imports ``app.*`` and never does raw ``requests.get``.
"""
from __future__ import annotations
from typing import TYPE_CHECKING, Protocol, runtime_checkable
from deerflow.agents.deep_research.types import ResearchMaterial
if TYPE_CHECKING:
from deerflow.agents.deep_research.cancellation import CancellationToken
@runtime_checkable
class MaterialProvider(Protocol):
"""Protocol every material channel implements (§8.3)."""
async def search(
self,
query: str,
*,
domains: list[str] | None = None,
limit: int = 8,
user_id: str = "",
session_id: str = "",
cancellation: CancellationToken | None = None,
) -> list[ResearchMaterial]:
"""Return fully-fetched materials for ``query``.
Implementations guarantee ``raw_content`` is non-empty for every item
that has fetchable text (the vendored writer relies on this).
"""
...
def dedupe_materials(materials: list[ResearchMaterial]) -> list[ResearchMaterial]:
"""Stable dedup by rec_uuid → canonical url → content_hash (§8.3 §6).
Keeps the first occurrence; later duplicates are dropped (but their queries
are noted on the survivor's metadata so multi-query hits aren't lost).
"""
seen: dict[str, int] = {} # key → index in result
out: list[ResearchMaterial] = []
for m in materials:
key = (m.rec_uuid and f"rec:{m.rec_uuid}") or (m.url and f"url:{m.url}") or m.compute_hash()
if key in seen:
idx = seen[key]
# Record that another query also surfaced this material.
surviving = out[idx]
extra_queries = surviving.metadata.setdefault("_also_matched_queries", [])
if m.query and m.query not in extra_queries:
extra_queries.append(m.query)
continue
seen[key] = len(out)
out.append(m)
return out
__all__ = ["MaterialProvider", "dedupe_materials"]
@@ -0,0 +1,601 @@
"""Deep Research material provider backed by DeerFlow's Q&A retrieval stack.
This is the only place adapted GPT Researcher runners retrieve materials. It
uses the same hierarchy as DeerFlow Q&A instead of treating the legacy
configurable ``web_search`` endpoint as the only possible source:
1. Run enabled Q&A research skills through the existing ``SubagentExecutor``
runtime when their required search tool is registered.
2. Use a configured ``web_search`` provider when the deployment has one.
3. Fall back to DeerFlow's existing keyless DuckDuckGo search provider.
The fallback is important for this offline deployment: its configurable search
entry is deliberately disabled while it still contains a placeholder intranet
endpoint. A missing endpoint must never turn every Deep Research report into
"no materials collected".
"""
from __future__ import annotations
import asyncio
import hashlib
import importlib.util
import json
import logging
import re
from datetime import datetime
from typing import Any
from urllib.parse import urlparse
from deerflow.agents.deep_research.adapters.material import dedupe_materials
from deerflow.agents.deep_research.types import ResearchMaterial
logger = logging.getLogger(__name__)
# Treat shorter content as a snippet and try a full-text fetch when available.
SNIPPET_THRESHOLD = 300
ONLINE_SEARCH_TIMEOUT_SECONDS = 45
SECONDARY_ONLINE_SEARCH_TIMEOUT_SECONDS = 20
# Same skills that guide the Q&A assistant's public-web / knowledge retrieval.
# The order deliberately prefers the multi-round skill for a Deep Research job.
QUESTION_ANSWER_RESEARCH_SKILLS: tuple[str, ...] = (
"deep-search",
"web-research",
"weknora-knowledge-search",
)
# A skill is run only when the Q&A runtime exposes the search tool it needs.
# This prevents a disabled placeholder web_search config from launching an
# agent which is incapable of searching and can only wait for its timeout.
_SKILL_REQUIRED_TOOLS: dict[str, set[str]] = {
"deep-search": {"web_search"},
"web-research": {"web_search"},
"weknora-knowledge-search": {"weknora_search"},
}
class DeerFlowMaterialProvider:
"""MaterialProvider backed by the Q&A skills and online retrieval stack.
A configurable ``web_search`` endpoint remains supported, but it is no
longer a single point of failure. All returned materials preserve the
original source, URL and retrieval channel for citations and auditing.
"""
def __init__(
self,
*,
user_id: str = "",
session_id: str = "",
runtime_thread_id: str = "",
model_name: str | None = None,
fetch_enabled: bool = True,
retrieval_skills: list[str] | None = None,
) -> None:
self._user_id = user_id
self._session_id = session_id
self._runtime_thread_id = runtime_thread_id
self._model_name = model_name
self._fetch_enabled = fetch_enabled
self._retrieval_skills = _normalize_named_skills(retrieval_skills)
self._settings = None
self._settings_error: str | None = None
self._question_answer_tool_names = _load_question_answer_tool_names(model_name)
self._research_skills = _load_question_answer_research_skills()
self._online_search_available = importlib.util.find_spec("ddgs") is not None
self._online_search_semaphore: asyncio.Semaphore | None = None
try:
from deerflow.community.configurable_search.tools import ConfigurableSearchSettings
from deerflow.config import get_app_config
cfg = get_app_config().get_tool_config("web_search")
self._settings = ConfigurableSearchSettings.from_tool_config(cfg)
if not self._settings.enabled:
self._settings_error = "web_search tool is disabled in config"
self._settings = None
except Exception as exc: # noqa: BLE001
self._settings_error = f"web_search not configured: {exc}"
logger.warning("Deep Research configurable web_search unavailable: %s", self._settings_error)
async def search(
self,
query: str,
*,
domains: list[str] | None = None,
limit: int = 8,
user_id: str = "",
session_id: str = "",
cancellation=None,
) -> list[ResearchMaterial]:
"""Collect actual Q&A-compatible materials for one research query."""
if cancellation is not None:
cancellation.raise_if_cancelled()
materials: list[ResearchMaterial] = []
seen_keys: set[str] = set()
directed_skills = bool(self._retrieval_skills)
# 1. The genuine Q&A skill runtime. Skills may call web, MCP or another
# deployment-specific source according to their SKILL.md instructions.
# A non-empty 检索来源 list replaces the default research-skill trio.
skill_results = await self._search_via_question_answer_skills(query, limit=limit)
materials.extend(
self._normalise_search_results(
skill_results,
query=query,
channel="skill",
domains=domains,
seen_keys=seen_keys,
)
)
if cancellation is not None:
cancellation.raise_if_cancelled()
# Specified skills are the retrieval source: do not also dump generic
# web_search / DDG on top. Empty 检索来源 keeps the original fallbacks.
if directed_skills:
hydrated: list[ResearchMaterial] = []
for material in materials[:limit]:
if cancellation is not None:
cancellation.raise_if_cancelled()
if material.url and len(material.raw_content) < SNIPPET_THRESHOLD and self._fetch_enabled:
fetched = await asyncio.to_thread(_safe_fetch, material.url)
if fetched:
material.raw_content = fetched
if material.raw_content:
hydrated.append(material)
return dedupe_materials(hydrated)[:limit]
# 2. Retain compatibility with deployed configurable search providers.
if len(materials) < limit:
configured_results = await self._search_configured_web(query)
materials.extend(
self._normalise_configured_results(
configured_results,
query=query,
domains=domains,
seen_keys=seen_keys,
)
)
if cancellation is not None:
cancellation.raise_if_cancelled()
# 3. Reuse DeerFlow's maintained online provider if the Q&A search tool
# is absent or the configured provider returned too few materials.
if len(materials) < limit:
online_results = await self._search_via_online_fallback(query, limit=limit - len(materials))
materials.extend(
self._normalise_search_results(
online_results,
query=query,
channel="web_search",
domains=domains,
seen_keys=seen_keys,
)
)
hydrated: list[ResearchMaterial] = []
for material in materials[:limit]:
if cancellation is not None:
cancellation.raise_if_cancelled()
if material.url and len(material.raw_content) < SNIPPET_THRESHOLD and self._fetch_enabled:
fetched = await asyncio.to_thread(_safe_fetch, material.url)
if fetched:
material.raw_content = fetched
if material.raw_content:
hydrated.append(material)
return dedupe_materials(hydrated)[:limit]
async def _search_via_question_answer_skills(self, query: str, *, limit: int) -> list[Any]:
"""Run eligible research skills through the same runtime as Q&A."""
runnable = self._runnable_research_skills()
if not runnable:
logger.info(
"Deep Research Q&A skills are enabled but their required search tools are unavailable; "
"using the online fallback"
)
return []
try:
from deerflow.agents.ai_writing.nodes._skill_agent import run_skill_via_real_agent
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research could not load the Q&A skill runtime: %s", exc)
return []
run_config = {
"configurable": {
"thread_id": self._runtime_thread_id,
"model_name": self._model_name or "",
}
}
state = {"model_name": self._model_name or ""}
collected: list[Any] = []
directed = bool(self._retrieval_skills)
for skill in runnable:
try:
results, error = await run_skill_via_real_agent(
skill=skill,
intent=f"Collect verifiable research materials for: {query}",
keywords=[query],
user_query=query,
config=run_config,
state=state,
model_name_fallback=self._model_name,
max_results=limit,
timeout_s=ONLINE_SEARCH_TIMEOUT_SECONDS,
)
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research Q&A skill %s failed: %s", skill.name, exc)
continue
if results:
collected.extend(list(results)[:limit])
if not directed or len(collected) >= limit:
return collected[:limit]
if error:
logger.info("Deep Research Q&A skill %s returned no materials: %s", skill.name, error)
return collected[:limit]
def _runnable_research_skills(self) -> list[Any]:
"""Default Q&A trio, or the report-structure 检索来源 when named."""
skills = (
_load_named_research_skills(self._retrieval_skills)
if self._retrieval_skills
else self._research_skills
)
if self._retrieval_skills:
# User-named skills may use any tool; skip the trio's required-tool gate.
return list(skills)
return [
skill
for skill in skills
if _SKILL_REQUIRED_TOOLS.get(skill.name, set()) <= self._question_answer_tool_names
]
async def _search_configured_web(self, query: str) -> list[dict[str, Any]]:
"""Run the existing configurable ``web_search`` adapter when enabled."""
if self._settings is None:
return []
try:
from deerflow.community.configurable_search.tools import execute_search
raw = await execute_search(query, self._settings)
envelope = json.loads(raw)
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research configured web_search failed: %s", exc)
return []
if not isinstance(envelope, dict) or envelope.get("error"):
logger.info("Deep Research configured web_search returned no materials: %s", envelope)
return []
return [item for item in envelope.get("results") or [] if isinstance(item, dict)]
async def _search_via_online_fallback(self, query: str, *, limit: int) -> list[Any]:
"""Use maintained DDG retrieval, then a relevant-result web fallback."""
if not self._online_search_available or limit < 1:
return []
if self._online_search_semaphore is None:
self._online_search_semaphore = asyncio.Semaphore(2)
try:
from deerflow.agents.ai_writing.search.duckduckgo_search import DuckDuckGoSearchProvider
async with self._online_search_semaphore:
provider = DuckDuckGoSearchProvider(max_timeout=ONLINE_SEARCH_TIMEOUT_SECONDS)
ddg_results = await asyncio.wait_for(
provider.search(query, max_results=limit),
timeout=ONLINE_SEARCH_TIMEOUT_SECONDS + 5,
)
relevant_results = _filter_relevant_search_results(ddg_results, query)
if relevant_results:
return relevant_results[:limit]
# DDGS can occasionally return generic login/tracking pages for
# Chinese queries. Do not turn those into report citations;
# collect a second set from the search HTML instead.
return await asyncio.to_thread(_search_360_web, query, limit)
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research online fallback search failed for %r: %s", query, exc)
return []
def _normalise_configured_results(
self,
results: list[dict[str, Any]],
*,
query: str,
domains: list[str] | None,
seen_keys: set[str],
) -> list[ResearchMaterial]:
materials: list[ResearchMaterial] = []
for item in results:
material = result_to_material(item, query=query)
if not _material_matches_domains(material, domains) or not _accept_material(material, seen_keys):
continue
material.metadata["retrieval_channel"] = "configured_web_search"
materials.append(material)
return materials
def _normalise_search_results(
self,
results: list[Any],
*,
query: str,
channel: str,
domains: list[str] | None,
seen_keys: set[str],
) -> list[ResearchMaterial]:
materials: list[ResearchMaterial] = []
for result in results:
material = _search_result_to_material(result, query=query, channel=channel)
if material is None or not _material_matches_domains(material, domains) or not _accept_material(material, seen_keys):
continue
materials.append(material)
return materials
@property
def is_configured(self) -> bool:
"""Whether at least one material channel can run."""
return bool(self.material_channels)
@property
def material_channels(self) -> list[str]:
"""Advertise active Q&A-compatible retrieval channels to the frontend."""
channels: list[str] = []
if any(
_SKILL_REQUIRED_TOOLS.get(skill.name, set()) <= self._question_answer_tool_names
for skill in self._research_skills
):
channels.append("skill")
if self._settings is not None or self._online_search_available:
channels.append("web_search")
return channels
@property
def settings_error(self) -> str | None:
return self._settings_error
def result_to_material(item: dict[str, Any], *, query: str) -> ResearchMaterial:
"""Map a configurable-search result item into a research material.
Public so the chat-collection thread harvester can reuse the exact same
field mapping for ``web_search`` tool envelopes harvested off a thread.
"""
rec_uuid = str(item.get("recUuid") or "").strip() or None
url = str(item.get("url") or "").strip() or None
title = str(item.get("title") or "").strip()
content = str(item.get("content") or "").strip()
source = str(item.get("source") or "").strip() or "web"
raw_hash = hashlib.sha256((url or rec_uuid or content).encode("utf-8")).hexdigest()[:32]
published_at = _try_parse_datetime(str(item.get("publish_time") or item.get("time") or "").strip())
return ResearchMaterial(
id=f"src_{raw_hash[:16]}",
query=query,
title=title,
url=url,
raw_content=content,
snippet=content[:200] if content else None,
source=source,
source_type="web",
published_at=published_at,
rec_uuid=rec_uuid,
content_hash=raw_hash,
)
def _search_result_to_material(result: Any, *, query: str, channel: str) -> ResearchMaterial | None:
"""Map a Q&A/AI-writing SearchResult-like object into ResearchMaterial."""
title = str(getattr(result, "title", "") or "").strip()
content = str(getattr(result, "content", "") or "").strip()
if not title and not content:
return None
url = str(getattr(result, "url", "") or "").strip() or None
rec_uuid = str(getattr(result, "recUuid", "") or "").strip() or None
source = str(getattr(result, "source", "") or "").strip() or channel
source_type = str(getattr(result, "source_type", "") or "").strip()
if source_type not in {"web", "knowledge", "skill", "mcp", "file", "other"}:
source_type = "skill" if channel == "skill" else "web"
raw_hash = hashlib.sha256((url or rec_uuid or title or content).encode("utf-8")).hexdigest()[:32]
return ResearchMaterial(
id=f"src_{raw_hash[:16]}",
query=query,
title=title,
url=url,
raw_content=content,
snippet=content[:200] if content else None,
source=source,
source_type=source_type,
published_at=_try_parse_datetime(str(getattr(result, "published_at", "") or "").strip()),
relevance_score=_safe_float(getattr(result, "relevance_score", None)),
rec_uuid=rec_uuid,
content_hash=raw_hash,
metadata={"retrieval_channel": channel},
)
def _load_question_answer_tool_names(model_name: str | None) -> set[str]:
"""Return exactly the tool names the Q&A agent can currently call."""
try:
from deerflow.tools import get_available_tools
return {str(tool.name) for tool in get_available_tools(model_name=model_name, include_mcp=True)}
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research failed to inspect Q&A tools: %s", exc)
return set()
def _load_question_answer_research_skills() -> list[Any]:
"""Load only enabled Q&A research skills, preserving normal skill state."""
try:
from deerflow.skills import get_or_new_skill_storage
by_name = {skill.name: skill for skill in get_or_new_skill_storage().load_skills(enabled_only=True)}
return [by_name[name] for name in QUESTION_ANSWER_RESEARCH_SKILLS if name in by_name]
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research failed to load Q&A research skills: %s", exc)
return []
def _load_named_research_skills(names: list[str]) -> list[Any]:
"""Load enabled skills by name, preserving the requested order."""
if not names:
return []
try:
from deerflow.skills import get_or_new_skill_storage
by_name = {skill.name: skill for skill in get_or_new_skill_storage().load_skills(enabled_only=True)}
return [by_name[name] for name in names if name in by_name]
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research failed to load named retrieval skills: %s", exc)
return []
def _normalize_named_skills(value: list[str] | None) -> list[str]:
from deerflow.persistence.report_structures.skills import normalize_retrieval_skills
return normalize_retrieval_skills(value)
def _material_matches_domains(material: ResearchMaterial, domains: list[str] | None) -> bool:
"""Apply the requested domain allow-list without dropping URL-less KB hits."""
if not domains or not material.url:
return True
host = (urlparse(material.url).hostname or "").lower()
allowed = {str(domain).lower().strip().lstrip(".") for domain in domains if str(domain).strip()}
return not allowed or any(host == domain or host.endswith(f".{domain}") for domain in allowed)
def _accept_material(material: ResearchMaterial, seen_keys: set[str]) -> bool:
"""Deduplicate cross-channel hits before optional fetch backfill."""
key = material.url or material.rec_uuid or material.content_hash
if not key or key in seen_keys:
return False
seen_keys.add(key)
return True
def _safe_float(value: Any) -> float | None:
try:
return float(value) if value is not None else None
except (TypeError, ValueError):
return None
def _filter_relevant_search_results(results: list[Any], query: str) -> list[Any]:
"""Discard obvious search-engine noise before it reaches report citations."""
tokens = _query_relevance_tokens(query)
if not tokens:
return list(results)
kept: list[Any] = []
for result in results:
haystack = " ".join(
[
str(getattr(result, "title", "") or ""),
str(getattr(result, "content", "") or ""),
str(getattr(result, "url", "") or ""),
]
).lower()
if any(token in haystack for token in tokens):
kept.append(result)
return kept
def _query_relevance_tokens(query: str) -> set[str]:
"""Build compact English/CJK tokens suitable for rejecting blank noise."""
tokens = {token.lower() for token in re.findall(r"[A-Za-z][A-Za-z0-9_-]{2,}", query)}
for chunk in re.findall(r"[\u4e00-\u9fff]+", query):
for width in (4, 3, 2):
tokens.update(chunk[index : index + width] for index in range(max(0, len(chunk) - width + 1)))
return tokens
def _search_360_web(query: str, limit: int) -> list[Any]:
"""Return parsed, real 360 Search snippets when DDG returns no relevant hit."""
try:
import httpx
from bs4 import BeautifulSoup
from deerflow.agents.ai_writing.search.base import SearchResult
response = httpx.get(
"https://www.so.com/s",
params={"q": query},
headers={"User-Agent": "Mozilla/5.0 (compatible; DeerFlow/1.0)"},
timeout=SECONDARY_ONLINE_SEARCH_TIMEOUT_SECONDS,
follow_redirects=True,
)
response.raise_for_status()
soup = BeautifulSoup(response.text, "html.parser")
rows: list[Any] = []
seen_urls: set[str] = set()
for card in soup.select("li.res-list"):
anchor = card.select_one("h3 a") or card.select_one("a.res-link")
if anchor is None:
continue
title = anchor.get_text(" ", strip=True)
url = str(anchor.get("href") or "").strip()
content_node = card.select_one(".res-desc") or card.select_one(".res-rich") or card
content = content_node.get_text(" ", strip=True)
if not title or not content or not url or url in seen_urls:
continue
seen_urls.add(url)
rows.append(
SearchResult(
title=title,
content=content[:500],
source=urlparse(url).hostname or "360 Search",
published_at="",
url=url,
relevance_score=0.7,
source_type="internet",
)
)
if len(rows) >= limit:
break
return _filter_relevant_search_results(rows, query)[:limit]
except Exception as exc: # noqa: BLE001
logger.warning("Deep Research 360 fallback search failed for %r: %s", query, exc)
return []
def _try_parse_datetime(value: str) -> datetime | None:
for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%dT%H:%M:%S"):
try:
return datetime.strptime(value[:19], fmt)
except ValueError:
continue
return None
def _safe_fetch(url: str) -> str:
"""Best-effort full-text fetch via the configured DeerFlow web_fetch tool."""
try:
from deerflow.config import get_app_config
fetch_cfg = get_app_config().get_tool_config("web_fetch")
if fetch_cfg is None or not fetch_cfg.enabled:
return ""
use = fetch_cfg.use or ""
html = ""
if "jina_ai" in use:
from deerflow.community.jina_ai.jina_client import JinaClient
html = JinaClient().crawl(url, return_format="html", timeout=10) or ""
elif "infoquest" in use:
from deerflow.community.infoquest.infoquest_client import InfoQuestClient
html = InfoQuestClient().fetch(url, return_format="html") or ""
if not html or html.startswith("Error"):
return ""
from deerflow.utils.readability import ReadabilityExtractor
return ReadabilityExtractor().extract_article(html).to_markdown(including_title=True) or ""
except Exception as exc: # noqa: BLE001
logger.debug("Deep Research material fetch failed for %s: %s", url, exc)
return ""
__all__ = [
"DeerFlowMaterialProvider",
"ONLINE_SEARCH_TIMEOUT_SECONDS",
"QUESTION_ANSWER_RESEARCH_SKILLS",
"SNIPPET_THRESHOLD",
"result_to_material",
]