sync update from LAN repo (source dhk-1 @ 3e87be55)

This commit is contained in:
gozeros committed 2026-10-21 15:06:08 +08:00
1 parent ab6068a7c1
commit 42f2a26fa8
1482 files changed
+227759 -9584

No files matched your search

@@ -31,7 +31,7 @@ from typing import Any, Literal
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.encoders import jsonable_encoder
from fastapi.responses import FileResponse, StreamingResponse
from pydantic import BaseModel, ConfigDict, Field
from pydantic import BaseModel, ConfigDict, Field, ValidationError
from app.gateway.deps import get_current_user
from app.gateway.document_rewrite_summary import build_document_rewrite_summary
@@ -41,6 +41,7 @@ from deerflow.agents.deep_research.config import (
DEFAULT_MAX_ACTIVE_JOBS_PER_USER,
MAX_GENERATED_IMAGES,
DeepResearchConfig,
config_from_dict,
)
from deerflow.persistence.deep_research_jobs.sql import (
TERMINAL_STATUSES,
@@ -51,9 +52,12 @@ logger = logging.getLogger(__name__)
_POLL_INTERVAL_S = 0.8
_LIVE_WAIT_S = 0.2
# Rows drained per durable replay poll. A full page means the cursor has not
# caught up yet, so the terminal transition has to wait (see ``stream_job``).
_DURABLE_PAGE_SIZE = 200
_STATUS_POLL_TIMEOUT_S = 1.0
# Idle harvest / outline / thinking can emit nothing for tens of seconds.
# Nginx and other proxies kill those silent SSE connections; comment pings
# plus a 2KiB pad keep the socket alive without becoming progress events.
_SSE_PING_INTERVAL_S = 8.0
_SSE_FLUSH_PADDING = " " * 2_048
_REPORT_IMAGE_REF_RE = re.compile(r"deep-research://([A-Za-z0-9._-]+)")
_INLINE_REPORT_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(deep-research://([A-Za-z0-9._-]+)\)")
# Reports produced by the research writer can contain either its in-progress
@@ -280,6 +284,32 @@ class UpdateSessionRequest(BaseModel):
default=None,
alias="collectorMaterials",
)
# After collection, the confirmation form asks the session to plan a
# topic-specific outline and write it into config_snapshot.custom_outline.
# Kept off the status flip so the card can appear immediately.
plan_outline: bool = Field(default=False, alias="planOutline")
model_name: str | None = Field(default=None, alias="modelName", max_length=160)
force_outline: bool = Field(default=False, alias="force")
custom_outline: str | None = Field(default=None, alias="customOutline", max_length=4_000)
class IntentOutlineRequest(BaseModel):
"""Generate a topic-specific outline after chat collection finishes."""
model_config = ConfigDict(populate_by_name=True)
model_name: str | None = Field(default=None, alias="modelName", max_length=160)
collector_materials: list[dict[str, Any]] | None = Field(
default=None,
alias="collectorMaterials",
)
force: bool = False
class IntentOutlineResponse(BaseModel):
outline: str
source: str = "intent"
reused: bool = False
class StartJobRequest(BaseModel):
@@ -850,6 +880,7 @@ async def get_capabilities() -> dict[str, Any]:
"maxActiveJobsPerUser": DEFAULT_MAX_ACTIVE_JOBS_PER_USER,
# The built-in collector agent the chat collection flow talks to.
"collectionAgentId": "deep-research-collector",
"outlinePlannerAgentId": "deep-research-planner",
}
@@ -861,8 +892,13 @@ async def create_session(body: CreateSessionRequest, request: Request) -> Sessio
user_id = await _require_user(request)
store = _get_session_store(request)
# Build + clamp the config.
config = DeepResearchConfig.model_validate(body.config or {}).clamp()
# Build + clamp the config. Unknown keys (newer clients / persistence
# metadata) are stripped; invalid known fields become 422, not 500.
try:
config = config_from_dict(body.config)
except ValidationError as exc:
logger.warning("deep-research create_session config rejected: %s", exc)
raise HTTPException(status_code=422, detail="研究配置不合法,请刷新页面后重试。") from exc
if config.mode not in ("basic", "quick", "detailed", "deep", "multi_agent"):
raise HTTPException(status_code=422, detail=f"Mode '{config.mode}' is not available")
@@ -970,7 +1006,7 @@ async def list_sessions(
visible = [
row
for row in unfiltered
if not bool((row.get("config_snapshot") or {}).get("report_variant"))
if not bool((row.get("config_snapshot") if isinstance(row.get("config_snapshot"), dict) else {}).get("report_variant"))
]
items = visible[offset : offset + limit]
total = len(visible)
@@ -1067,10 +1103,202 @@ async def update_session(session_id: str, body: UpdateSessionRequest, request: R
fields["source_count"] = int(row.get("source_count") or 0) + persisted
else:
fields["source_count"] = int(row.get("source_count") or 0) + persisted
if not fields:
if body.custom_outline is not None:
snapshot = dict(row.get("config_snapshot") or {})
snapshot["custom_outline"] = str(body.custom_outline).strip()[:4_000]
fields["config_snapshot"] = snapshot
should_plan = body.plan_outline
if fields:
row = await store.update(session_id, user_id=user_id, **fields) or row
if should_plan:
try:
await _generate_and_store_intent_outline(
request,
session_id=session_id,
user_id=user_id,
row=row,
model_name=body.model_name,
collector_materials=list(body.collector_materials or []),
force=body.force_outline,
)
row = await store.get(session_id, user_id=user_id) or row
except HTTPException:
# Explicit 「重新规划」 must surface the error in the form.
# Auto-plan on the awaiting_report flip must not block that flip.
if body.plan_outline:
raise
logger.warning("intent outline planning skipped for session %s", session_id)
if not fields and not should_plan:
return _session_to_response(row, full=True)
updated = await store.update(session_id, user_id=user_id, **fields)
return _session_to_response(updated or row, full=True)
return _session_to_response(row, full=True)
@router.api_route(
"/sessions/{session_id}/intent-outline",
methods=["POST", "PATCH"],
response_model=IntentOutlineResponse,
)
async def plan_intent_outline(
session_id: str,
body: IntentOutlineRequest,
request: Request,
) -> IntentOutlineResponse:
"""Plan a report structure from the topic and collected materials.
Used after chat collection when the user did not pick a preset structure.
The outline is written into the session snapshot so the confirmation form
can show it for editing before「开始撰写」.
"""
user_id = await _require_user(request)
store = _get_session_store(request)
row = await store.get(session_id, user_id=user_id)
if row is None:
raise HTTPException(status_code=404, detail="Session not found")
outline, reused = await _generate_and_store_intent_outline(
request,
session_id=session_id,
user_id=user_id,
row=row,
model_name=body.model_name,
collector_materials=list(body.collector_materials or []),
force=body.force,
)
return IntentOutlineResponse(
outline=outline,
source="session" if reused else "intent",
reused=reused,
)
async def _generate_and_store_intent_outline(
request: Request,
*,
session_id: str,
user_id: str,
row: dict[str, Any],
model_name: str | None,
collector_materials: list[dict[str, Any]],
force: bool,
) -> tuple[str, bool]:
store = _get_session_store(request)
snapshot = dict(row.get("config_snapshot") or {})
existing = str(snapshot.get("custom_outline") or "").strip()
if existing and not force:
return existing[:4_000], True
from deerflow.agents.deep_research.runners.basic import (
_SYSTEM,
build_intent_outline_prompt,
extract_generated_outline,
)
from deerflow.models import create_chat_model
from langchain_core.messages import HumanMessage, SystemMessage
context_text = await _outline_material_digest(
request,
session_id=session_id,
user_id=user_id,
query=str(row.get("query") or ""),
collector_materials=collector_materials,
)
resolved_model = (model_name or "").strip() or _session_model_name(snapshot)
app_config = getattr(request.app.state, "config", None)
try:
model = create_chat_model(
name=resolved_model or None,
thinking_enabled=False,
app_config=app_config,
force_disable_thinking=True,
)
response = await asyncio.wait_for(
model.ainvoke(
[
SystemMessage(content=_SYSTEM),
HumanMessage(content=build_intent_outline_prompt(str(row.get("query") or ""), context_text)),
]
),
timeout=45,
)
except Exception as exc: # noqa: BLE001 - the form stays editable; write-time still falls back
logger.warning("intent outline planning failed for session %s: %s", session_id, type(exc).__name__)
raise HTTPException(
status_code=422,
detail="未能根据课题生成报告结构,请手动填写或直接开始撰写。",
) from exc
outline = extract_generated_outline(_model_text(getattr(response, "content", None) if response is not None else None))
if not outline:
raise HTTPException(status_code=422, detail="模型未返回可用的报告结构,请手动填写或重试。")
outline = outline[:4_000]
snapshot["custom_outline"] = outline
await store.update(session_id, user_id=user_id, config_snapshot=snapshot)
return outline, False
def _model_text(content: Any) -> str:
if isinstance(content, str):
return content
if isinstance(content, list):
parts: list[str] = []
for block in content:
if isinstance(block, str):
parts.append(block)
elif isinstance(block, dict):
text = block.get("text")
if isinstance(text, str):
parts.append(text)
else:
text = getattr(block, "text", None)
if isinstance(text, str):
parts.append(text)
return "".join(parts)
return str(content or "")
def _session_model_name(snapshot: dict[str, Any]) -> str | None:
for key in ("fast_model", "smart_model", "strategic_model", "model_name"):
value = snapshot.get(key)
if isinstance(value, str) and value.strip():
return value.strip()
return None
async def _outline_material_digest(
request: Request,
*,
session_id: str,
user_id: str,
query: str,
collector_materials: list[dict[str, Any]],
) -> str:
parts: list[str] = []
source_store = getattr(request.app.state, "deep_research_source_store", None)
if source_store is not None:
try:
rows = await source_store.list_by_session(
session_id,
user_id=user_id,
limit=40,
include_content=True,
)
except Exception: # noqa: BLE001 - browser snapshot is enough for planning
rows = []
for index, item in enumerate(rows, 1):
title = str(item.get("title") or f"来源{index}").strip()
body = str(item.get("snippet") or item.get("raw_content") or "").strip()
parts.append(f"{title}\n{body[:280]}" if body else title)
if not parts and collector_materials:
from app.gateway.deep_research_job_executor import _materials_from_browser_snapshot
try:
materials = _materials_from_browser_snapshot(collector_materials, query=query)
except Exception: # noqa: BLE001
materials = []
for index, material in enumerate(materials[:40], 1):
title = str(getattr(material, "title", None) or f"来源{index}").strip()
body = str(getattr(material, "raw_content", None) or getattr(material, "snippet", None) or "").strip()
parts.append(f"{title}\n{body[:280]}" if body else title)
return "\n\n".join(parts)
@router.delete("/sessions/{session_id}", status_code=204)
@@ -1283,7 +1511,7 @@ async def start_job(
# latter swaps network collection for the already selected source pool.
merged = {**config_dict, **body.config}
try:
merged_config = DeepResearchConfig.model_validate(merged).clamp()
merged_config = config_from_dict(merged)
except Exception as exc: # noqa: BLE001
raise HTTPException(status_code=422, detail=f"Invalid report config: {exc}") from None
config_dict = merged_config.model_dump()
@@ -1329,19 +1557,9 @@ async def start_job(
)
diagnostics: list[dict[str, Any]] = []
persisted_source_count = 0
# Keep the compact browser list on the job. First-write reads this list.
# Also copy it into the source table so refresh / regenerate still have
# rows after the collector chat is gone from memory.
if body.entry == "chat" and collector_materials:
persisted_source_count, persist_diagnostics = await _persist_browser_collector_materials(
request,
session_id=session_id,
user_id=user_id,
job_id=job_id,
query=query,
collector_materials=collector_materials,
)
diagnostics.extend(persist_diagnostics)
# Do not wait on the source table before dispatching. That upsert is the
# production hang before the first write frame. The job snapshot is the
# writing input; leftover rows can persist in the background.
snapshot_materials = (
list(collector_materials or [])[:WRITE_FALLBACK_MATERIAL_LIMIT]
if body.entry == "chat"
@@ -1370,22 +1588,24 @@ async def start_job(
input_snapshot=input_snapshot,
)
reused_job_id = str(job["id"])
if (
not created
and body.entry == "chat"
and collector_materials
and reused_job_id != job_id
):
extra_count, extra_diagnostics = await _persist_browser_collector_materials(
request,
session_id=session_id,
user_id=user_id,
job_id=reused_job_id,
query=query,
collector_materials=collector_materials,
)
persisted_source_count = max(persisted_source_count, extra_count)
diagnostics.extend(extra_diagnostics)
if body.entry == "chat" and collector_materials:
persist_job_id = reused_job_id
async def _persist_sources_after_write_starts() -> None:
try:
await _persist_browser_collector_materials(
request,
session_id=session_id,
user_id=user_id,
job_id=persist_job_id,
query=query,
collector_materials=collector_materials,
)
except Exception: # noqa: BLE001 - writing already has the snapshot
logger.warning("background collector material persist failed", exc_info=True)
with suppress(RuntimeError):
asyncio.get_running_loop().create_task(_persist_sources_after_write_starts())
if (
not created
and body.entry == "chat"
@@ -1510,6 +1730,8 @@ async def start_job(
# queued until the next periodic scan and make the UI appear stuck.
dispatcher = _get_dispatcher(request)
if dispatcher is not None:
with suppress(Exception):
dispatcher.ensure_started()
dispatcher.nudge()
else:
diagnostics.append(
@@ -1703,6 +1925,8 @@ async def start_full_report_rewrite_job(
dispatcher = _get_dispatcher(request)
if dispatcher is None:
raise HTTPException(status_code=503, detail="Deep research job dispatcher not available")
with suppress(Exception):
dispatcher.ensure_started()
if created:
dispatcher.nudge()
return {
@@ -1761,7 +1985,7 @@ async def stream_job(
"""
user_id = await _require_user(request)
job_store = _get_job_store(request)
event_store = _get_event_store(request)
_ = after
# Verify ownership.
job = await job_store.get(job_id, user_id=user_id)
@@ -1769,110 +1993,83 @@ async def stream_job(
raise HTTPException(status_code=404, detail="Job not found")
session_id = job.get("session_id", "")
live_hub = getattr(request.app.state, "deep_research_live_hub", None)
# Opening the stream also wakes dispatch. A dead loop after restart would
# otherwise leave the job queued while the UI waits on an empty SSE.
dispatcher = _get_dispatcher(request)
if dispatcher is not None:
with suppress(Exception):
dispatcher.ensure_started()
with suppress(Exception):
dispatcher.nudge()
async def gen():
cursor = after
live_queue = live_hub.subscribe(job_id) if live_hub is not None else None
loop = asyncio.get_running_loop()
deadline = loop.time() + 30 * 60
next_durable_poll = 0.0
last_bytes = 0.0
next_status_poll = 0.0
def ping_frame() -> str:
return f": ping{_SSE_FLUSH_PADDING}\n\n"
try:
# Live frames first. Never wait on the event table to show writing.
yield ping_frame()
last_bytes = loop.time()
while loop.time() < deadline:
if await request.is_disconnected():
return
if loop.time() >= next_durable_poll:
# Durable replay repairs a missed live delta after
# reconnect or when another worker owned the job.
if live_queue is not None:
try:
events = await event_store.list_after(
job_id, after=cursor, limit=_DURABLE_PAGE_SIZE
live_event = await asyncio.wait_for(live_queue.get(), timeout=_LIVE_WAIT_S)
yield format_sse(
"progress",
live_event,
event_id=str(live_event.get("liveId") or live_event.get("seq") or "live"),
)
for ev in events:
payload = {
"seq": ev["seq"],
"sessionId": session_id,
"jobId": job_id,
"type": ev["event_type"],
"phase": ev.get("phase", "initializing"),
"timestamp": _iso(ev.get("created_at")),
"payload": ev.get("payload") or {},
}
yield format_sse("progress", payload, event_id=str(ev["seq"]))
cursor = max(cursor, ev["seq"])
if len(events) >= _DURABLE_PAGE_SIZE:
# A full page means more rows are already waiting.
# Drain them before looking at the job status: an
# already-terminal job would otherwise close the
# stream with part of its own replay unsent, and
# the client would render a truncated report that
# still ends in a clean `completed` frame.
next_durable_poll = loop.time()
continue
current = await job_store.get_unscoped(job_id)
if current and current.get("status") in TERMINAL_STATUSES:
yield format_sse(
"progress",
{
"seq": cursor + 1,
"sessionId": session_id,
"jobId": job_id,
"type": "job_status",
"phase": current.get("phase", "done"),
"timestamp": _iso(datetime.now(UTC)),
"payload": {
"status": current["status"],
"progress": current.get("progress", 0),
"errorCode": current.get("error_code"),
"errorMessage": current.get("error_message"),
},
},
event_id=str(cursor + 1),
)
return
next_durable_poll = loop.time() + _POLL_INTERVAL_S
except asyncio.CancelledError:
raise
except Exception: # noqa: BLE001
# Do not let one read-replica / transient DB failure
# abort the ASGI generator (which surfaces as an
# ExceptionGroup and leaves the page stuck). Live
# frames can still reach an attached browser, and the
# next durable poll repairs the cursor on recovery.
last_bytes = loop.time()
continue
except TimeoutError:
pass
if loop.time() - last_bytes >= _SSE_PING_INTERVAL_S:
yield ping_frame()
last_bytes = loop.time()
if loop.time() >= next_status_poll:
current = None
try:
current = await asyncio.wait_for(
job_store.get_unscoped(job_id),
timeout=_STATUS_POLL_TIMEOUT_S,
)
except Exception: # noqa: BLE001 - keep the live stream open
logger.warning(
"deep-research SSE durable replay temporarily failed for job %s; keeping stream open",
"deep-research SSE status poll failed for job %s; keeping stream open",
job_id,
exc_info=True,
)
if current and current.get("status") in TERMINAL_STATUSES:
yield format_sse(
"progress",
{
"seq": 0,
"liveId": f"durable-replay-retry:{job_id}:{int(loop.time() * 1000)}",
"seq": 1,
"sessionId": session_id,
"jobId": job_id,
"type": "warning",
"phase": "initializing",
"type": "job_status",
"phase": current.get("phase", "done"),
"timestamp": _iso(datetime.now(UTC)),
"payload": {
"code": "durable_replay_temporarily_unavailable",
"message": "进度同步暂时不可用,任务仍在继续;正在自动重试。",
"recoverable": True,
"status": current["status"],
"progress": current.get("progress", 0),
"errorCode": current.get("error_code"),
"errorMessage": current.get("error_message"),
},
},
event_id="job-status",
)
next_durable_poll = loop.time() + max(_POLL_INTERVAL_S, 3.0)
return
next_status_poll = loop.time() + _POLL_INTERVAL_S
if live_queue is None:
await asyncio.sleep(max(0.01, next_durable_poll - loop.time()))
continue
try:
timeout = min(_LIVE_WAIT_S, max(0.01, next_durable_poll - loop.time()))
live_event = await asyncio.wait_for(live_queue.get(), timeout=timeout)
except TimeoutError:
continue
# Persisted envelopes may arrive through the hub too; frontend
# deduplicates those by seq. Native report_delta frames use
# seq=0 + liveId and are intentionally not in the DB log.
yield format_sse("progress", live_event, event_id=str(live_event.get("liveId") or live_event.get("seq") or "live"))
await asyncio.sleep(max(0.01, min(_LIVE_WAIT_S, _POLL_INTERVAL_S)))
finally:
if live_queue is not None:
live_hub.unsubscribe(job_id, live_queue)
@@ -1881,7 +2078,7 @@ async def stream_job(
gen(),
media_type="text/event-stream",
headers={
"Cache-Control": "no-cache",
"Cache-Control": "no-cache, no-transform",
"Connection": "keep-alive",
"X-Accel-Buffering": "no",
},
@@ -1929,6 +2126,8 @@ async def resume_job(job_id: str, body: ResumeJobRequest, request: Request) -> d
raise HTTPException(status_code=409, detail="Job could not be resumed")
dispatcher = _get_dispatcher(request)
if dispatcher is not None:
with suppress(Exception):
dispatcher.ensure_started()
dispatcher.nudge()
session_store = _get_session_store(request)
await session_store.update(row["session_id"], user_id=user_id, status="running", active_job_id=job_id)