feat(deep-research): 强化任务链路可靠性并接入 agentfx 报告助手种子
功能说明:\n- 重构 Deep Research 调度与执行链,补齐 dispatcher、executor、live hub 与依赖注入入口,提升任务下发、执行、回调的稳定性\n- 增强运行时持久化与恢复能力,接入业务映射、任务来源与任务事件日志模型,覆盖持久化与执行可靠性关键路径测试\n- 新增 agentfx 报告助手 seed 与任务报告导入脚本配置,支持离线环境快速初始化与任务流模板回填\n- 完善 deep-research 与 thread_runs 路由参数编排,兼容不同业务映射与运行入口的前后端联动\n- 补充前端 agentfx deep-link 解析,保证 deep-research 场景下的登录跳转与任务上下文一致
This commit is contained in:
1 parent
438772f0b7
commit
5c75e1af16
36 files changed
+3163
-312
No files matched your search
@@ -21,15 +21,17 @@ import hashlib
|
||||
import json
|
||||
import logging
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query, Request
|
||||
from fastapi.encoders import jsonable_encoder
|
||||
from fastapi.responses import FileResponse, StreamingResponse
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from app.gateway.deps import get_current_user
|
||||
from app.gateway.document_rewrite_summary import build_document_rewrite_summary
|
||||
@@ -49,6 +51,9 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
_POLL_INTERVAL_S = 0.8
|
||||
_LIVE_WAIT_S = 0.2
|
||||
# Rows drained per durable replay poll. A full page means the cursor has not
|
||||
# caught up yet, so the terminal transition has to wait (see ``stream_job``).
|
||||
_DURABLE_PAGE_SIZE = 200
|
||||
_REPORT_IMAGE_REF_RE = re.compile(r"deep-research://([A-Za-z0-9._-]+)")
|
||||
_INLINE_REPORT_IMAGE_RE = re.compile(r"!\[([^\]]*)\]\(deep-research://([A-Za-z0-9._-]+)\)")
|
||||
# Reports produced by the research writer can contain either its in-progress
|
||||
@@ -62,6 +67,9 @@ _REPORT_SOURCE_MARKER_RE = re.compile(
|
||||
)
|
||||
_MAX_HTML_EXPORT_IMAGE_BYTES = 20 * 1024 * 1024
|
||||
_RUNTIME_DIAGNOSTICS_REVISION = "deep-research-observability-20260903-v1"
|
||||
# First-write job snapshot only carries a small fallback. The full collected
|
||||
# pool lives in the source table (persisted when collection finishes).
|
||||
WRITE_FALLBACK_MATERIAL_LIMIT = 20
|
||||
|
||||
|
||||
# ── store getters ───────────────────────────────────────────────────────────
|
||||
@@ -107,6 +115,136 @@ def _get_dispatcher(request: Request):
|
||||
return getattr(request.app.state, "deep_research_dispatcher", None)
|
||||
|
||||
|
||||
def _status_after_job_stop(session: dict[str, Any]) -> str:
|
||||
"""Free the concurrency slot without deleting a chat-collection draft."""
|
||||
config = session.get("config_snapshot") or {}
|
||||
if not isinstance(config, dict):
|
||||
config = {}
|
||||
if session.get("runtime_thread_id") or config.get("collection_mode") == "chat":
|
||||
return "awaiting_report"
|
||||
if str(session.get("report_markdown") or "").strip():
|
||||
return "completed"
|
||||
return "cancelled"
|
||||
|
||||
|
||||
def _pid_is_alive(pid: int) -> bool:
|
||||
"""Best-effort check that ``pid`` still exists on this machine."""
|
||||
if pid <= 0:
|
||||
return False
|
||||
if os.name == "nt":
|
||||
try:
|
||||
import ctypes
|
||||
|
||||
# Avoid os.kill on Windows: signal 0 against a bogus pid can raise
|
||||
# ERROR_INVALID_PARAMETER and disturb the pytest asyncio loop.
|
||||
process_query_limited_information = 0x1000
|
||||
handle = ctypes.windll.kernel32.OpenProcess(process_query_limited_information, False, pid)
|
||||
if handle:
|
||||
ctypes.windll.kernel32.CloseHandle(handle)
|
||||
return True
|
||||
return ctypes.windll.kernel32.GetLastError() == 5 # ERROR_ACCESS_DENIED
|
||||
except Exception: # noqa: BLE001
|
||||
return False
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except PermissionError:
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _lease_owner_process_alive(lease_owner: Any) -> bool:
|
||||
"""Whether the worker pid encoded in ``w-{pid}-{hex}`` is still running.
|
||||
|
||||
After a Gateway restart the MySQL rows stay ``running`` for up to the
|
||||
lease TTL (3 minutes). Those rows are not actually executing, but they
|
||||
still trip the per-user 429 latch. A dead pid is a reliable signal that
|
||||
this process (or another live uvicorn worker) is not that owner.
|
||||
"""
|
||||
parts = str(lease_owner or "").split("-")
|
||||
if len(parts) < 3 or parts[0] != "w":
|
||||
return False
|
||||
try:
|
||||
pid = int(parts[1])
|
||||
except ValueError:
|
||||
return False
|
||||
return _pid_is_alive(pid)
|
||||
|
||||
|
||||
def _job_is_stale(job: dict[str, Any] | None, *, now: datetime) -> bool:
|
||||
if job is None:
|
||||
return True
|
||||
if job.get("status") in TERMINAL_STATUSES:
|
||||
return True
|
||||
if job.get("cancel_requested"):
|
||||
return True
|
||||
if job.get("status") == "running":
|
||||
lease_until = job.get("lease_until")
|
||||
if lease_until is None or lease_until < now:
|
||||
return True
|
||||
return not _lease_owner_process_alive(job.get("lease_owner"))
|
||||
return False
|
||||
|
||||
|
||||
async def _park_session_after_job_stop(
|
||||
request: Request,
|
||||
*,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
) -> None:
|
||||
store = _get_session_store(request)
|
||||
row = await store.get(session_id, user_id=user_id)
|
||||
if row is None:
|
||||
return
|
||||
if row.get("status") not in ("running", "awaiting_input"):
|
||||
if row.get("active_job_id"):
|
||||
await store.update(session_id, user_id=user_id, active_job_id=None)
|
||||
return
|
||||
await store.update(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
status=_status_after_job_stop(row),
|
||||
active_job_id=None,
|
||||
)
|
||||
|
||||
|
||||
async def _release_stale_in_flight_sessions(request: Request, user_id: str) -> int:
|
||||
"""Drop paused/zombie sessions so a restart cannot keep the 429 latch closed."""
|
||||
session_store = _get_session_store(request)
|
||||
job_store = _get_job_store(request)
|
||||
list_by_user = getattr(session_store, "list_by_user", None)
|
||||
if not callable(list_by_user):
|
||||
return 0
|
||||
now = datetime.now(UTC)
|
||||
released = 0
|
||||
for status in ("running", "awaiting_input"):
|
||||
try:
|
||||
rows = await list_by_user(user_id=user_id, status=status, limit=100)
|
||||
except TypeError:
|
||||
rows = await list_by_user(user_id=user_id, status=status)
|
||||
except Exception: # noqa: BLE001 - a list failure must not block a new write
|
||||
logger.warning("could not list %s deep-research sessions for stale release", status, exc_info=True)
|
||||
continue
|
||||
for summary in rows or []:
|
||||
session_id = str(summary.get("id") or "")
|
||||
if not session_id:
|
||||
continue
|
||||
row = await session_store.get(session_id, user_id=user_id) or summary
|
||||
job_id = str(row.get("active_job_id") or "")
|
||||
job = await job_store.get(job_id, user_id=user_id) if job_id else None
|
||||
if not _job_is_stale(job, now=now):
|
||||
continue
|
||||
if job is not None and job.get("status") not in TERMINAL_STATUSES:
|
||||
with suppress(Exception):
|
||||
await job_store.request_cancel(job_id, user_id=user_id)
|
||||
with suppress(Exception):
|
||||
await job_store.finalize_cancel(job_id)
|
||||
await _park_session_after_job_stop(request, session_id=session_id, user_id=user_id)
|
||||
released += 1
|
||||
return released
|
||||
|
||||
|
||||
def _get_thread_store(request: Request):
|
||||
return getattr(request.app.state, "thread_store", None)
|
||||
|
||||
@@ -129,13 +267,24 @@ class CreateSessionRequest(BaseModel):
|
||||
|
||||
|
||||
class UpdateSessionRequest(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
title: str | None = None
|
||||
# Chat-collection lifecycle transitions driven by the frontend state
|
||||
# machine (turn finished → awaiting the user's report confirmation).
|
||||
status: str | None = None
|
||||
# Compact collector evidence frozen when the chat turn ends. Persisting it
|
||||
# here means a refresh still has rows in the source table before the user
|
||||
# clicks 「开始撰写」 or 「选择结构再生成」.
|
||||
collector_materials: list[dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
alias="collectorMaterials",
|
||||
)
|
||||
|
||||
|
||||
class StartJobRequest(BaseModel):
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
request_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
# ``chat`` — materials were already collected by the collector-agent
|
||||
# conversation on the runtime thread; the job harvests and writes only.
|
||||
@@ -147,7 +296,37 @@ class StartJobRequest(BaseModel):
|
||||
# Browser-side compact evidence snapshot for chat collection. This removes
|
||||
# the assumption that the report worker shares the collector worker's local
|
||||
# checkpointer and remains small enough for ordinary reverse-proxy limits.
|
||||
collector_materials: list[dict[str, Any]] | None = None
|
||||
# Accept both snake_case and the frontend camelCase name so a proxy or
|
||||
# older client cannot silently drop the only cross-worker evidence.
|
||||
collector_materials: list[dict[str, Any]] | None = Field(
|
||||
default=None,
|
||||
alias="collectorMaterials",
|
||||
)
|
||||
query: str | None = Field(default=None, max_length=4_000)
|
||||
title: str | None = Field(default=None, max_length=500)
|
||||
force_no_materials: bool = Field(default=False, alias="forceNoMaterials")
|
||||
|
||||
|
||||
class WriteReportRequest(BaseModel):
|
||||
"""First-write request.
|
||||
|
||||
``GET /jobs/{id}/stream`` only replays progress. The writer prefers the
|
||||
source table. ``collector_materials`` is a small fallback (capped at 20)
|
||||
used only when that table is empty. An empty list (or ``forceNoMaterials``)
|
||||
selects the no-material writer.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(populate_by_name=True)
|
||||
|
||||
request_id: str = Field(default_factory=lambda: str(uuid.uuid4()))
|
||||
query: str | None = Field(default=None, max_length=4_000)
|
||||
title: str | None = Field(default=None, max_length=500)
|
||||
config: dict[str, Any] | None = None
|
||||
collector_materials: list[dict[str, Any]] = Field(
|
||||
default_factory=list,
|
||||
alias="collectorMaterials",
|
||||
)
|
||||
force_no_materials: bool = Field(default=False, alias="forceNoMaterials")
|
||||
|
||||
|
||||
class ResumeJobRequest(BaseModel):
|
||||
@@ -417,17 +596,20 @@ async def _repair_invalid_persisted_report(
|
||||
"""Repair a refusal report that an older/bypassing worker marked complete.
|
||||
|
||||
The executor has the primary terminal gate. This read-boundary gate covers
|
||||
already persisted history and any execution path that incorrectly reaches
|
||||
``completed`` without going through the current executor guard.
|
||||
already persisted history. It must never touch a live job: rewriting
|
||||
``report_markdown``, clearing ``active_job_id`` and writing ``error`` while
|
||||
a worker is still streaming is what made the sandbox open, show a few
|
||||
lines, then roll back or look failed.
|
||||
"""
|
||||
if row.get("status") not in ("completed", "failed"):
|
||||
return row
|
||||
if row.get("active_job_id"):
|
||||
return row
|
||||
report = str(row.get("report_markdown") or "").strip()
|
||||
if not report:
|
||||
return row
|
||||
|
||||
from app.gateway.deep_research_job_executor import (
|
||||
EXECUTOR_REVISION,
|
||||
_terminal_report_validation_error,
|
||||
)
|
||||
from app.gateway.deep_research_job_executor import _terminal_report_validation_error
|
||||
from deerflow.agents.deep_research.runners.basic import _build_report_fallback
|
||||
|
||||
validation_error = _terminal_report_validation_error(report)
|
||||
@@ -436,7 +618,6 @@ async def _repair_invalid_persisted_report(
|
||||
|
||||
session_id = str(row["id"])
|
||||
rows: list[dict[str, Any]] = []
|
||||
source_read_error: str | None = None
|
||||
try:
|
||||
rows = await _get_source_store(request).list_by_session(
|
||||
session_id,
|
||||
@@ -445,8 +626,8 @@ async def _repair_invalid_persisted_report(
|
||||
limit=500,
|
||||
include_content=True,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - an empty-source fallback is valid
|
||||
source_read_error = f"{exc.__class__.__name__}: {exc}"[:1200]
|
||||
except Exception: # noqa: BLE001 - an empty-source fallback is valid
|
||||
logger.warning("session_read_repair could not list sources for %s", session_id, exc_info=True)
|
||||
|
||||
fallback_parts: list[str] = []
|
||||
for index, source in enumerate(rows, 1):
|
||||
@@ -457,12 +638,6 @@ async def _repair_invalid_persisted_report(
|
||||
source_content = str(source.get("raw_content") or source.get("snippet") or "")
|
||||
fallback_parts.append(f"{source_header}\n{source_content}")
|
||||
|
||||
repair_detail = (
|
||||
f"[session_read_repair revision={EXECUTOR_REVISION}] "
|
||||
f"{validation_error}; original={report[:800]}"
|
||||
)
|
||||
if source_read_error:
|
||||
repair_detail += f"; sourceReadError={source_read_error}"
|
||||
replacement = _build_report_fallback(
|
||||
str(row.get("query") or "研究课题"),
|
||||
"\n\n---\n\n".join(fallback_parts),
|
||||
@@ -474,10 +649,9 @@ async def _repair_invalid_persisted_report(
|
||||
report_markdown=replacement,
|
||||
report_html=None,
|
||||
source_count=max(int(row.get("source_count") or 0), len(rows)),
|
||||
active_job_id=None,
|
||||
error=repair_detail,
|
||||
error=None,
|
||||
)
|
||||
return updated or {**row, "report_markdown": replacement, "error": repair_detail}
|
||||
return updated or {**row, "report_markdown": replacement, "error": None}
|
||||
|
||||
|
||||
def _job_to_response(row: dict[str, Any]) -> JobResponse:
|
||||
@@ -816,11 +990,15 @@ async def get_session(session_id: str, request: Request) -> SessionResponse:
|
||||
if row is None:
|
||||
raise HTTPException(status_code=404, detail="Session not found")
|
||||
row = await _repair_invalid_persisted_report(request, row, user_id=user_id)
|
||||
runtime_diagnostics = await _build_session_runtime_diagnostics(
|
||||
request,
|
||||
row,
|
||||
user_id=user_id,
|
||||
)
|
||||
runtime_diagnostics = None
|
||||
if row.get("status") in ("failed", "cancelled") or (
|
||||
row.get("status") == "completed" and not str(row.get("report_markdown") or "").strip()
|
||||
):
|
||||
runtime_diagnostics = await _build_session_runtime_diagnostics(
|
||||
request,
|
||||
row,
|
||||
user_id=user_id,
|
||||
)
|
||||
return _session_to_response(row, full=True, runtime_diagnostics=runtime_diagnostics)
|
||||
|
||||
|
||||
@@ -868,6 +1046,27 @@ async def update_session(session_id: str, body: UpdateSessionRequest, request: R
|
||||
if row.get("active_job_id"):
|
||||
raise HTTPException(status_code=409, detail="A research job owns this session's status")
|
||||
fields["status"] = body.status
|
||||
if body.collector_materials:
|
||||
persisted, _persist_diagnostics = await _persist_browser_collector_materials(
|
||||
request,
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
job_id=str(row.get("active_job_id") or "") or None,
|
||||
query=str(row.get("query") or ""),
|
||||
collector_materials=list(body.collector_materials),
|
||||
)
|
||||
if persisted:
|
||||
source_store = _get_source_store(request)
|
||||
count_by_session = getattr(source_store, "count_by_session", None)
|
||||
if callable(count_by_session):
|
||||
try:
|
||||
fields["source_count"] = int(
|
||||
await count_by_session(session_id, user_id=user_id)
|
||||
)
|
||||
except Exception: # noqa: BLE001 - keep the in-memory fallback
|
||||
fields["source_count"] = int(row.get("source_count") or 0) + persisted
|
||||
else:
|
||||
fields["source_count"] = int(row.get("source_count") or 0) + persisted
|
||||
if not fields:
|
||||
return _session_to_response(row, full=True)
|
||||
updated = await store.update(session_id, user_id=user_id, **fields)
|
||||
@@ -904,8 +1103,149 @@ async def delete_session(session_id: str, request: Request) -> None:
|
||||
# ── job lifecycle ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _persist_browser_collector_materials(
|
||||
request: Request,
|
||||
*,
|
||||
session_id: str,
|
||||
user_id: str,
|
||||
job_id: str | None,
|
||||
query: str,
|
||||
collector_materials: list[dict[str, Any]],
|
||||
) -> tuple[int, list[dict[str, Any]]]:
|
||||
"""Copy the browser snapshot into the shared source table.
|
||||
|
||||
First-write still uses the job snapshot. These rows exist so refresh and
|
||||
「选择结构再生成」 can read the same pool. Failures are diagnostics only.
|
||||
"""
|
||||
from app.gateway.deep_research_job_executor import _materials_from_browser_snapshot
|
||||
|
||||
diagnostics: list[dict[str, Any]] = []
|
||||
source_store = getattr(request.app.state, "deep_research_source_store", None)
|
||||
if source_store is None:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "source_store_unavailable",
|
||||
"stage": "write_report",
|
||||
"message": "素材表不可用,写作仍使用请求体素材",
|
||||
"recoverable": True,
|
||||
}
|
||||
)
|
||||
return 0, diagnostics
|
||||
try:
|
||||
materials = _materials_from_browser_snapshot(collector_materials, query=query)
|
||||
except Exception as exc: # noqa: BLE001 - writing still uses the raw request body
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "client_materials_unparsed",
|
||||
"stage": "write_report",
|
||||
"message": "前端素材无法解析,将按无素材模式撰写",
|
||||
"recoverable": True,
|
||||
"errorType": type(exc).__name__,
|
||||
"detail": str(exc)[:1200] or type(exc).__name__,
|
||||
"receivedCount": len(collector_materials),
|
||||
}
|
||||
)
|
||||
return 0, diagnostics
|
||||
if collector_materials and not materials:
|
||||
sample_keys = sorted(
|
||||
{
|
||||
str(key)
|
||||
for item in collector_materials[:3]
|
||||
if isinstance(item, dict)
|
||||
for key in item
|
||||
}
|
||||
)
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "client_materials_unparsed",
|
||||
"stage": "write_report",
|
||||
"message": f"前端传入 {len(collector_materials)} 条素材,后端未能解析成写作条目,将按无素材模式撰写",
|
||||
"recoverable": True,
|
||||
"receivedCount": len(collector_materials),
|
||||
"parsedCount": 0,
|
||||
"sampleKeys": sample_keys[:20],
|
||||
}
|
||||
)
|
||||
persisted = 0
|
||||
persist_failures: list[dict[str, str]] = []
|
||||
persist_concurrency = 8
|
||||
|
||||
async def _upsert_one(material: Any) -> str | None:
|
||||
await source_store.upsert(
|
||||
session_id=session_id,
|
||||
job_id=job_id,
|
||||
user_id=user_id or None,
|
||||
selected=True,
|
||||
**material.to_persistence_dict(material.query),
|
||||
)
|
||||
return material.id
|
||||
|
||||
for offset in range(0, len(materials), persist_concurrency):
|
||||
chunk = materials[offset : offset + persist_concurrency]
|
||||
results = await asyncio.gather(*(_upsert_one(item) for item in chunk), return_exceptions=True)
|
||||
for material, result in zip(chunk, results, strict=True):
|
||||
if isinstance(result, Exception):
|
||||
persist_failures.append(
|
||||
{
|
||||
"materialId": material.id,
|
||||
"errorType": type(result).__name__,
|
||||
"detail": str(result)[:600] or type(result).__name__,
|
||||
}
|
||||
)
|
||||
logger.warning(
|
||||
"deep-research browser snapshot persist failed for %s: %s",
|
||||
material.id,
|
||||
result,
|
||||
)
|
||||
continue
|
||||
persisted += 1
|
||||
if persist_failures:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "client_materials_persist_failed",
|
||||
"stage": "write_report",
|
||||
"message": f"{len(persist_failures)} 条素材写入素材表失败,写作仍使用请求体",
|
||||
"recoverable": True,
|
||||
"failures": persist_failures[:8],
|
||||
}
|
||||
)
|
||||
return persisted, diagnostics
|
||||
|
||||
|
||||
@router.post("/sessions/{session_id}/report")
|
||||
async def write_report(session_id: str, body: WriteReportRequest, request: Request) -> dict[str, Any]:
|
||||
"""Start the first report write from browser-supplied materials.
|
||||
|
||||
This is the writing API. ``GET /jobs/{id}/stream`` only listens for
|
||||
progress after the job exists. The writer prefers the source table. A
|
||||
non-empty ``collector_materials`` list is a small fallback stored on the
|
||||
job snapshot (capped at 20). An empty list is valid and continues through
|
||||
the no-material writer.
|
||||
"""
|
||||
return await start_job(
|
||||
session_id,
|
||||
StartJobRequest(
|
||||
request_id=body.request_id,
|
||||
entry="chat",
|
||||
config=body.config,
|
||||
collector_materials=[] if body.force_no_materials else list(body.collector_materials or []),
|
||||
query=body.query,
|
||||
title=body.title,
|
||||
force_no_materials=body.force_no_materials,
|
||||
),
|
||||
request,
|
||||
materials_authoritative=True,
|
||||
)
|
||||
|
||||
|
||||
@router.post("/sessions/{session_id}/jobs")
|
||||
async def start_job(session_id: str, body: StartJobRequest, request: Request) -> dict[str, Any]:
|
||||
async def start_job(
|
||||
session_id: str,
|
||||
body: StartJobRequest,
|
||||
request: Request,
|
||||
*,
|
||||
materials_authoritative: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
user_id = await _require_user(request)
|
||||
session_store = _get_session_store(request)
|
||||
job_store = _get_job_store(request)
|
||||
@@ -959,7 +1299,10 @@ async def start_job(session_id: str, body: StartJobRequest, request: Request) ->
|
||||
mode=merged_config.mode,
|
||||
)
|
||||
|
||||
# Per-user active job limit (§19.4).
|
||||
# Per-user active job limit (§19.4). Paused/zombie sessions from an earlier
|
||||
# stop or a Gateway restart still look ``running`` in MySQL; release them
|
||||
# before counting so a restart cannot keep the 429 latch closed.
|
||||
await _release_stale_in_flight_sessions(request, user_id)
|
||||
active = await session_store.count_active_by_user(user_id=user_id)
|
||||
if active >= DEFAULT_MAX_ACTIVE_JOBS_PER_USER:
|
||||
existing = await job_store.get_active_for_session(session_id, user_id=user_id)
|
||||
@@ -967,18 +1310,55 @@ async def start_job(session_id: str, body: StartJobRequest, request: Request) ->
|
||||
raise HTTPException(status_code=429, detail="Maximum concurrent research jobs reached")
|
||||
|
||||
job_id = _new_id("drj")
|
||||
request_hash = json.dumps({"query": session.get("query"), "config": config_dict}, sort_keys=True, ensure_ascii=False)
|
||||
import hashlib
|
||||
|
||||
collector_materials = body.collector_materials if body.entry == "chat" else None
|
||||
force_no_materials = body.entry == "chat" and bool(body.force_no_materials)
|
||||
if force_no_materials:
|
||||
collector_materials = []
|
||||
# A list (including empty) on the first-write API is the writing input.
|
||||
# Missing/None keeps the older harvest fallback for legacy clients.
|
||||
authoritative = materials_authoritative or (
|
||||
body.entry == "chat" and body.collector_materials is not None
|
||||
) or force_no_materials
|
||||
query = str(body.query or "").strip() or str(session.get("query") or "")
|
||||
title = str(body.title or "").strip() or str(session.get("title") or "")
|
||||
request_hash = json.dumps({"query": query, "config": config_dict}, sort_keys=True, ensure_ascii=False)
|
||||
collector_material_count = (
|
||||
len(collector_materials or []) if body.entry == "chat" else None
|
||||
)
|
||||
diagnostics: list[dict[str, Any]] = []
|
||||
persisted_source_count = 0
|
||||
# Keep the compact browser list on the job. First-write reads this list.
|
||||
# Also copy it into the source table so refresh / regenerate still have
|
||||
# rows after the collector chat is gone from memory.
|
||||
if body.entry == "chat" and collector_materials:
|
||||
persisted_source_count, persist_diagnostics = await _persist_browser_collector_materials(
|
||||
request,
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
job_id=job_id,
|
||||
query=query,
|
||||
collector_materials=collector_materials,
|
||||
)
|
||||
diagnostics.extend(persist_diagnostics)
|
||||
snapshot_materials = (
|
||||
list(collector_materials or [])[:WRITE_FALLBACK_MATERIAL_LIMIT]
|
||||
if body.entry == "chat"
|
||||
else None
|
||||
)
|
||||
input_snapshot = {
|
||||
"session_id": session_id,
|
||||
"user_id": user_id,
|
||||
"query": session.get("query", ""),
|
||||
"query": query,
|
||||
"config": config_dict,
|
||||
"title": session.get("title", ""),
|
||||
"title": title,
|
||||
"runtime_thread_id": session.get("runtime_thread_id"),
|
||||
"entry": body.entry,
|
||||
"collector_materials": body.collector_materials if body.entry == "chat" else None,
|
||||
"collector_materials": snapshot_materials,
|
||||
"collector_material_count": collector_material_count,
|
||||
"client_materials_authoritative": authoritative,
|
||||
"force_no_materials": force_no_materials,
|
||||
}
|
||||
|
||||
job, created = await job_store.try_create_or_get_active(
|
||||
@@ -989,6 +1369,108 @@ async def start_job(session_id: str, body: StartJobRequest, request: Request) ->
|
||||
request_hash=hashlib.sha256(request_hash.encode()).hexdigest()[:32],
|
||||
input_snapshot=input_snapshot,
|
||||
)
|
||||
reused_job_id = str(job["id"])
|
||||
if (
|
||||
not created
|
||||
and body.entry == "chat"
|
||||
and collector_materials
|
||||
and reused_job_id != job_id
|
||||
):
|
||||
extra_count, extra_diagnostics = await _persist_browser_collector_materials(
|
||||
request,
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
job_id=reused_job_id,
|
||||
query=query,
|
||||
collector_materials=collector_materials,
|
||||
)
|
||||
persisted_source_count = max(persisted_source_count, extra_count)
|
||||
diagnostics.extend(extra_diagnostics)
|
||||
if (
|
||||
not created
|
||||
and body.entry == "chat"
|
||||
and (authoritative or collector_materials)
|
||||
and job.get("status") in ("queued", "running")
|
||||
):
|
||||
merged = dict(job.get("input_snapshot") or {})
|
||||
merged["collector_materials"] = snapshot_materials
|
||||
merged["collector_material_count"] = collector_material_count
|
||||
merged["entry"] = "chat"
|
||||
merged["client_materials_authoritative"] = authoritative
|
||||
merged["force_no_materials"] = force_no_materials
|
||||
merged["query"] = query
|
||||
merged["title"] = title
|
||||
merged["config"] = config_dict
|
||||
replace_snapshot = getattr(job_store, "replace_input_snapshot", None)
|
||||
if callable(replace_snapshot):
|
||||
try:
|
||||
updated = await replace_snapshot(job["id"], merged)
|
||||
if updated is not None:
|
||||
job = updated
|
||||
else:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "job_snapshot_replace_failed",
|
||||
"stage": "write_report",
|
||||
"message": "复用任务未能覆盖素材快照,写作仍继续",
|
||||
"recoverable": True,
|
||||
"jobId": job["id"],
|
||||
"status": job.get("status"),
|
||||
}
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - source table is the writing input
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "job_snapshot_replace_failed",
|
||||
"stage": "write_report",
|
||||
"message": "复用任务覆盖素材快照异常,写作仍继续",
|
||||
"recoverable": True,
|
||||
"errorType": type(exc).__name__,
|
||||
"detail": str(exc)[:1200] or type(exc).__name__,
|
||||
}
|
||||
)
|
||||
logger.warning("deep-research replace_input_snapshot failed", exc_info=True)
|
||||
else:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "job_snapshot_replace_failed",
|
||||
"stage": "write_report",
|
||||
"message": "当前任务存储不支持覆盖素材快照",
|
||||
"recoverable": True,
|
||||
}
|
||||
)
|
||||
|
||||
if body.entry == "chat" and force_no_materials:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "force_no_materials",
|
||||
"stage": "write_report",
|
||||
"message": "用户勾选了无素材写作,已忽略对话资料和素材表残留",
|
||||
"recoverable": True,
|
||||
"receivedCount": 0,
|
||||
}
|
||||
)
|
||||
elif body.entry == "chat" and authoritative and not collector_materials:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "empty_client_materials",
|
||||
"stage": "write_report",
|
||||
"message": "前端传入素材为空,已进入无素材写作",
|
||||
"recoverable": True,
|
||||
"receivedCount": 0,
|
||||
}
|
||||
)
|
||||
if not created and body.entry == "chat":
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "job_reused",
|
||||
"stage": "write_report",
|
||||
"message": "复用了已有写作任务,已尝试用本次请求覆盖素材快照",
|
||||
"recoverable": True,
|
||||
"jobId": job["id"],
|
||||
"status": job.get("status"),
|
||||
}
|
||||
)
|
||||
|
||||
# The durable job is the source of truth once it has been committed. A
|
||||
# transient failure updating the display projection must not turn that
|
||||
@@ -1005,7 +1487,17 @@ async def start_job(session_id: str, body: StartJobRequest, request: Request) ->
|
||||
# persisted error as though the newly queued job raised it.
|
||||
error=None,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception as exc: # noqa: BLE001
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "session_projection_failed",
|
||||
"stage": "write_report",
|
||||
"message": "会话状态投影更新失败,写作任务已入队",
|
||||
"recoverable": True,
|
||||
"errorType": type(exc).__name__,
|
||||
"detail": str(exc)[:1200] or type(exc).__name__,
|
||||
}
|
||||
)
|
||||
logger.exception(
|
||||
"deep-research job %s was queued but its session running projection could not be updated",
|
||||
job["id"],
|
||||
@@ -1019,12 +1511,24 @@ async def start_job(session_id: str, body: StartJobRequest, request: Request) ->
|
||||
dispatcher = _get_dispatcher(request)
|
||||
if dispatcher is not None:
|
||||
dispatcher.nudge()
|
||||
else:
|
||||
diagnostics.append(
|
||||
{
|
||||
"code": "dispatcher_unavailable",
|
||||
"stage": "write_report",
|
||||
"message": "写作调度器不可用,任务已入队但可能不会立即开始",
|
||||
"recoverable": True,
|
||||
}
|
||||
)
|
||||
|
||||
return {
|
||||
"jobId": job["id"],
|
||||
"sessionId": session_id,
|
||||
"status": job["status"],
|
||||
"reused": not created,
|
||||
"collectorMaterialCount": len(collector_materials or []),
|
||||
"persistedSourceCount": persisted_source_count,
|
||||
"diagnostics": diagnostics,
|
||||
}
|
||||
|
||||
|
||||
@@ -1250,6 +1754,11 @@ async def stream_job(
|
||||
request: Request,
|
||||
after: int = Query(default=0, ge=0),
|
||||
) -> StreamingResponse:
|
||||
"""Replay job progress. This is not the writing API and accepts no materials.
|
||||
|
||||
Start writing with ``POST /sessions/{session_id}/report`` (first write) or
|
||||
``POST /sessions/{session_id}/jobs`` (regenerate / legacy).
|
||||
"""
|
||||
user_id = await _require_user(request)
|
||||
job_store = _get_job_store(request)
|
||||
event_store = _get_event_store(request)
|
||||
@@ -1275,7 +1784,9 @@ async def stream_job(
|
||||
# Durable replay repairs a missed live delta after
|
||||
# reconnect or when another worker owned the job.
|
||||
try:
|
||||
events = await event_store.list_after(job_id, after=cursor, limit=200)
|
||||
events = await event_store.list_after(
|
||||
job_id, after=cursor, limit=_DURABLE_PAGE_SIZE
|
||||
)
|
||||
for ev in events:
|
||||
payload = {
|
||||
"seq": ev["seq"],
|
||||
@@ -1288,6 +1799,15 @@ async def stream_job(
|
||||
}
|
||||
yield format_sse("progress", payload, event_id=str(ev["seq"]))
|
||||
cursor = max(cursor, ev["seq"])
|
||||
if len(events) >= _DURABLE_PAGE_SIZE:
|
||||
# A full page means more rows are already waiting.
|
||||
# Drain them before looking at the job status: an
|
||||
# already-terminal job would otherwise close the
|
||||
# stream with part of its own replay unsent, and
|
||||
# the client would render a truncated report that
|
||||
# still ends in a clean `completed` frame.
|
||||
next_durable_poll = loop.time()
|
||||
continue
|
||||
current = await job_store.get_unscoped(job_id)
|
||||
if current and current.get("status") in TERMINAL_STATUSES:
|
||||
yield format_sse(
|
||||
@@ -1372,21 +1892,22 @@ async def stream_job(
|
||||
async def cancel_job(job_id: str, request: Request) -> dict[str, Any]:
|
||||
user_id = await _require_user(request)
|
||||
job_store = _get_job_store(request)
|
||||
row = await job_store.request_cancel(job_id, user_id=user_id)
|
||||
row = await job_store.get(job_id, user_id=user_id)
|
||||
if row is None:
|
||||
raise HTTPException(status_code=409, detail="Job is not in a cancellable state")
|
||||
# If no lease owner (queued/awaiting), finalize immediately.
|
||||
if not row.get("lease_owner"):
|
||||
row = await job_store.finalize_cancel(job_id) or row
|
||||
session_id = row.get("session_id")
|
||||
if session_id:
|
||||
session_store = _get_session_store(request)
|
||||
await session_store.update(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
status="cancelled",
|
||||
active_job_id=None,
|
||||
)
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
session_id = str(row.get("session_id") or "")
|
||||
dispatcher = _get_dispatcher(request)
|
||||
if dispatcher is not None:
|
||||
with suppress(Exception):
|
||||
dispatcher.cancel_running(job_id)
|
||||
if row.get("status") not in TERMINAL_STATUSES:
|
||||
requested = await job_store.request_cancel(job_id, user_id=user_id)
|
||||
if requested is not None:
|
||||
row = requested
|
||||
finalized = await job_store.finalize_cancel(job_id) or row
|
||||
row = finalized
|
||||
if session_id:
|
||||
await _park_session_after_job_stop(request, session_id=session_id, user_id=user_id)
|
||||
return {"jobId": job_id, "status": row.get("status", "cancelled")}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user