sync update from LAN repo (source ji @ 85078a31)
This commit is contained in:
1 parent
5d6627af50
commit
e602d88695
864 files changed
+99150
-6346
No files matched your search
@@ -61,9 +61,11 @@ logger = logging.getLogger(__name__)
|
||||
# ── constants ───────────────────────────────────────────────────────────────
|
||||
|
||||
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
|
||||
EXECUTOR_REVISION = "deep-research-db-first-client-fallback-20260904-v13"
|
||||
_SOURCE_READ_ATTEMPTS = 16
|
||||
EXECUTOR_REVISION = "deep-research-write-live-then-persist-20260916-v18"
|
||||
_SOURCE_READ_ATTEMPTS = 4
|
||||
_SOURCE_READ_DELAY_SECONDS = 0.5
|
||||
_SOURCE_READ_TIMEOUT_SECONDS = 3.0
|
||||
_CHECKPOINT_HARVEST_TIMEOUT_SECONDS = 8.0
|
||||
_LEASE_TTL_SECONDS = 180
|
||||
_HEARTBEAT_INTERVAL = 20.0
|
||||
_CANCEL_POLL_INTERVAL = 5.0
|
||||
@@ -219,6 +221,35 @@ class _JobUser:
|
||||
self.id = user_id
|
||||
|
||||
|
||||
class LiveFirstEventSink:
|
||||
"""Show write progress immediately. Persist nothing until the report exists.
|
||||
|
||||
``emit`` is persist-then-stream on the production MySQL. That wait is what
|
||||
hid the first frame and then froze the writer. Chat / regenerate writes
|
||||
therefore fan out to the live hub only; ``_persist_completed_report_projection``
|
||||
stores the finished markdown afterwards.
|
||||
"""
|
||||
|
||||
def __init__(self, inner) -> None:
|
||||
self._inner = inner
|
||||
self.diagnostics: list[dict[str, Any]] = list(getattr(inner, "diagnostics", []) or [])
|
||||
|
||||
async def emit(self, type, *, phase="initializing", payload=None): # noqa: A002
|
||||
diagnostic = (payload or {}).get("diagnostic")
|
||||
if isinstance(diagnostic, dict):
|
||||
self.diagnostics.append(
|
||||
{
|
||||
"code": str((payload or {}).get("code") or type),
|
||||
**diagnostic,
|
||||
}
|
||||
)
|
||||
await self.publish_live(type, phase=phase, payload=payload or {})
|
||||
return None
|
||||
|
||||
async def publish_live(self, type, *, phase="initializing", payload=None): # noqa: A002
|
||||
await self._inner.publish_live(type, phase=phase, payload=payload or {})
|
||||
|
||||
|
||||
# ── recording event sink (source persistence + progress fan-out) ─────────────
|
||||
|
||||
|
||||
@@ -1710,9 +1741,25 @@ class DeepResearchJobExecutor:
|
||||
user_id=user_id or None,
|
||||
lease_owner=lease_owner,
|
||||
)
|
||||
# Chat/regenerate writing is live-first. Persist the finished report
|
||||
# after the model returns; do not wait on the event table to display.
|
||||
events_for_write = (
|
||||
LiveFirstEventSink(recording_sink) if entry in ("chat", "regenerate") else recording_sink
|
||||
)
|
||||
started_payload = {
|
||||
"code": "write_job_started",
|
||||
"message": "写作任务已启动,正在读取已收集材料",
|
||||
"recoverable": True,
|
||||
}
|
||||
with suppress(Exception):
|
||||
await events_for_write.publish_live(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload=started_payload,
|
||||
)
|
||||
|
||||
# LLM backend (single chokepoint).
|
||||
llm_backend = DeerFlowCompletionBackend(config, event_sink=recording_sink, cancellation=cancel_token)
|
||||
llm_backend = DeerFlowCompletionBackend(config, event_sink=events_for_write, cancellation=cancel_token)
|
||||
|
||||
# Material provider. First write prefers the source table (full pool
|
||||
# persisted when collection finished). The job snapshot only carries a
|
||||
@@ -1725,7 +1772,7 @@ class DeepResearchJobExecutor:
|
||||
if force_no_materials:
|
||||
material_provider = StaticHarvestMaterialProvider([])
|
||||
try:
|
||||
await recording_sink.emit(
|
||||
await events_for_write.publish_live(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
@@ -1734,7 +1781,7 @@ class DeepResearchJobExecutor:
|
||||
"recoverable": True,
|
||||
},
|
||||
)
|
||||
await recording_sink.emit(
|
||||
await events_for_write.publish_live(
|
||||
"collection_completed",
|
||||
phase="collecting",
|
||||
payload={"source": "force_no_materials", "materialCount": 0},
|
||||
@@ -1748,7 +1795,7 @@ class DeepResearchJobExecutor:
|
||||
user_id=user_id,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
query=query,
|
||||
events=recording_sink,
|
||||
events=events_for_write,
|
||||
cancellation=cancel_token,
|
||||
collector_materials=collector_materials,
|
||||
materials_authoritative=materials_authoritative,
|
||||
@@ -1782,18 +1829,29 @@ class DeepResearchJobExecutor:
|
||||
materials=material_provider,
|
||||
context=context_compressor,
|
||||
artifacts=artifact_writer,
|
||||
events=recording_sink,
|
||||
events=events_for_write,
|
||||
images=build_image_provider(),
|
||||
)
|
||||
|
||||
job = await self._job_store.get_unscoped(job_id)
|
||||
job = None
|
||||
if self._job_store is not None:
|
||||
try:
|
||||
job = await self._job_store.get_unscoped(job_id)
|
||||
except Exception: # noqa: BLE001 - writing does not need the job row
|
||||
logger.warning("deep-research job %s could not reload its row before writing", job_id, exc_info=True)
|
||||
interrupted_plan = None
|
||||
if entry != "chat":
|
||||
try:
|
||||
interrupted_plan = await self._load_interrupted_plan(job_id)
|
||||
except Exception: # noqa: BLE001 - first write has no plan review
|
||||
logger.warning("deep-research job %s could not reload an interrupted plan", job_id, exc_info=True)
|
||||
request = DeepResearchRequest(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
config=config,
|
||||
resume_payload=(job or {}).get("awaiting_payload") or {},
|
||||
interrupted_plan=await self._load_interrupted_plan(job_id),
|
||||
interrupted_plan=interrupted_plan,
|
||||
)
|
||||
|
||||
engine = DeepResearchEngine()
|
||||
@@ -1805,7 +1863,7 @@ class DeepResearchJobExecutor:
|
||||
llm=llm_backend,
|
||||
report_markdown=result.report_markdown,
|
||||
language=config.language,
|
||||
events=recording_sink,
|
||||
events=events_for_write,
|
||||
cancellation=cancel_token,
|
||||
)
|
||||
# Fold the summary call's tokens into the billed totals.
|
||||
@@ -1813,6 +1871,7 @@ class DeepResearchJobExecutor:
|
||||
diagnostics = [
|
||||
*getattr(base_sink, "diagnostics", []),
|
||||
*recording_sink.diagnostics,
|
||||
*getattr(events_for_write, "diagnostics", []),
|
||||
]
|
||||
if diagnostics:
|
||||
result = result.model_copy(update={"diagnostics": diagnostics})
|
||||
@@ -1902,14 +1961,11 @@ class DeepResearchJobExecutor:
|
||||
):
|
||||
"""Build the first-write material pool.
|
||||
|
||||
Prefer the source table (the full collected pool). The browser list is
|
||||
a small fallback used only when that table is empty. Checkpoint harvest
|
||||
is last. An authoritative empty list (count 0) is a true no-materials
|
||||
write and must not pick leftover source rows.
|
||||
|
||||
Emits ``collection_completed`` plus one ``source_added`` per material so
|
||||
the sources panel fills exactly like the legacy channel (the recording
|
||||
sink persists them to the source store).
|
||||
If the write POST already carried collector materials, those are the
|
||||
writing input. The production MySQL source table is the path that
|
||||
hangs after the first live frame; do not read or upsert it before the
|
||||
report starts. The table is only consulted when the snapshot is empty
|
||||
(refresh / older clients). Checkpoint harvest stays last.
|
||||
"""
|
||||
from deerflow.agents.deep_research.adapters.material import dedupe_materials
|
||||
from deerflow.agents.deep_research.collection.thread_harvester import (
|
||||
@@ -1919,6 +1975,15 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
async def _live_progress(type: str, *, phase: str = "collecting", payload: dict[str, Any] | None = None) -> None: # noqa: A002
|
||||
publish_live = getattr(events, "publish_live", None)
|
||||
if callable(publish_live):
|
||||
await publish_live(type, phase=phase, payload=payload or {})
|
||||
return
|
||||
# Tests and older sinks without a live hub still record through emit.
|
||||
await events.emit(type, phase=phase, payload=payload or {})
|
||||
|
||||
harvested: list[ResearchMaterial] = []
|
||||
agent_summary = ""
|
||||
browser_material_count = 0
|
||||
@@ -1959,7 +2024,7 @@ class DeepResearchJobExecutor:
|
||||
if row.get("id")
|
||||
]
|
||||
|
||||
async def try_browser_snapshot() -> None:
|
||||
async def try_browser_snapshot(*, persist: bool = True) -> None:
|
||||
nonlocal browser_material_count, browser_snapshot_persisted, harvested, material_channel
|
||||
raw_count = snapshot_count
|
||||
if raw_count == 0 and expected_material_count > 0:
|
||||
@@ -1971,30 +2036,31 @@ class DeepResearchJobExecutor:
|
||||
else []
|
||||
)
|
||||
try:
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"code": "browser_snapshot_received",
|
||||
"message": (
|
||||
f"后端已收到前端素材快照 {raw_count} 条"
|
||||
if raw_count
|
||||
else (
|
||||
"后端未收到前端素材快照(字段缺失或空数组)。"
|
||||
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
|
||||
)
|
||||
),
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
|
||||
"receivedType": type(collector_materials).__name__,
|
||||
"receivedCount": raw_count,
|
||||
"expectedCount": expected_material_count,
|
||||
"sampleKeys": sample_keys[:20],
|
||||
},
|
||||
publish_live = getattr(events, "publish_live", None)
|
||||
receipt_payload = {
|
||||
"code": "browser_snapshot_received",
|
||||
"message": (
|
||||
f"后端已收到前端素材快照 {raw_count} 条"
|
||||
if raw_count
|
||||
else (
|
||||
"后端未收到前端素材快照(字段缺失或空数组)。"
|
||||
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
|
||||
)
|
||||
),
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
|
||||
"receivedType": type(collector_materials).__name__,
|
||||
"receivedCount": raw_count,
|
||||
"expectedCount": expected_material_count,
|
||||
"sampleKeys": sample_keys[:20],
|
||||
},
|
||||
)
|
||||
}
|
||||
if callable(publish_live):
|
||||
await publish_live("warning", phase="collecting", payload=receipt_payload)
|
||||
elif persist:
|
||||
await events.emit("warning", phase="collecting", payload=receipt_payload)
|
||||
except Exception: # noqa: BLE001 - diagnostics never block writing
|
||||
logger.warning("failed to publish browser snapshot receipt", exc_info=True)
|
||||
if not collector_materials:
|
||||
@@ -2023,7 +2089,7 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
return
|
||||
material_channel = "browser_snapshot"
|
||||
if self._source_store is None:
|
||||
if not persist or self._source_store is None:
|
||||
return
|
||||
browser_snapshot_persisted = True
|
||||
for material in harvested:
|
||||
@@ -2080,12 +2146,15 @@ class DeepResearchJobExecutor:
|
||||
for attempt in range(attempts):
|
||||
cancellation.raise_if_cancelled()
|
||||
try:
|
||||
rows = await self._source_store.list_by_session(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
selected=None,
|
||||
limit=500,
|
||||
include_content=True,
|
||||
rows = await asyncio.wait_for(
|
||||
self._source_store.list_by_session(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
selected=None,
|
||||
limit=500,
|
||||
include_content=True,
|
||||
),
|
||||
timeout=_SOURCE_READ_TIMEOUT_SECONDS,
|
||||
)
|
||||
persisted_materials = persisted_rows_to_materials(rows)
|
||||
durable_material_count = len(persisted_materials)
|
||||
@@ -2093,7 +2162,7 @@ class DeepResearchJobExecutor:
|
||||
if harvested:
|
||||
material_channel = "durable_sources"
|
||||
try:
|
||||
await events.emit(
|
||||
await _live_progress(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
@@ -2141,7 +2210,7 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
|
||||
if skip_leftover_sources:
|
||||
await try_browser_snapshot()
|
||||
await try_browser_snapshot(persist=False)
|
||||
if not harvested:
|
||||
channel_failures.append(
|
||||
{
|
||||
@@ -2150,20 +2219,32 @@ class DeepResearchJobExecutor:
|
||||
"detail": "前端传入素材为空,已进入无素材写作",
|
||||
}
|
||||
)
|
||||
elif snapshot_count > 0:
|
||||
# Same lesson as the empty-table hang: the collected materials are
|
||||
# already on this job. Do not go back to the source table before
|
||||
# writing — that read is what stalls on the production server.
|
||||
await try_browser_snapshot(persist=False)
|
||||
else:
|
||||
await try_durable_sources(retry=True)
|
||||
# Prefer the source table, but do not stall on an empty/slow table
|
||||
# when the write POST already carried a compact material snapshot.
|
||||
await try_durable_sources(retry=False)
|
||||
if not harvested:
|
||||
await try_browser_snapshot()
|
||||
if not harvested:
|
||||
await try_durable_sources(retry=True)
|
||||
if not harvested:
|
||||
try:
|
||||
if self._checkpointer is None:
|
||||
raise RuntimeError("Deep Research checkpointer is unavailable")
|
||||
if not runtime_thread_id:
|
||||
raise RuntimeError("Deep Research collector thread is unavailable")
|
||||
checkpoint_materials, checkpoint_summary = await harvest_thread_materials(
|
||||
checkpointer=self._checkpointer,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
query=query,
|
||||
checkpoint_materials, checkpoint_summary = await asyncio.wait_for(
|
||||
harvest_thread_materials(
|
||||
checkpointer=self._checkpointer,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
query=query,
|
||||
),
|
||||
timeout=_CHECKPOINT_HARVEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
checkpoint_material_count = len(checkpoint_materials)
|
||||
agent_summary = checkpoint_summary or ""
|
||||
@@ -2201,70 +2282,43 @@ class DeepResearchJobExecutor:
|
||||
# temporarily unavailable event table must never prevent the writer
|
||||
# from receiving this already-collected evidence.
|
||||
material_provider = StaticHarvestMaterialProvider(harvested)
|
||||
try:
|
||||
await events.emit(
|
||||
"collection_completed",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"source": material_channel,
|
||||
"materialCount": material_provider.material_count,
|
||||
completed_payload = {
|
||||
"source": material_channel,
|
||||
"materialCount": material_provider.material_count,
|
||||
"browserMaterialCount": browser_material_count,
|
||||
"checkpointMaterialCount": checkpoint_material_count,
|
||||
"durableMaterialCount": durable_material_count,
|
||||
"expectedMaterialCount": expected_material_count,
|
||||
"agentSummaryChars": len(agent_summary),
|
||||
}
|
||||
with suppress(Exception):
|
||||
await _live_progress("collection_completed", phase="collecting", payload=completed_payload)
|
||||
# Do not persist source_added during writing. Snapshot / leftover
|
||||
# collection already owns the evidence; MySQL upserts here freeze the
|
||||
# first frame. The finished report is stored afterwards.
|
||||
if material_provider.material_count and material_channel == "browser_snapshot":
|
||||
snapshot_payload = {
|
||||
"code": "material_channel_snapshot_used",
|
||||
"message": "已使用本次请求携带的已收集素材撰写,未再等待素材表",
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "client_material_snapshot",
|
||||
"selectedChannel": material_channel,
|
||||
"browserMaterialCount": browser_material_count,
|
||||
"checkpointMaterialCount": checkpoint_material_count,
|
||||
"durableMaterialCount": durable_material_count,
|
||||
"expectedMaterialCount": expected_material_count,
|
||||
"agentSummaryChars": len(agent_summary),
|
||||
"runtimeThreadId": runtime_thread_id,
|
||||
"failures": channel_failures,
|
||||
},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - progress telemetry must not lose evidence
|
||||
logger.warning("chat deep-research collection event failed; retaining harvested materials", exc_info=True)
|
||||
# A last-resort browser snapshot is written directly above. Avoid a
|
||||
# second source-added projection only when every direct write succeeded.
|
||||
if material_channel != "browser_snapshot" or not browser_snapshot_persisted:
|
||||
for material in harvested:
|
||||
try:
|
||||
await events.emit(
|
||||
"source_added",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"sourceId": material.id,
|
||||
"title": material.title,
|
||||
"url": material.url,
|
||||
"sourceType": material.source_type,
|
||||
"source": material.to_persistence_dict(material.query),
|
||||
},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - source rows are a projection, not the input
|
||||
logger.warning(
|
||||
"chat deep-research source event failed for %s; retaining harvested materials",
|
||||
material.id,
|
||||
exc_info=True,
|
||||
)
|
||||
if material_provider.material_count and material_channel == "browser_snapshot":
|
||||
}
|
||||
try:
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"code": "material_channel_fallback_used",
|
||||
"message": "素材表为空,已使用前端兜底素材撰写",
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "client_material_fallback",
|
||||
"selectedChannel": material_channel,
|
||||
"browserMaterialCount": browser_material_count,
|
||||
"checkpointMaterialCount": checkpoint_material_count,
|
||||
"durableMaterialCount": durable_material_count,
|
||||
"runtimeThreadId": runtime_thread_id,
|
||||
"failures": channel_failures,
|
||||
},
|
||||
},
|
||||
)
|
||||
await _live_progress("warning", phase="collecting", payload=snapshot_payload)
|
||||
except Exception: # noqa: BLE001 - diagnostics never block writing
|
||||
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
|
||||
logger.warning("failed to publish material snapshot diagnostic", exc_info=True)
|
||||
elif material_provider.material_count and material_channel == "runtime_checkpoint":
|
||||
try:
|
||||
await events.emit(
|
||||
await _live_progress(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
@@ -2287,7 +2341,7 @@ class DeepResearchJobExecutor:
|
||||
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
|
||||
if not material_provider.material_count and not force_no_materials:
|
||||
try:
|
||||
await events.emit(
|
||||
await _live_progress(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
|
||||
Reference in new issue
Block a user