sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
@@ -31,8 +31,10 @@ import uuid as _uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import PurePosixPath
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.gateway.document_rewrite_diagnostics import build_document_rewrite_diagnostic
|
||||
from deerflow.agents.deep_research.cancellation import (
|
||||
CancellationToken,
|
||||
ResearchAwaitingInput,
|
||||
@@ -61,9 +63,11 @@ logger = logging.getLogger(__name__)
|
||||
# ── constants ───────────────────────────────────────────────────────────────
|
||||
|
||||
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
|
||||
EXECUTOR_REVISION = "deep-research-db-first-client-fallback-20260904-v13"
|
||||
_SOURCE_READ_ATTEMPTS = 16
|
||||
EXECUTOR_REVISION = "deep-research-write-live-then-persist-20260916-v18"
|
||||
_SOURCE_READ_ATTEMPTS = 4
|
||||
_SOURCE_READ_DELAY_SECONDS = 0.5
|
||||
_SOURCE_READ_TIMEOUT_SECONDS = 3.0
|
||||
_CHECKPOINT_HARVEST_TIMEOUT_SECONDS = 8.0
|
||||
_LEASE_TTL_SECONDS = 180
|
||||
_HEARTBEAT_INTERVAL = 20.0
|
||||
_CANCEL_POLL_INTERVAL = 5.0
|
||||
@@ -219,6 +223,35 @@ class _JobUser:
|
||||
self.id = user_id
|
||||
|
||||
|
||||
class LiveFirstEventSink:
|
||||
"""Show write progress immediately. Persist nothing until the report exists.
|
||||
|
||||
``emit`` is persist-then-stream on the production MySQL. That wait is what
|
||||
hid the first frame and then froze the writer. Chat / regenerate writes
|
||||
therefore fan out to the live hub only; ``_persist_completed_report_projection``
|
||||
stores the finished markdown afterwards.
|
||||
"""
|
||||
|
||||
def __init__(self, inner) -> None:
|
||||
self._inner = inner
|
||||
self.diagnostics: list[dict[str, Any]] = list(getattr(inner, "diagnostics", []) or [])
|
||||
|
||||
async def emit(self, type, *, phase="initializing", payload=None): # noqa: A002
|
||||
diagnostic = (payload or {}).get("diagnostic")
|
||||
if isinstance(diagnostic, dict):
|
||||
self.diagnostics.append(
|
||||
{
|
||||
"code": str((payload or {}).get("code") or type),
|
||||
**diagnostic,
|
||||
}
|
||||
)
|
||||
await self.publish_live(type, phase=phase, payload=payload or {})
|
||||
return None
|
||||
|
||||
async def publish_live(self, type, *, phase="initializing", payload=None): # noqa: A002
|
||||
await self._inner.publish_live(type, phase=phase, payload=payload or {})
|
||||
|
||||
|
||||
# ── recording event sink (source persistence + progress fan-out) ─────────────
|
||||
|
||||
|
||||
@@ -684,7 +717,17 @@ class DeepResearchJobExecutor:
|
||||
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("deep-research job %s crashed", job_id)
|
||||
await self._mark_failed(job_id, session_id, lease_owner, type(exc).__name__, str(exc))
|
||||
if snapshot.get("kind") == "artifact_document_rewrite":
|
||||
await self._mark_artifact_document_rewrite_failed(
|
||||
job_id,
|
||||
session_id,
|
||||
lease_owner,
|
||||
type(exc).__name__,
|
||||
str(exc),
|
||||
exc=exc,
|
||||
)
|
||||
else:
|
||||
await self._mark_failed(job_id, session_id, lease_owner, type(exc).__name__, str(exc))
|
||||
|
||||
finally:
|
||||
watcher.cancel()
|
||||
@@ -1063,6 +1106,7 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
thread_id = str(snapshot.get("thread_id") or "")
|
||||
virtual_path = str(snapshot.get("path") or "").strip()
|
||||
output_virtual_path = str(snapshot.get("output_path") or "").strip()
|
||||
original = str(snapshot.get("original_content") or "")
|
||||
base_hash = str(snapshot.get("base_hash") or "")
|
||||
instruction = str(snapshot.get("instruction") or "").strip()
|
||||
@@ -1074,63 +1118,85 @@ class DeepResearchJobExecutor:
|
||||
except (TypeError, ValueError):
|
||||
max_generated_images = 2
|
||||
|
||||
async def fail(code: str, message: str, *, stage: str = "generation") -> None:
|
||||
await events.emit(
|
||||
"document_rewrite",
|
||||
phase="done",
|
||||
payload={"type": "error", "message": message, "stage": stage},
|
||||
)
|
||||
await self._job_store.update_progress(
|
||||
async def fail(
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
stage: str = "generation",
|
||||
exc: BaseException | None = None,
|
||||
) -> None:
|
||||
await self._mark_artifact_document_rewrite_failed(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="failed",
|
||||
phase="done",
|
||||
progress=100,
|
||||
error_code=code,
|
||||
error_message=message,
|
||||
result_snapshot={"kind": "artifact_document_rewrite", "status": "failed"},
|
||||
session_id,
|
||||
lease_owner,
|
||||
code,
|
||||
message,
|
||||
events=events,
|
||||
stage=stage,
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
if self._app_config is None:
|
||||
await fail("CONFIG_UNAVAILABLE", "改写服务配置不可用,任务未写入文件。")
|
||||
return
|
||||
if not thread_id or not virtual_path or not original.strip() or not base_hash or not instruction:
|
||||
if (
|
||||
not thread_id
|
||||
or not virtual_path
|
||||
or not original.strip()
|
||||
or not base_hash
|
||||
or not instruction
|
||||
):
|
||||
await fail("INVALID_SNAPSHOT", "改写任务快照不完整,无法继续执行。")
|
||||
return
|
||||
|
||||
try:
|
||||
actual_path = await aresolve_thread_virtual_path(thread_id, virtual_path)
|
||||
current = await asyncio.to_thread(actual_path.read_text, encoding="utf-8")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
await fail("SOURCE_UNAVAILABLE", f"无法读取待改写文件:{exc}")
|
||||
await fail("SOURCE_UNAVAILABLE", f"无法读取待改写文件:{exc}", exc=exc)
|
||||
return
|
||||
|
||||
source_virtual = PurePosixPath(virtual_path)
|
||||
if output_virtual_path:
|
||||
output_virtual = PurePosixPath(output_virtual_path)
|
||||
if (
|
||||
output_virtual.parent != source_virtual.parent
|
||||
or output_virtual.name == source_virtual.name
|
||||
or output_virtual.suffix.lower() not in {".md", ".markdown"}
|
||||
):
|
||||
await fail("INVALID_OUTPUT_PATH", "改写版输出路径无效,任务未写入文件。")
|
||||
return
|
||||
output_path = actual_path.with_name(output_virtual.name)
|
||||
else:
|
||||
# Jobs created before independent output files were introduced did
|
||||
# not retain a destination. Keep them resumable while still
|
||||
# producing a new file instead of mutating their source artifact.
|
||||
for index in range(1, 10_000):
|
||||
output_name = (
|
||||
f"{actual_path.stem}-改写版{actual_path.suffix}"
|
||||
if index == 1
|
||||
else f"{actual_path.stem}-改写版-{index}{actual_path.suffix}"
|
||||
)
|
||||
candidate = actual_path.with_name(output_name)
|
||||
if not candidate.exists():
|
||||
output_path = candidate
|
||||
output_virtual = source_virtual.with_name(output_name)
|
||||
output_virtual_path = str(output_virtual)
|
||||
break
|
||||
else:
|
||||
await fail("OUTPUT_NAME_EXHAUSTED", "改写版文件过多,请整理后再试。", stage="commit")
|
||||
return
|
||||
|
||||
await events.emit(
|
||||
"document_rewrite",
|
||||
phase="initializing",
|
||||
payload={
|
||||
"type": "snapshot_fixed",
|
||||
"fileName": actual_path.name,
|
||||
"fileName": output_path.name,
|
||||
"outputPath": output_virtual_path,
|
||||
"originalCharCount": visible_document_char_count(original),
|
||||
"baseHash": base_hash,
|
||||
},
|
||||
)
|
||||
if content_hash(current) != base_hash:
|
||||
message = "文件在任务启动前已被修改,已取消覆盖并保留原文件。"
|
||||
await events.emit(
|
||||
"document_rewrite",
|
||||
phase="done",
|
||||
payload={"type": "conflict", "message": message},
|
||||
)
|
||||
await self._job_store.update_progress(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="completed",
|
||||
phase="done",
|
||||
progress=100,
|
||||
result_snapshot={"kind": "artifact_document_rewrite", "status": "conflict"},
|
||||
)
|
||||
return
|
||||
|
||||
rewrite_source = original
|
||||
if generate_images:
|
||||
@@ -1156,7 +1222,7 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("could not initialise artifact rewrite model for %s", job_id)
|
||||
await fail(type(exc).__name__, str(exc), stage="requirements")
|
||||
await fail(type(exc).__name__, str(exc), stage="requirements", exc=exc)
|
||||
return
|
||||
|
||||
# Requirements are a visible, streamed preflight rather than a hidden
|
||||
@@ -1319,7 +1385,7 @@ class DeepResearchJobExecutor:
|
||||
raise
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("artifact document rewrite %s failed", job_id)
|
||||
await fail(type(exc).__name__, str(exc))
|
||||
await fail(type(exc).__name__, str(exc), exc=exc)
|
||||
return
|
||||
|
||||
tail, inline_thinking_tail = inline_think_filter.finish_parts()
|
||||
@@ -1418,39 +1484,13 @@ class DeepResearchJobExecutor:
|
||||
generated_illustrations, generated_warnings = await generate_document_illustrations(
|
||||
provider,
|
||||
markdown=rewritten,
|
||||
document_name=actual_path.name,
|
||||
document_name=output_path.name,
|
||||
maximum=max_generated_images,
|
||||
on_started=on_image_started,
|
||||
on_generated=on_image_generated,
|
||||
)
|
||||
illustration_warnings.extend(generated_warnings)
|
||||
|
||||
# Do not save images if a manual edit has invalidated the frozen
|
||||
# source version while image calls were in flight.
|
||||
current_before_images = await asyncio.to_thread(actual_path.read_text, encoding="utf-8")
|
||||
if content_hash(current_before_images) != base_hash:
|
||||
message = "文件在生成配图期间已被修改,已保留原文件。"
|
||||
await events.emit(
|
||||
"document_rewrite",
|
||||
phase="done",
|
||||
payload={"type": "conflict", "message": message},
|
||||
)
|
||||
await self._job_store.update_progress(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="completed",
|
||||
phase="done",
|
||||
progress=100,
|
||||
result_snapshot={
|
||||
"kind": "artifact_document_rewrite",
|
||||
"status": "conflict",
|
||||
"content": rewritten,
|
||||
"originalContent": original,
|
||||
"requirementsPlan": requirements_plan,
|
||||
},
|
||||
)
|
||||
return
|
||||
|
||||
if generated_illustrations:
|
||||
from app.gateway.document_rewrite_illustrations import (
|
||||
insert_document_illustrations,
|
||||
@@ -1461,8 +1501,8 @@ class DeepResearchJobExecutor:
|
||||
persisted_illustrations, persistence_warnings = await asyncio.to_thread(
|
||||
persist_document_illustrations,
|
||||
generated_illustrations,
|
||||
markdown_path=actual_path,
|
||||
virtual_markdown_path=virtual_path,
|
||||
markdown_path=output_path,
|
||||
virtual_markdown_path=output_virtual_path,
|
||||
job_id=job_id,
|
||||
)
|
||||
illustration_warnings.extend(persistence_warnings)
|
||||
@@ -1522,80 +1562,31 @@ class DeepResearchJobExecutor:
|
||||
await events.emit("document_rewrite", phase="reviewing", payload={"type": "comparison_ready", **summary})
|
||||
await events.emit("document_rewrite", phase="exporting", payload={"type": "commit_started"})
|
||||
|
||||
current_before_commit = await asyncio.to_thread(actual_path.read_text, encoding="utf-8")
|
||||
if content_hash(current_before_commit) != base_hash:
|
||||
message = "文件在改写期间已被修改,已保留候选稿但没有覆盖原文件。"
|
||||
await events.emit(
|
||||
"document_rewrite",
|
||||
phase="done",
|
||||
payload={"type": "conflict", "message": message},
|
||||
)
|
||||
await self._job_store.update_progress(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="completed",
|
||||
phase="done",
|
||||
progress=100,
|
||||
result_snapshot={
|
||||
"kind": "artifact_document_rewrite",
|
||||
"status": "conflict",
|
||||
"content": rewritten,
|
||||
"originalContent": original,
|
||||
"summary": summary,
|
||||
"requirementsPlan": requirements_plan,
|
||||
},
|
||||
committed_hash = content_hash(rewritten)
|
||||
if output_path.exists():
|
||||
await fail(
|
||||
"OUTPUT_ALREADY_EXISTS",
|
||||
f"改写版文件已存在:{output_virtual_path}。为避免覆盖已有文件,本次未写入结果。",
|
||||
stage="commit",
|
||||
)
|
||||
return
|
||||
|
||||
committed_hash = content_hash(rewritten)
|
||||
try:
|
||||
await asyncio.to_thread(atomic_write_text, actual_path, rewritten)
|
||||
version_id: str | None = None
|
||||
version_snapshot_failed = False
|
||||
if self._document_rewrite_version_store is not None:
|
||||
try:
|
||||
version_id = f"drv_{_uuid.uuid4().hex}"
|
||||
await self._document_rewrite_version_store.create(
|
||||
id=version_id,
|
||||
user_id=user_id or None,
|
||||
source_type="artifact",
|
||||
source_id=thread_id,
|
||||
source_path=virtual_path,
|
||||
original_content=original,
|
||||
original_hash=base_hash,
|
||||
committed_hash=committed_hash,
|
||||
instruction=instruction,
|
||||
model_name=resolved_name,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
# The file is already committed. Keep it available even
|
||||
# if its optional undo snapshot cannot be persisted.
|
||||
logger.exception(
|
||||
"could not persist artifact rewrite version for %s; keeping rewritten file",
|
||||
actual_path,
|
||||
)
|
||||
version_id = None
|
||||
version_snapshot_failed = True
|
||||
else:
|
||||
try:
|
||||
await self._document_rewrite_version_store.prune_source(
|
||||
user_id=user_id or None,
|
||||
source_type="artifact",
|
||||
source_id=thread_id,
|
||||
source_path=virtual_path,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("could not prune artifact rewrite versions for %s", actual_path)
|
||||
await asyncio.to_thread(atomic_write_text, output_path, rewritten)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("artifact document rewrite %s commit failed", job_id)
|
||||
await fail(type(exc).__name__, str(exc), stage="commit")
|
||||
await fail(type(exc).__name__, str(exc), stage="commit", exc=exc)
|
||||
return
|
||||
|
||||
version_id: str | None = None
|
||||
version_snapshot_failed = False
|
||||
|
||||
result_snapshot = {
|
||||
"kind": "artifact_document_rewrite",
|
||||
"status": "completed",
|
||||
"content": rewritten,
|
||||
"originalContent": original,
|
||||
"outputPath": output_virtual_path,
|
||||
"outputFileName": output_path.name,
|
||||
"committedHash": committed_hash,
|
||||
"versionId": version_id,
|
||||
"versionSnapshotFailed": version_snapshot_failed,
|
||||
@@ -1608,6 +1599,8 @@ class DeepResearchJobExecutor:
|
||||
payload={
|
||||
"type": "committed",
|
||||
"content": rewritten,
|
||||
"outputPath": output_virtual_path,
|
||||
"outputFileName": output_path.name,
|
||||
"committedHash": committed_hash,
|
||||
"versionId": version_id,
|
||||
"versionSnapshotFailed": version_snapshot_failed,
|
||||
@@ -1618,6 +1611,8 @@ class DeepResearchJobExecutor:
|
||||
phase="done",
|
||||
payload={
|
||||
"type": "done",
|
||||
"outputPath": output_virtual_path,
|
||||
"outputFileName": output_path.name,
|
||||
"committedHash": committed_hash,
|
||||
"versionId": version_id,
|
||||
"versionSnapshotFailed": version_snapshot_failed,
|
||||
@@ -1632,6 +1627,60 @@ class DeepResearchJobExecutor:
|
||||
result_snapshot=result_snapshot,
|
||||
)
|
||||
|
||||
async def _mark_artifact_document_rewrite_failed(
|
||||
self,
|
||||
job_id: str,
|
||||
session_id: str,
|
||||
lease_owner: str | None,
|
||||
code: str,
|
||||
message: str,
|
||||
*,
|
||||
events: Any | None = None,
|
||||
stage: str = "generation",
|
||||
exc: BaseException | None = None,
|
||||
) -> None:
|
||||
if events is None:
|
||||
from deerflow.agents.deep_research.adapters.event_sink import PersistingEventSink
|
||||
|
||||
events = PersistingEventSink(
|
||||
self._event_store,
|
||||
session_id=session_id,
|
||||
job_id=job_id,
|
||||
on_event=self._live_publisher,
|
||||
)
|
||||
diagnostic = build_document_rewrite_diagnostic(
|
||||
job_id=job_id,
|
||||
stage=stage,
|
||||
error_code=code,
|
||||
message=message,
|
||||
exc=exc,
|
||||
)
|
||||
await events.emit(
|
||||
"document_rewrite",
|
||||
phase="done",
|
||||
payload={
|
||||
"type": "error",
|
||||
"message": diagnostic["message"],
|
||||
"stage": stage,
|
||||
"errorCode": code,
|
||||
"diagnostic": diagnostic,
|
||||
},
|
||||
)
|
||||
await self._job_store.update_progress(
|
||||
job_id,
|
||||
lease_owner=lease_owner,
|
||||
status="failed",
|
||||
phase="done",
|
||||
progress=100,
|
||||
error_code=code,
|
||||
error_message=diagnostic["message"],
|
||||
result_snapshot={
|
||||
"kind": "artifact_document_rewrite",
|
||||
"status": "failed",
|
||||
"diagnostic": diagnostic,
|
||||
},
|
||||
)
|
||||
|
||||
async def _mark_document_rewrite_failed(
|
||||
self,
|
||||
job_id: str,
|
||||
@@ -1710,9 +1759,25 @@ class DeepResearchJobExecutor:
|
||||
user_id=user_id or None,
|
||||
lease_owner=lease_owner,
|
||||
)
|
||||
# Chat/regenerate writing is live-first. Persist the finished report
|
||||
# after the model returns; do not wait on the event table to display.
|
||||
events_for_write = (
|
||||
LiveFirstEventSink(recording_sink) if entry in ("chat", "regenerate") else recording_sink
|
||||
)
|
||||
started_payload = {
|
||||
"code": "write_job_started",
|
||||
"message": "写作任务已启动,正在读取已收集材料",
|
||||
"recoverable": True,
|
||||
}
|
||||
with suppress(Exception):
|
||||
await events_for_write.publish_live(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload=started_payload,
|
||||
)
|
||||
|
||||
# LLM backend (single chokepoint).
|
||||
llm_backend = DeerFlowCompletionBackend(config, event_sink=recording_sink, cancellation=cancel_token)
|
||||
llm_backend = DeerFlowCompletionBackend(config, event_sink=events_for_write, cancellation=cancel_token)
|
||||
|
||||
# Material provider. First write prefers the source table (full pool
|
||||
# persisted when collection finished). The job snapshot only carries a
|
||||
@@ -1725,7 +1790,7 @@ class DeepResearchJobExecutor:
|
||||
if force_no_materials:
|
||||
material_provider = StaticHarvestMaterialProvider([])
|
||||
try:
|
||||
await recording_sink.emit(
|
||||
await events_for_write.publish_live(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
@@ -1734,7 +1799,7 @@ class DeepResearchJobExecutor:
|
||||
"recoverable": True,
|
||||
},
|
||||
)
|
||||
await recording_sink.emit(
|
||||
await events_for_write.publish_live(
|
||||
"collection_completed",
|
||||
phase="collecting",
|
||||
payload={"source": "force_no_materials", "materialCount": 0},
|
||||
@@ -1748,7 +1813,7 @@ class DeepResearchJobExecutor:
|
||||
user_id=user_id,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
query=query,
|
||||
events=recording_sink,
|
||||
events=events_for_write,
|
||||
cancellation=cancel_token,
|
||||
collector_materials=collector_materials,
|
||||
materials_authoritative=materials_authoritative,
|
||||
@@ -1782,18 +1847,29 @@ class DeepResearchJobExecutor:
|
||||
materials=material_provider,
|
||||
context=context_compressor,
|
||||
artifacts=artifact_writer,
|
||||
events=recording_sink,
|
||||
events=events_for_write,
|
||||
images=build_image_provider(),
|
||||
)
|
||||
|
||||
job = await self._job_store.get_unscoped(job_id)
|
||||
job = None
|
||||
if self._job_store is not None:
|
||||
try:
|
||||
job = await self._job_store.get_unscoped(job_id)
|
||||
except Exception: # noqa: BLE001 - writing does not need the job row
|
||||
logger.warning("deep-research job %s could not reload its row before writing", job_id, exc_info=True)
|
||||
interrupted_plan = None
|
||||
if entry != "chat":
|
||||
try:
|
||||
interrupted_plan = await self._load_interrupted_plan(job_id)
|
||||
except Exception: # noqa: BLE001 - first write has no plan review
|
||||
logger.warning("deep-research job %s could not reload an interrupted plan", job_id, exc_info=True)
|
||||
request = DeepResearchRequest(
|
||||
session_id=session_id,
|
||||
user_id=user_id,
|
||||
query=query,
|
||||
config=config,
|
||||
resume_payload=(job or {}).get("awaiting_payload") or {},
|
||||
interrupted_plan=await self._load_interrupted_plan(job_id),
|
||||
interrupted_plan=interrupted_plan,
|
||||
)
|
||||
|
||||
engine = DeepResearchEngine()
|
||||
@@ -1805,7 +1881,7 @@ class DeepResearchJobExecutor:
|
||||
llm=llm_backend,
|
||||
report_markdown=result.report_markdown,
|
||||
language=config.language,
|
||||
events=recording_sink,
|
||||
events=events_for_write,
|
||||
cancellation=cancel_token,
|
||||
)
|
||||
# Fold the summary call's tokens into the billed totals.
|
||||
@@ -1813,6 +1889,7 @@ class DeepResearchJobExecutor:
|
||||
diagnostics = [
|
||||
*getattr(base_sink, "diagnostics", []),
|
||||
*recording_sink.diagnostics,
|
||||
*getattr(events_for_write, "diagnostics", []),
|
||||
]
|
||||
if diagnostics:
|
||||
result = result.model_copy(update={"diagnostics": diagnostics})
|
||||
@@ -1902,14 +1979,11 @@ class DeepResearchJobExecutor:
|
||||
):
|
||||
"""Build the first-write material pool.
|
||||
|
||||
Prefer the source table (the full collected pool). The browser list is
|
||||
a small fallback used only when that table is empty. Checkpoint harvest
|
||||
is last. An authoritative empty list (count 0) is a true no-materials
|
||||
write and must not pick leftover source rows.
|
||||
|
||||
Emits ``collection_completed`` plus one ``source_added`` per material so
|
||||
the sources panel fills exactly like the legacy channel (the recording
|
||||
sink persists them to the source store).
|
||||
If the write POST already carried collector materials, those are the
|
||||
writing input. The production MySQL source table is the path that
|
||||
hangs after the first live frame; do not read or upsert it before the
|
||||
report starts. The table is only consulted when the snapshot is empty
|
||||
(refresh / older clients). Checkpoint harvest stays last.
|
||||
"""
|
||||
from deerflow.agents.deep_research.adapters.material import dedupe_materials
|
||||
from deerflow.agents.deep_research.collection.thread_harvester import (
|
||||
@@ -1919,6 +1993,15 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
|
||||
cancellation.raise_if_cancelled()
|
||||
|
||||
async def _live_progress(type: str, *, phase: str = "collecting", payload: dict[str, Any] | None = None) -> None: # noqa: A002
|
||||
publish_live = getattr(events, "publish_live", None)
|
||||
if callable(publish_live):
|
||||
await publish_live(type, phase=phase, payload=payload or {})
|
||||
return
|
||||
# Tests and older sinks without a live hub still record through emit.
|
||||
await events.emit(type, phase=phase, payload=payload or {})
|
||||
|
||||
harvested: list[ResearchMaterial] = []
|
||||
agent_summary = ""
|
||||
browser_material_count = 0
|
||||
@@ -1959,7 +2042,7 @@ class DeepResearchJobExecutor:
|
||||
if row.get("id")
|
||||
]
|
||||
|
||||
async def try_browser_snapshot() -> None:
|
||||
async def try_browser_snapshot(*, persist: bool = True) -> None:
|
||||
nonlocal browser_material_count, browser_snapshot_persisted, harvested, material_channel
|
||||
raw_count = snapshot_count
|
||||
if raw_count == 0 and expected_material_count > 0:
|
||||
@@ -1971,30 +2054,31 @@ class DeepResearchJobExecutor:
|
||||
else []
|
||||
)
|
||||
try:
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"code": "browser_snapshot_received",
|
||||
"message": (
|
||||
f"后端已收到前端素材快照 {raw_count} 条"
|
||||
if raw_count
|
||||
else (
|
||||
"后端未收到前端素材快照(字段缺失或空数组)。"
|
||||
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
|
||||
)
|
||||
),
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
|
||||
"receivedType": type(collector_materials).__name__,
|
||||
"receivedCount": raw_count,
|
||||
"expectedCount": expected_material_count,
|
||||
"sampleKeys": sample_keys[:20],
|
||||
},
|
||||
publish_live = getattr(events, "publish_live", None)
|
||||
receipt_payload = {
|
||||
"code": "browser_snapshot_received",
|
||||
"message": (
|
||||
f"后端已收到前端素材快照 {raw_count} 条"
|
||||
if raw_count
|
||||
else (
|
||||
"后端未收到前端素材快照(字段缺失或空数组)。"
|
||||
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
|
||||
)
|
||||
),
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
|
||||
"receivedType": type(collector_materials).__name__,
|
||||
"receivedCount": raw_count,
|
||||
"expectedCount": expected_material_count,
|
||||
"sampleKeys": sample_keys[:20],
|
||||
},
|
||||
)
|
||||
}
|
||||
if callable(publish_live):
|
||||
await publish_live("warning", phase="collecting", payload=receipt_payload)
|
||||
elif persist:
|
||||
await events.emit("warning", phase="collecting", payload=receipt_payload)
|
||||
except Exception: # noqa: BLE001 - diagnostics never block writing
|
||||
logger.warning("failed to publish browser snapshot receipt", exc_info=True)
|
||||
if not collector_materials:
|
||||
@@ -2023,7 +2107,7 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
return
|
||||
material_channel = "browser_snapshot"
|
||||
if self._source_store is None:
|
||||
if not persist or self._source_store is None:
|
||||
return
|
||||
browser_snapshot_persisted = True
|
||||
for material in harvested:
|
||||
@@ -2080,12 +2164,15 @@ class DeepResearchJobExecutor:
|
||||
for attempt in range(attempts):
|
||||
cancellation.raise_if_cancelled()
|
||||
try:
|
||||
rows = await self._source_store.list_by_session(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
selected=None,
|
||||
limit=500,
|
||||
include_content=True,
|
||||
rows = await asyncio.wait_for(
|
||||
self._source_store.list_by_session(
|
||||
session_id,
|
||||
user_id=user_id,
|
||||
selected=None,
|
||||
limit=500,
|
||||
include_content=True,
|
||||
),
|
||||
timeout=_SOURCE_READ_TIMEOUT_SECONDS,
|
||||
)
|
||||
persisted_materials = persisted_rows_to_materials(rows)
|
||||
durable_material_count = len(persisted_materials)
|
||||
@@ -2093,7 +2180,7 @@ class DeepResearchJobExecutor:
|
||||
if harvested:
|
||||
material_channel = "durable_sources"
|
||||
try:
|
||||
await events.emit(
|
||||
await _live_progress(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
@@ -2141,7 +2228,7 @@ class DeepResearchJobExecutor:
|
||||
)
|
||||
|
||||
if skip_leftover_sources:
|
||||
await try_browser_snapshot()
|
||||
await try_browser_snapshot(persist=False)
|
||||
if not harvested:
|
||||
channel_failures.append(
|
||||
{
|
||||
@@ -2150,20 +2237,32 @@ class DeepResearchJobExecutor:
|
||||
"detail": "前端传入素材为空,已进入无素材写作",
|
||||
}
|
||||
)
|
||||
elif snapshot_count > 0:
|
||||
# Same lesson as the empty-table hang: the collected materials are
|
||||
# already on this job. Do not go back to the source table before
|
||||
# writing — that read is what stalls on the production server.
|
||||
await try_browser_snapshot(persist=False)
|
||||
else:
|
||||
await try_durable_sources(retry=True)
|
||||
# Prefer the source table, but do not stall on an empty/slow table
|
||||
# when the write POST already carried a compact material snapshot.
|
||||
await try_durable_sources(retry=False)
|
||||
if not harvested:
|
||||
await try_browser_snapshot()
|
||||
if not harvested:
|
||||
await try_durable_sources(retry=True)
|
||||
if not harvested:
|
||||
try:
|
||||
if self._checkpointer is None:
|
||||
raise RuntimeError("Deep Research checkpointer is unavailable")
|
||||
if not runtime_thread_id:
|
||||
raise RuntimeError("Deep Research collector thread is unavailable")
|
||||
checkpoint_materials, checkpoint_summary = await harvest_thread_materials(
|
||||
checkpointer=self._checkpointer,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
query=query,
|
||||
checkpoint_materials, checkpoint_summary = await asyncio.wait_for(
|
||||
harvest_thread_materials(
|
||||
checkpointer=self._checkpointer,
|
||||
runtime_thread_id=runtime_thread_id,
|
||||
query=query,
|
||||
),
|
||||
timeout=_CHECKPOINT_HARVEST_TIMEOUT_SECONDS,
|
||||
)
|
||||
checkpoint_material_count = len(checkpoint_materials)
|
||||
agent_summary = checkpoint_summary or ""
|
||||
@@ -2201,70 +2300,43 @@ class DeepResearchJobExecutor:
|
||||
# temporarily unavailable event table must never prevent the writer
|
||||
# from receiving this already-collected evidence.
|
||||
material_provider = StaticHarvestMaterialProvider(harvested)
|
||||
try:
|
||||
await events.emit(
|
||||
"collection_completed",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"source": material_channel,
|
||||
"materialCount": material_provider.material_count,
|
||||
completed_payload = {
|
||||
"source": material_channel,
|
||||
"materialCount": material_provider.material_count,
|
||||
"browserMaterialCount": browser_material_count,
|
||||
"checkpointMaterialCount": checkpoint_material_count,
|
||||
"durableMaterialCount": durable_material_count,
|
||||
"expectedMaterialCount": expected_material_count,
|
||||
"agentSummaryChars": len(agent_summary),
|
||||
}
|
||||
with suppress(Exception):
|
||||
await _live_progress("collection_completed", phase="collecting", payload=completed_payload)
|
||||
# Do not persist source_added during writing. Snapshot / leftover
|
||||
# collection already owns the evidence; MySQL upserts here freeze the
|
||||
# first frame. The finished report is stored afterwards.
|
||||
if material_provider.material_count and material_channel == "browser_snapshot":
|
||||
snapshot_payload = {
|
||||
"code": "material_channel_snapshot_used",
|
||||
"message": "已使用本次请求携带的已收集素材撰写,未再等待素材表",
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "client_material_snapshot",
|
||||
"selectedChannel": material_channel,
|
||||
"browserMaterialCount": browser_material_count,
|
||||
"checkpointMaterialCount": checkpoint_material_count,
|
||||
"durableMaterialCount": durable_material_count,
|
||||
"expectedMaterialCount": expected_material_count,
|
||||
"agentSummaryChars": len(agent_summary),
|
||||
"runtimeThreadId": runtime_thread_id,
|
||||
"failures": channel_failures,
|
||||
},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - progress telemetry must not lose evidence
|
||||
logger.warning("chat deep-research collection event failed; retaining harvested materials", exc_info=True)
|
||||
# A last-resort browser snapshot is written directly above. Avoid a
|
||||
# second source-added projection only when every direct write succeeded.
|
||||
if material_channel != "browser_snapshot" or not browser_snapshot_persisted:
|
||||
for material in harvested:
|
||||
try:
|
||||
await events.emit(
|
||||
"source_added",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"sourceId": material.id,
|
||||
"title": material.title,
|
||||
"url": material.url,
|
||||
"sourceType": material.source_type,
|
||||
"source": material.to_persistence_dict(material.query),
|
||||
},
|
||||
)
|
||||
except Exception: # noqa: BLE001 - source rows are a projection, not the input
|
||||
logger.warning(
|
||||
"chat deep-research source event failed for %s; retaining harvested materials",
|
||||
material.id,
|
||||
exc_info=True,
|
||||
)
|
||||
if material_provider.material_count and material_channel == "browser_snapshot":
|
||||
}
|
||||
try:
|
||||
await events.emit(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
"code": "material_channel_fallback_used",
|
||||
"message": "素材表为空,已使用前端兜底素材撰写",
|
||||
"recoverable": True,
|
||||
"diagnostic": {
|
||||
"stage": "collecting",
|
||||
"outcome": "client_material_fallback",
|
||||
"selectedChannel": material_channel,
|
||||
"browserMaterialCount": browser_material_count,
|
||||
"checkpointMaterialCount": checkpoint_material_count,
|
||||
"durableMaterialCount": durable_material_count,
|
||||
"runtimeThreadId": runtime_thread_id,
|
||||
"failures": channel_failures,
|
||||
},
|
||||
},
|
||||
)
|
||||
await _live_progress("warning", phase="collecting", payload=snapshot_payload)
|
||||
except Exception: # noqa: BLE001 - diagnostics never block writing
|
||||
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
|
||||
logger.warning("failed to publish material snapshot diagnostic", exc_info=True)
|
||||
elif material_provider.material_count and material_channel == "runtime_checkpoint":
|
||||
try:
|
||||
await events.emit(
|
||||
await _live_progress(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
@@ -2287,7 +2359,7 @@ class DeepResearchJobExecutor:
|
||||
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
|
||||
if not material_provider.material_count and not force_no_materials:
|
||||
try:
|
||||
await events.emit(
|
||||
await _live_progress(
|
||||
"warning",
|
||||
phase="collecting",
|
||||
payload={
|
||||
|
||||
Reference in new issue
Block a user