sync update from LAN repo (source ji @ 85078a31)

This commit is contained in:
gozeros committed 2026-10-21 15:09:16 +08:00
1 parent 5d6627af50
commit e602d88695
864 files changed
+99150 -6346

No files matched your search

@@ -61,9 +61,11 @@ logger = logging.getLogger(__name__)
# ── constants ───────────────────────────────────────────────────────────────
WORKER_ID = f"w-{os.getpid()}-{_uuid.uuid4().hex[:8]}"
EXECUTOR_REVISION = "deep-research-db-first-client-fallback-20260904-v13"
_SOURCE_READ_ATTEMPTS = 16
EXECUTOR_REVISION = "deep-research-write-live-then-persist-20260916-v18"
_SOURCE_READ_ATTEMPTS = 4
_SOURCE_READ_DELAY_SECONDS = 0.5
_SOURCE_READ_TIMEOUT_SECONDS = 3.0
_CHECKPOINT_HARVEST_TIMEOUT_SECONDS = 8.0
_LEASE_TTL_SECONDS = 180
_HEARTBEAT_INTERVAL = 20.0
_CANCEL_POLL_INTERVAL = 5.0
@@ -219,6 +221,35 @@ class _JobUser:
self.id = user_id
class LiveFirstEventSink:
"""Show write progress immediately. Persist nothing until the report exists.
``emit`` is persist-then-stream on the production MySQL. That wait is what
hid the first frame and then froze the writer. Chat / regenerate writes
therefore fan out to the live hub only; ``_persist_completed_report_projection``
stores the finished markdown afterwards.
"""
def __init__(self, inner) -> None:
self._inner = inner
self.diagnostics: list[dict[str, Any]] = list(getattr(inner, "diagnostics", []) or [])
async def emit(self, type, *, phase="initializing", payload=None): # noqa: A002
diagnostic = (payload or {}).get("diagnostic")
if isinstance(diagnostic, dict):
self.diagnostics.append(
{
"code": str((payload or {}).get("code") or type),
**diagnostic,
}
)
await self.publish_live(type, phase=phase, payload=payload or {})
return None
async def publish_live(self, type, *, phase="initializing", payload=None): # noqa: A002
await self._inner.publish_live(type, phase=phase, payload=payload or {})
# ── recording event sink (source persistence + progress fan-out) ─────────────
@@ -1710,9 +1741,25 @@ class DeepResearchJobExecutor:
user_id=user_id or None,
lease_owner=lease_owner,
)
# Chat/regenerate writing is live-first. Persist the finished report
# after the model returns; do not wait on the event table to display.
events_for_write = (
LiveFirstEventSink(recording_sink) if entry in ("chat", "regenerate") else recording_sink
)
started_payload = {
"code": "write_job_started",
"message": "写作任务已启动,正在读取已收集材料",
"recoverable": True,
}
with suppress(Exception):
await events_for_write.publish_live(
"warning",
phase="collecting",
payload=started_payload,
)
# LLM backend (single chokepoint).
llm_backend = DeerFlowCompletionBackend(config, event_sink=recording_sink, cancellation=cancel_token)
llm_backend = DeerFlowCompletionBackend(config, event_sink=events_for_write, cancellation=cancel_token)
# Material provider. First write prefers the source table (full pool
# persisted when collection finished). The job snapshot only carries a
@@ -1725,7 +1772,7 @@ class DeepResearchJobExecutor:
if force_no_materials:
material_provider = StaticHarvestMaterialProvider([])
try:
await recording_sink.emit(
await events_for_write.publish_live(
"warning",
phase="collecting",
payload={
@@ -1734,7 +1781,7 @@ class DeepResearchJobExecutor:
"recoverable": True,
},
)
await recording_sink.emit(
await events_for_write.publish_live(
"collection_completed",
phase="collecting",
payload={"source": "force_no_materials", "materialCount": 0},
@@ -1748,7 +1795,7 @@ class DeepResearchJobExecutor:
user_id=user_id,
runtime_thread_id=runtime_thread_id,
query=query,
events=recording_sink,
events=events_for_write,
cancellation=cancel_token,
collector_materials=collector_materials,
materials_authoritative=materials_authoritative,
@@ -1782,18 +1829,29 @@ class DeepResearchJobExecutor:
materials=material_provider,
context=context_compressor,
artifacts=artifact_writer,
events=recording_sink,
events=events_for_write,
images=build_image_provider(),
)
job = await self._job_store.get_unscoped(job_id)
job = None
if self._job_store is not None:
try:
job = await self._job_store.get_unscoped(job_id)
except Exception: # noqa: BLE001 - writing does not need the job row
logger.warning("deep-research job %s could not reload its row before writing", job_id, exc_info=True)
interrupted_plan = None
if entry != "chat":
try:
interrupted_plan = await self._load_interrupted_plan(job_id)
except Exception: # noqa: BLE001 - first write has no plan review
logger.warning("deep-research job %s could not reload an interrupted plan", job_id, exc_info=True)
request = DeepResearchRequest(
session_id=session_id,
user_id=user_id,
query=query,
config=config,
resume_payload=(job or {}).get("awaiting_payload") or {},
interrupted_plan=await self._load_interrupted_plan(job_id),
interrupted_plan=interrupted_plan,
)
engine = DeepResearchEngine()
@@ -1805,7 +1863,7 @@ class DeepResearchJobExecutor:
llm=llm_backend,
report_markdown=result.report_markdown,
language=config.language,
events=recording_sink,
events=events_for_write,
cancellation=cancel_token,
)
# Fold the summary call's tokens into the billed totals.
@@ -1813,6 +1871,7 @@ class DeepResearchJobExecutor:
diagnostics = [
*getattr(base_sink, "diagnostics", []),
*recording_sink.diagnostics,
*getattr(events_for_write, "diagnostics", []),
]
if diagnostics:
result = result.model_copy(update={"diagnostics": diagnostics})
@@ -1902,14 +1961,11 @@ class DeepResearchJobExecutor:
):
"""Build the first-write material pool.
Prefer the source table (the full collected pool). The browser list is
a small fallback used only when that table is empty. Checkpoint harvest
is last. An authoritative empty list (count 0) is a true no-materials
write and must not pick leftover source rows.
Emits ``collection_completed`` plus one ``source_added`` per material so
the sources panel fills exactly like the legacy channel (the recording
sink persists them to the source store).
If the write POST already carried collector materials, those are the
writing input. The production MySQL source table is the path that
hangs after the first live frame; do not read or upsert it before the
report starts. The table is only consulted when the snapshot is empty
(refresh / older clients). Checkpoint harvest stays last.
"""
from deerflow.agents.deep_research.adapters.material import dedupe_materials
from deerflow.agents.deep_research.collection.thread_harvester import (
@@ -1919,6 +1975,15 @@ class DeepResearchJobExecutor:
)
cancellation.raise_if_cancelled()
async def _live_progress(type: str, *, phase: str = "collecting", payload: dict[str, Any] | None = None) -> None: # noqa: A002
publish_live = getattr(events, "publish_live", None)
if callable(publish_live):
await publish_live(type, phase=phase, payload=payload or {})
return
# Tests and older sinks without a live hub still record through emit.
await events.emit(type, phase=phase, payload=payload or {})
harvested: list[ResearchMaterial] = []
agent_summary = ""
browser_material_count = 0
@@ -1959,7 +2024,7 @@ class DeepResearchJobExecutor:
if row.get("id")
]
async def try_browser_snapshot() -> None:
async def try_browser_snapshot(*, persist: bool = True) -> None:
nonlocal browser_material_count, browser_snapshot_persisted, harvested, material_channel
raw_count = snapshot_count
if raw_count == 0 and expected_material_count > 0:
@@ -1971,30 +2036,31 @@ class DeepResearchJobExecutor:
else []
)
try:
await events.emit(
"warning",
phase="collecting",
payload={
"code": "browser_snapshot_received",
"message": (
f"后端已收到前端素材快照 {raw_count} 条"
if raw_count
else (
"后端未收到前端素材快照(字段缺失或空数组)。"
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
)
),
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
"receivedType": type(collector_materials).__name__,
"receivedCount": raw_count,
"expectedCount": expected_material_count,
"sampleKeys": sample_keys[:20],
},
publish_live = getattr(events, "publish_live", None)
receipt_payload = {
"code": "browser_snapshot_received",
"message": (
f"后端已收到前端素材快照 {raw_count} 条"
if raw_count
else (
"后端未收到前端素材快照(字段缺失或空数组)。"
"若页面上已能看到检索结果,说明快照在发请求前就被抽空了。"
)
),
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "browser_snapshot_received" if raw_count else "browser_snapshot_missing",
"receivedType": type(collector_materials).__name__,
"receivedCount": raw_count,
"expectedCount": expected_material_count,
"sampleKeys": sample_keys[:20],
},
)
}
if callable(publish_live):
await publish_live("warning", phase="collecting", payload=receipt_payload)
elif persist:
await events.emit("warning", phase="collecting", payload=receipt_payload)
except Exception: # noqa: BLE001 - diagnostics never block writing
logger.warning("failed to publish browser snapshot receipt", exc_info=True)
if not collector_materials:
@@ -2023,7 +2089,7 @@ class DeepResearchJobExecutor:
)
return
material_channel = "browser_snapshot"
if self._source_store is None:
if not persist or self._source_store is None:
return
browser_snapshot_persisted = True
for material in harvested:
@@ -2080,12 +2146,15 @@ class DeepResearchJobExecutor:
for attempt in range(attempts):
cancellation.raise_if_cancelled()
try:
rows = await self._source_store.list_by_session(
session_id,
user_id=user_id,
selected=None,
limit=500,
include_content=True,
rows = await asyncio.wait_for(
self._source_store.list_by_session(
session_id,
user_id=user_id,
selected=None,
limit=500,
include_content=True,
),
timeout=_SOURCE_READ_TIMEOUT_SECONDS,
)
persisted_materials = persisted_rows_to_materials(rows)
durable_material_count = len(persisted_materials)
@@ -2093,7 +2162,7 @@ class DeepResearchJobExecutor:
if harvested:
material_channel = "durable_sources"
try:
await events.emit(
await _live_progress(
"warning",
phase="collecting",
payload={
@@ -2141,7 +2210,7 @@ class DeepResearchJobExecutor:
)
if skip_leftover_sources:
await try_browser_snapshot()
await try_browser_snapshot(persist=False)
if not harvested:
channel_failures.append(
{
@@ -2150,20 +2219,32 @@ class DeepResearchJobExecutor:
"detail": "前端传入素材为空,已进入无素材写作",
}
)
elif snapshot_count > 0:
# Same lesson as the empty-table hang: the collected materials are
# already on this job. Do not go back to the source table before
# writing — that read is what stalls on the production server.
await try_browser_snapshot(persist=False)
else:
await try_durable_sources(retry=True)
# Prefer the source table, but do not stall on an empty/slow table
# when the write POST already carried a compact material snapshot.
await try_durable_sources(retry=False)
if not harvested:
await try_browser_snapshot()
if not harvested:
await try_durable_sources(retry=True)
if not harvested:
try:
if self._checkpointer is None:
raise RuntimeError("Deep Research checkpointer is unavailable")
if not runtime_thread_id:
raise RuntimeError("Deep Research collector thread is unavailable")
checkpoint_materials, checkpoint_summary = await harvest_thread_materials(
checkpointer=self._checkpointer,
runtime_thread_id=runtime_thread_id,
query=query,
checkpoint_materials, checkpoint_summary = await asyncio.wait_for(
harvest_thread_materials(
checkpointer=self._checkpointer,
runtime_thread_id=runtime_thread_id,
query=query,
),
timeout=_CHECKPOINT_HARVEST_TIMEOUT_SECONDS,
)
checkpoint_material_count = len(checkpoint_materials)
agent_summary = checkpoint_summary or ""
@@ -2201,70 +2282,43 @@ class DeepResearchJobExecutor:
# temporarily unavailable event table must never prevent the writer
# from receiving this already-collected evidence.
material_provider = StaticHarvestMaterialProvider(harvested)
try:
await events.emit(
"collection_completed",
phase="collecting",
payload={
"source": material_channel,
"materialCount": material_provider.material_count,
completed_payload = {
"source": material_channel,
"materialCount": material_provider.material_count,
"browserMaterialCount": browser_material_count,
"checkpointMaterialCount": checkpoint_material_count,
"durableMaterialCount": durable_material_count,
"expectedMaterialCount": expected_material_count,
"agentSummaryChars": len(agent_summary),
}
with suppress(Exception):
await _live_progress("collection_completed", phase="collecting", payload=completed_payload)
# Do not persist source_added during writing. Snapshot / leftover
# collection already owns the evidence; MySQL upserts here freeze the
# first frame. The finished report is stored afterwards.
if material_provider.material_count and material_channel == "browser_snapshot":
snapshot_payload = {
"code": "material_channel_snapshot_used",
"message": "已使用本次请求携带的已收集素材撰写,未再等待素材表",
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "client_material_snapshot",
"selectedChannel": material_channel,
"browserMaterialCount": browser_material_count,
"checkpointMaterialCount": checkpoint_material_count,
"durableMaterialCount": durable_material_count,
"expectedMaterialCount": expected_material_count,
"agentSummaryChars": len(agent_summary),
"runtimeThreadId": runtime_thread_id,
"failures": channel_failures,
},
)
except Exception: # noqa: BLE001 - progress telemetry must not lose evidence
logger.warning("chat deep-research collection event failed; retaining harvested materials", exc_info=True)
# A last-resort browser snapshot is written directly above. Avoid a
# second source-added projection only when every direct write succeeded.
if material_channel != "browser_snapshot" or not browser_snapshot_persisted:
for material in harvested:
try:
await events.emit(
"source_added",
phase="collecting",
payload={
"sourceId": material.id,
"title": material.title,
"url": material.url,
"sourceType": material.source_type,
"source": material.to_persistence_dict(material.query),
},
)
except Exception: # noqa: BLE001 - source rows are a projection, not the input
logger.warning(
"chat deep-research source event failed for %s; retaining harvested materials",
material.id,
exc_info=True,
)
if material_provider.material_count and material_channel == "browser_snapshot":
}
try:
await events.emit(
"warning",
phase="collecting",
payload={
"code": "material_channel_fallback_used",
"message": "素材表为空,已使用前端兜底素材撰写",
"recoverable": True,
"diagnostic": {
"stage": "collecting",
"outcome": "client_material_fallback",
"selectedChannel": material_channel,
"browserMaterialCount": browser_material_count,
"checkpointMaterialCount": checkpoint_material_count,
"durableMaterialCount": durable_material_count,
"runtimeThreadId": runtime_thread_id,
"failures": channel_failures,
},
},
)
await _live_progress("warning", phase="collecting", payload=snapshot_payload)
except Exception: # noqa: BLE001 - diagnostics never block writing
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
logger.warning("failed to publish material snapshot diagnostic", exc_info=True)
elif material_provider.material_count and material_channel == "runtime_checkpoint":
try:
await events.emit(
await _live_progress(
"warning",
phase="collecting",
payload={
@@ -2287,7 +2341,7 @@ class DeepResearchJobExecutor:
logger.warning("failed to publish material fallback diagnostic", exc_info=True)
if not material_provider.material_count and not force_no_materials:
try:
await events.emit(
await _live_progress(
"warning",
phase="collecting",
payload={