sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)

This commit is contained in:
gozeros committed 2026-10-21 14:51:31 +08:00
1 parent 4f7f07504a
commit 8bc8b0e082
1651 files changed
+278599 -11416

No files matched your search

@@ -19,8 +19,10 @@ import asyncio
import copy
import inspect
import logging
from collections.abc import Awaitable
from dataclasses import dataclass, field
from functools import lru_cache
from time import perf_counter
from typing import TYPE_CHECKING, Any, Literal, cast
from langgraph.checkpoint.base import empty_checkpoint
@@ -43,6 +45,11 @@ logger = logging.getLogger(__name__)
# Valid stream_mode values for LangGraph's graph.astream()
_VALID_LG_MODES = {"values", "updates", "checkpoints", "tasks", "debug", "messages", "custom"}
# This budget applies only after graph execution has returned. Long-running
# tools, research and subagents retain their own execution/cancellation policy.
_RUN_FINALIZATION_TIMEOUT_SECONDS = 10.0
_PENDING_COMPLETION_TASKS: set[asyncio.Future[Any]] = set()
def _log_background_title_finalize_failure(thread_id: str):
@@ -131,6 +138,29 @@ def _agent_factory_supports_app_config(agent_factory: Any) -> bool:
return _compute_agent_factory_supports_app_config(agent_factory)
def _build_agent_with_timing(agent_factory: Any, config: Any, app_config: AppConfig | None, run_id: str) -> Any:
# Preserve the original task/ContextVars/event loop. Moving this factory to
# a thread changes deferred-tool and MCP lifecycle behavior; measure first.
started = perf_counter()
outcome = "error"
try:
if app_config is not None and _agent_factory_supports_app_config(agent_factory):
agent = agent_factory(config=config, app_config=app_config)
else:
agent = agent_factory(config=config)
outcome = "ok"
return agent
finally:
elapsed_ms = (perf_counter() - started) * 1000
log = logger.warning if elapsed_ms >= 500 else logger.debug
try:
log("AGENT_BUILD_TIMING run_id=%s outcome=%s elapsed_ms=%.2f", run_id, outcome, elapsed_ms)
except Exception:
# Timing is diagnostic only; preserve the factory result/exception
# even if a deployment's custom log handler is unavailable.
pass
async def run_agent(
bridge: StreamBridge,
run_manager: RunManager,
@@ -160,11 +190,11 @@ async def run_agent(
store = ctx.store
event_store = ctx.event_store
run_events_config = ctx.run_events_config
thread_store = ctx.thread_store
run_id = record.run_id
thread_id = record.thread_id
requested_modes: set[str] = set(stream_modes or ["values"])
internal_stream_call_ids: set[str] = set()
pre_run_checkpoint_id: str | None = None
pre_run_snapshot: dict[str, Any] | None = None
snapshot_capture_failed = False
@@ -219,14 +249,17 @@ async def run_agent(
logger.warning("Could not capture pre-run checkpoint snapshot for run %s", run_id, exc_info=True)
# 2. Publish metadata — useStream needs both run_id AND thread_id
await bridge.publish(
run_id,
"metadata",
{
"run_id": run_id,
"thread_id": thread_id,
},
)
metadata_event: dict[str, Any] = {
"run_id": run_id,
"thread_id": thread_id,
}
retrieval_trace = (record.metadata or {}).get("knowledge_retrieval_trace")
if isinstance(retrieval_trace, dict):
metadata_event["knowledge_retrieval_trace"] = retrieval_trace
metadata_event["knowledge_retrieval_url"] = (
f"/api/threads/{thread_id}/reference-batches"
)
await bridge.publish(run_id, "metadata", metadata_event)
# 3. Build the agent
from langchain_core.runnables import RunnableConfig
@@ -247,10 +280,7 @@ async def run_agent(
config.setdefault("callbacks", []).append(journal)
runnable_config = RunnableConfig(**config)
if ctx.app_config is not None and _agent_factory_supports_app_config(agent_factory):
agent = agent_factory(config=runnable_config, app_config=ctx.app_config)
else:
agent = agent_factory(config=runnable_config)
agent = _build_agent_with_timing(agent_factory, runnable_config, ctx.app_config, run_id)
# 4. Attach checkpointer and store
if checkpointer is not None:
@@ -324,7 +354,13 @@ async def run_agent(
logger.info("Run %s abort requested — stopping", run_id)
break
sse_event = _lg_mode_to_sse_event(single_mode)
await bridge.publish(run_id, sse_event, serialize(chunk, mode=single_mode))
serialized = serialize(
chunk,
mode=single_mode,
known_internal_tool_call_ids=internal_stream_call_ids,
)
if serialized is not None:
await bridge.publish(run_id, sse_event, serialized)
else:
# Multiple modes or subgraphs: astream yields tuples
async for item in agent.astream(
@@ -342,13 +378,19 @@ async def run_agent(
continue
sse_event = _lg_mode_to_sse_event(mode)
await bridge.publish(run_id, sse_event, serialize(chunk, mode=mode))
serialized = serialize(
chunk,
mode=mode,
known_internal_tool_call_ids=internal_stream_call_ids,
)
if serialized is not None:
await bridge.publish(run_id, sse_event, serialized)
# 8. Final status
if record.abort_event.is_set():
action = record.abort_action
if action == "rollback":
await run_manager.set_status(run_id, RunStatus.error, error="Rolled back by user")
await _set_terminal_status(run_manager, record, RunStatus.error, error="Rolled back by user")
try:
await _rollback_to_pre_run_checkpoint(
checkpointer=checkpointer,
@@ -362,7 +404,7 @@ async def run_agent(
except Exception:
logger.warning("Failed to rollback checkpoint for run %s", run_id, exc_info=True)
else:
await run_manager.set_status(run_id, RunStatus.interrupted)
await _set_terminal_status(run_manager, record, RunStatus.interrupted)
else:
# LLMErrorHandlingMiddleware intentionally converts an exhausted
# provider failure into a user-safe AIMessage instead of raising.
@@ -372,9 +414,10 @@ async def run_agent(
# generation had silently stopped halfway through. Inspect the
# terminal checkpoint before declaring success so the persisted run
# state and the user-visible SSE outcome stay consistent.
terminal_llm_error = await _terminal_llm_error_reason(
checkpointer=checkpointer,
thread_id=thread_id,
terminal_llm_error = await _await_completion_io(
_terminal_llm_error_reason(checkpointer=checkpointer, thread_id=thread_id),
run_id=run_id,
phase="terminal_checkpoint",
)
if terminal_llm_error is not None:
error_message = "本轮模型生成未能完成,已保留当前对话内容,请稍后重试。"
@@ -383,8 +426,9 @@ async def run_agent(
run_id,
terminal_llm_error,
)
await run_manager.set_status(
run_id,
await _set_terminal_status(
run_manager,
record,
RunStatus.error,
error=error_message,
)
@@ -397,12 +441,12 @@ async def run_agent(
},
)
else:
await run_manager.set_status(run_id, RunStatus.success)
await _set_terminal_status(run_manager, record, RunStatus.success)
except asyncio.CancelledError:
action = record.abort_action
if action == "rollback":
await run_manager.set_status(run_id, RunStatus.error, error="Rolled back by user")
await _set_terminal_status(run_manager, record, RunStatus.error, error="Rolled back by user")
try:
await _rollback_to_pre_run_checkpoint(
checkpointer=checkpointer,
@@ -416,13 +460,13 @@ async def run_agent(
except Exception:
logger.warning("Run %s cancellation rollback failed", run_id, exc_info=True)
else:
await run_manager.set_status(run_id, RunStatus.interrupted)
await _set_terminal_status(run_manager, record, RunStatus.interrupted)
logger.info("Run %s was cancelled", run_id)
except Exception as exc:
error_msg = f"{exc}"
logger.exception("Run %s failed: %s", run_id, error_msg)
await run_manager.set_status(run_id, RunStatus.error, error=error_msg)
await _set_terminal_status(run_manager, record, RunStatus.error, error=error_msg)
await bridge.publish(
run_id,
"error",
@@ -433,72 +477,170 @@ async def run_agent(
)
finally:
# Flush any buffered journal events and persist completion data
if journal is not None:
try:
await journal.flush()
except Exception:
logger.warning("Failed to flush journal for run %s", run_id, exc_info=True)
try:
# Persist token usage + convenience fields to RunStore
completion = journal.get_completion_data()
await run_manager.update_run_completion(run_id, status=record.status.value, **completion)
except Exception:
logger.warning("Failed to persist run completion for %s (non-fatal)", run_id, exc_info=True)
# Sync title from checkpoint to threads_meta.display_name
if checkpointer is not None and thread_store is not None:
try:
ckpt_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
ckpt_tuple = await checkpointer.aget_tuple(ckpt_config)
if ckpt_tuple is not None:
ckpt = getattr(ckpt_tuple, "checkpoint", {}) or {}
channel_values = ckpt.get("channel_values", {}) or {}
title = channel_values.get("title")
if title:
await thread_store.update_display_name(thread_id, title)
if (
record.status == RunStatus.success
and channel_values.get("title_provisional") is True
):
log_title_debug(
"Run %s: scheduling background title generation for thread %s",
run_id,
thread_id,
)
title_task = asyncio.create_task(
finalize_thread_title(
checkpointer=checkpointer,
thread_store=thread_store,
thread_id=thread_id,
source_state=copy.deepcopy(channel_values),
app_config=ctx.app_config,
),
name=f"title-finalize:{thread_id}",
)
title_task.add_done_callback(_log_background_title_finalize_failure(thread_id))
elif record.status == RunStatus.success:
log_title_debug(
"Run %s: background title generation not scheduled for thread %s because title_provisional=%r title=%r",
run_id,
thread_id,
channel_values.get("title_provisional"),
channel_values.get("title"),
)
except Exception:
logger.debug("Failed to sync title for thread %s (non-fatal)", thread_id)
# Update threads_meta status based on run outcome
if thread_store is not None:
try:
final_status = "idle" if record.status == RunStatus.success else record.status.value
await thread_store.update_status(thread_id, final_status)
except Exception:
logger.debug("Failed to update thread_meta status for %s (non-fatal)", thread_id)
await bridge.publish_end(run_id)
asyncio.create_task(bridge.cleanup(run_id, delay=60))
try:
await _await_completion_io(
_finalize_run(
journal=journal,
run_manager=run_manager,
record=record,
ctx=ctx,
),
run_id=run_id,
phase="metadata_cleanup",
)
except TimeoutError:
logger.warning("Run %s completion metadata exceeded its time budget; ending SSE", run_id)
except asyncio.CancelledError:
# A second cancellation can arrive while handling the original
# cancellation or during normal cleanup. Never lose the terminator.
logger.info("Run %s completion metadata was cancelled; ending SSE", run_id)
except Exception:
logger.warning("Failed to finalize run %s (non-fatal)", run_id, exc_info=True)
finally:
await bridge.publish_end(run_id)
asyncio.create_task(bridge.cleanup(run_id, delay=60))
async def _await_completion_io[T](
operation: Awaitable[T],
*,
run_id: str,
phase: str,
) -> T:
"""Bound completion-only I/O even if a backend delays cancellation.
Unlike wait_for, wait does not wait indefinitely for the cancelled
operation to acknowledge cancellation. Do not use this for executing the
graph: detaching a live tool would release run concurrency prematurely.
"""
task = asyncio.ensure_future(operation)
_PENDING_COMPLETION_TASKS.add(task)
def consume_result(completed: asyncio.Future) -> None:
_PENDING_COMPLETION_TASKS.discard(completed)
if not completed.cancelled():
completed.exception()
task.add_done_callback(consume_result)
try:
done, _ = await asyncio.wait({task}, timeout=_RUN_FINALIZATION_TIMEOUT_SECONDS)
if not done:
logger.warning("RUN_COMPLETION_TIMEOUT run_id=%s phase=%s", run_id, phase)
raise TimeoutError(f"Run completion timed out during {phase}")
return task.result()
finally:
if not task.done():
task.cancel()
async def _set_terminal_status(run_manager: RunManager, record: RunRecord, status: RunStatus, *, error: str | None = None) -> None:
try:
await _await_completion_io(
run_manager.set_status(record.run_id, status, error=error),
run_id=record.run_id,
phase="terminal_status",
)
except TimeoutError:
# set_status updates the registry before awaiting its best-effort store.
# Keep that local outcome authoritative if persistence never returns.
record.status = status
if error is not None:
record.error = error
logger.warning("Run %s terminal status persistence timed out", record.run_id)
async def _finalize_run(*, journal: Any, run_manager: RunManager, record: RunRecord, ctx: RunContext) -> None:
"""Best-effort metadata writes after graph execution, under one budget."""
run_id = record.run_id
thread_id = record.thread_id
checkpointer = ctx.checkpointer
thread_store = ctx.thread_store
# Flush any buffered journal events and persist completion data
if journal is not None:
try:
await journal.flush()
_check_completion_cancellation()
except Exception:
logger.warning("Failed to flush journal for run %s", run_id, exc_info=True)
try:
# Persist token usage + convenience fields to RunStore
completion = journal.get_completion_data()
await run_manager.update_run_completion(run_id, status=record.status.value, **completion)
_check_completion_cancellation()
except Exception:
logger.warning("Failed to persist run completion for %s (non-fatal)", run_id, exc_info=True)
# Sync title from checkpoint to threads_meta.display_name
if checkpointer is not None and thread_store is not None:
try:
ckpt_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
ckpt_tuple = await checkpointer.aget_tuple(ckpt_config)
if not await _completion_is_current(run_manager, record):
return
if ckpt_tuple is not None:
ckpt = getattr(ckpt_tuple, "checkpoint", {}) or {}
channel_values = ckpt.get("channel_values", {}) or {}
title = channel_values.get("title")
if title:
await thread_store.update_display_name(thread_id, title)
if not await _completion_is_current(run_manager, record):
return
if (
record.status == RunStatus.success
and channel_values.get("title_provisional") is True
):
log_title_debug(
"Run %s: scheduling background title generation for thread %s",
run_id,
thread_id,
)
title_task = asyncio.create_task(
finalize_thread_title(
checkpointer=checkpointer,
thread_store=thread_store,
thread_id=thread_id,
source_state=copy.deepcopy(channel_values),
app_config=ctx.app_config,
),
name=f"title-finalize:{thread_id}",
)
title_task.add_done_callback(_log_background_title_finalize_failure(thread_id))
elif record.status == RunStatus.success:
log_title_debug(
"Run %s: background title generation not scheduled for thread %s because title_provisional=%r title=%r",
run_id,
thread_id,
channel_values.get("title_provisional"),
channel_values.get("title"),
)
except Exception:
logger.debug("Failed to sync title for thread %s (non-fatal)", thread_id)
# Update threads_meta status based on run outcome
if thread_store is not None:
try:
if not await _completion_is_current(run_manager, record):
return
final_status = "idle" if record.status == RunStatus.success else record.status.value
await thread_store.update_status(thread_id, final_status)
except Exception:
logger.debug("Failed to update thread_meta status for %s (non-fatal)", thread_id)
def _check_completion_cancellation() -> None:
task = asyncio.current_task()
if task is not None and task.cancelling():
# Some database adapters delay or suppress cancellation. They must not
# resume subsequent title/status writes after the cleanup budget ends.
raise asyncio.CancelledError
async def _completion_is_current(run_manager: RunManager, record: RunRecord) -> bool:
_check_completion_cancellation()
records = await run_manager.list_by_thread(record.thread_id)
_check_completion_cancellation()
return not records or records[-1].run_id == record.run_id
# ---------------------------------------------------------------------------