sync: 鍚屾灞€鍩熺綉鏈€鏂颁唬鐮?(source master @ b69568e4)
This commit is contained in:
1 parent
4f7f07504a
commit
8bc8b0e082
1651 files changed
+278599
-11416
No files matched your search
+234
-92
@@ -19,8 +19,10 @@ import asyncio
|
||||
import copy
|
||||
import inspect
|
||||
import logging
|
||||
from collections.abc import Awaitable
|
||||
from dataclasses import dataclass, field
|
||||
from functools import lru_cache
|
||||
from time import perf_counter
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
@@ -43,6 +45,11 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# Valid stream_mode values for LangGraph's graph.astream()
|
||||
_VALID_LG_MODES = {"values", "updates", "checkpoints", "tasks", "debug", "messages", "custom"}
|
||||
|
||||
# This budget applies only after graph execution has returned. Long-running
|
||||
# tools, research and subagents retain their own execution/cancellation policy.
|
||||
_RUN_FINALIZATION_TIMEOUT_SECONDS = 10.0
|
||||
_PENDING_COMPLETION_TASKS: set[asyncio.Future[Any]] = set()
|
||||
|
||||
|
||||
def _log_background_title_finalize_failure(thread_id: str):
|
||||
@@ -131,6 +138,29 @@ def _agent_factory_supports_app_config(agent_factory: Any) -> bool:
|
||||
return _compute_agent_factory_supports_app_config(agent_factory)
|
||||
|
||||
|
||||
def _build_agent_with_timing(agent_factory: Any, config: Any, app_config: AppConfig | None, run_id: str) -> Any:
|
||||
# Preserve the original task/ContextVars/event loop. Moving this factory to
|
||||
# a thread changes deferred-tool and MCP lifecycle behavior; measure first.
|
||||
started = perf_counter()
|
||||
outcome = "error"
|
||||
try:
|
||||
if app_config is not None and _agent_factory_supports_app_config(agent_factory):
|
||||
agent = agent_factory(config=config, app_config=app_config)
|
||||
else:
|
||||
agent = agent_factory(config=config)
|
||||
outcome = "ok"
|
||||
return agent
|
||||
finally:
|
||||
elapsed_ms = (perf_counter() - started) * 1000
|
||||
log = logger.warning if elapsed_ms >= 500 else logger.debug
|
||||
try:
|
||||
log("AGENT_BUILD_TIMING run_id=%s outcome=%s elapsed_ms=%.2f", run_id, outcome, elapsed_ms)
|
||||
except Exception:
|
||||
# Timing is diagnostic only; preserve the factory result/exception
|
||||
# even if a deployment's custom log handler is unavailable.
|
||||
pass
|
||||
|
||||
|
||||
async def run_agent(
|
||||
bridge: StreamBridge,
|
||||
run_manager: RunManager,
|
||||
@@ -160,11 +190,11 @@ async def run_agent(
|
||||
store = ctx.store
|
||||
event_store = ctx.event_store
|
||||
run_events_config = ctx.run_events_config
|
||||
thread_store = ctx.thread_store
|
||||
|
||||
run_id = record.run_id
|
||||
thread_id = record.thread_id
|
||||
requested_modes: set[str] = set(stream_modes or ["values"])
|
||||
internal_stream_call_ids: set[str] = set()
|
||||
pre_run_checkpoint_id: str | None = None
|
||||
pre_run_snapshot: dict[str, Any] | None = None
|
||||
snapshot_capture_failed = False
|
||||
@@ -219,14 +249,17 @@ async def run_agent(
|
||||
logger.warning("Could not capture pre-run checkpoint snapshot for run %s", run_id, exc_info=True)
|
||||
|
||||
# 2. Publish metadata — useStream needs both run_id AND thread_id
|
||||
await bridge.publish(
|
||||
run_id,
|
||||
"metadata",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"thread_id": thread_id,
|
||||
},
|
||||
)
|
||||
metadata_event: dict[str, Any] = {
|
||||
"run_id": run_id,
|
||||
"thread_id": thread_id,
|
||||
}
|
||||
retrieval_trace = (record.metadata or {}).get("knowledge_retrieval_trace")
|
||||
if isinstance(retrieval_trace, dict):
|
||||
metadata_event["knowledge_retrieval_trace"] = retrieval_trace
|
||||
metadata_event["knowledge_retrieval_url"] = (
|
||||
f"/api/threads/{thread_id}/reference-batches"
|
||||
)
|
||||
await bridge.publish(run_id, "metadata", metadata_event)
|
||||
|
||||
# 3. Build the agent
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
@@ -247,10 +280,7 @@ async def run_agent(
|
||||
config.setdefault("callbacks", []).append(journal)
|
||||
|
||||
runnable_config = RunnableConfig(**config)
|
||||
if ctx.app_config is not None and _agent_factory_supports_app_config(agent_factory):
|
||||
agent = agent_factory(config=runnable_config, app_config=ctx.app_config)
|
||||
else:
|
||||
agent = agent_factory(config=runnable_config)
|
||||
agent = _build_agent_with_timing(agent_factory, runnable_config, ctx.app_config, run_id)
|
||||
|
||||
# 4. Attach checkpointer and store
|
||||
if checkpointer is not None:
|
||||
@@ -324,7 +354,13 @@ async def run_agent(
|
||||
logger.info("Run %s abort requested — stopping", run_id)
|
||||
break
|
||||
sse_event = _lg_mode_to_sse_event(single_mode)
|
||||
await bridge.publish(run_id, sse_event, serialize(chunk, mode=single_mode))
|
||||
serialized = serialize(
|
||||
chunk,
|
||||
mode=single_mode,
|
||||
known_internal_tool_call_ids=internal_stream_call_ids,
|
||||
)
|
||||
if serialized is not None:
|
||||
await bridge.publish(run_id, sse_event, serialized)
|
||||
else:
|
||||
# Multiple modes or subgraphs: astream yields tuples
|
||||
async for item in agent.astream(
|
||||
@@ -342,13 +378,19 @@ async def run_agent(
|
||||
continue
|
||||
|
||||
sse_event = _lg_mode_to_sse_event(mode)
|
||||
await bridge.publish(run_id, sse_event, serialize(chunk, mode=mode))
|
||||
serialized = serialize(
|
||||
chunk,
|
||||
mode=mode,
|
||||
known_internal_tool_call_ids=internal_stream_call_ids,
|
||||
)
|
||||
if serialized is not None:
|
||||
await bridge.publish(run_id, sse_event, serialized)
|
||||
|
||||
# 8. Final status
|
||||
if record.abort_event.is_set():
|
||||
action = record.abort_action
|
||||
if action == "rollback":
|
||||
await run_manager.set_status(run_id, RunStatus.error, error="Rolled back by user")
|
||||
await _set_terminal_status(run_manager, record, RunStatus.error, error="Rolled back by user")
|
||||
try:
|
||||
await _rollback_to_pre_run_checkpoint(
|
||||
checkpointer=checkpointer,
|
||||
@@ -362,7 +404,7 @@ async def run_agent(
|
||||
except Exception:
|
||||
logger.warning("Failed to rollback checkpoint for run %s", run_id, exc_info=True)
|
||||
else:
|
||||
await run_manager.set_status(run_id, RunStatus.interrupted)
|
||||
await _set_terminal_status(run_manager, record, RunStatus.interrupted)
|
||||
else:
|
||||
# LLMErrorHandlingMiddleware intentionally converts an exhausted
|
||||
# provider failure into a user-safe AIMessage instead of raising.
|
||||
@@ -372,9 +414,10 @@ async def run_agent(
|
||||
# generation had silently stopped halfway through. Inspect the
|
||||
# terminal checkpoint before declaring success so the persisted run
|
||||
# state and the user-visible SSE outcome stay consistent.
|
||||
terminal_llm_error = await _terminal_llm_error_reason(
|
||||
checkpointer=checkpointer,
|
||||
thread_id=thread_id,
|
||||
terminal_llm_error = await _await_completion_io(
|
||||
_terminal_llm_error_reason(checkpointer=checkpointer, thread_id=thread_id),
|
||||
run_id=run_id,
|
||||
phase="terminal_checkpoint",
|
||||
)
|
||||
if terminal_llm_error is not None:
|
||||
error_message = "本轮模型生成未能完成,已保留当前对话内容,请稍后重试。"
|
||||
@@ -383,8 +426,9 @@ async def run_agent(
|
||||
run_id,
|
||||
terminal_llm_error,
|
||||
)
|
||||
await run_manager.set_status(
|
||||
run_id,
|
||||
await _set_terminal_status(
|
||||
run_manager,
|
||||
record,
|
||||
RunStatus.error,
|
||||
error=error_message,
|
||||
)
|
||||
@@ -397,12 +441,12 @@ async def run_agent(
|
||||
},
|
||||
)
|
||||
else:
|
||||
await run_manager.set_status(run_id, RunStatus.success)
|
||||
await _set_terminal_status(run_manager, record, RunStatus.success)
|
||||
|
||||
except asyncio.CancelledError:
|
||||
action = record.abort_action
|
||||
if action == "rollback":
|
||||
await run_manager.set_status(run_id, RunStatus.error, error="Rolled back by user")
|
||||
await _set_terminal_status(run_manager, record, RunStatus.error, error="Rolled back by user")
|
||||
try:
|
||||
await _rollback_to_pre_run_checkpoint(
|
||||
checkpointer=checkpointer,
|
||||
@@ -416,13 +460,13 @@ async def run_agent(
|
||||
except Exception:
|
||||
logger.warning("Run %s cancellation rollback failed", run_id, exc_info=True)
|
||||
else:
|
||||
await run_manager.set_status(run_id, RunStatus.interrupted)
|
||||
await _set_terminal_status(run_manager, record, RunStatus.interrupted)
|
||||
logger.info("Run %s was cancelled", run_id)
|
||||
|
||||
except Exception as exc:
|
||||
error_msg = f"{exc}"
|
||||
logger.exception("Run %s failed: %s", run_id, error_msg)
|
||||
await run_manager.set_status(run_id, RunStatus.error, error=error_msg)
|
||||
await _set_terminal_status(run_manager, record, RunStatus.error, error=error_msg)
|
||||
await bridge.publish(
|
||||
run_id,
|
||||
"error",
|
||||
@@ -433,72 +477,170 @@ async def run_agent(
|
||||
)
|
||||
|
||||
finally:
|
||||
# Flush any buffered journal events and persist completion data
|
||||
if journal is not None:
|
||||
try:
|
||||
await journal.flush()
|
||||
except Exception:
|
||||
logger.warning("Failed to flush journal for run %s", run_id, exc_info=True)
|
||||
|
||||
try:
|
||||
# Persist token usage + convenience fields to RunStore
|
||||
completion = journal.get_completion_data()
|
||||
await run_manager.update_run_completion(run_id, status=record.status.value, **completion)
|
||||
except Exception:
|
||||
logger.warning("Failed to persist run completion for %s (non-fatal)", run_id, exc_info=True)
|
||||
|
||||
# Sync title from checkpoint to threads_meta.display_name
|
||||
if checkpointer is not None and thread_store is not None:
|
||||
try:
|
||||
ckpt_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
ckpt_tuple = await checkpointer.aget_tuple(ckpt_config)
|
||||
if ckpt_tuple is not None:
|
||||
ckpt = getattr(ckpt_tuple, "checkpoint", {}) or {}
|
||||
channel_values = ckpt.get("channel_values", {}) or {}
|
||||
title = channel_values.get("title")
|
||||
if title:
|
||||
await thread_store.update_display_name(thread_id, title)
|
||||
if (
|
||||
record.status == RunStatus.success
|
||||
and channel_values.get("title_provisional") is True
|
||||
):
|
||||
log_title_debug(
|
||||
"Run %s: scheduling background title generation for thread %s",
|
||||
run_id,
|
||||
thread_id,
|
||||
)
|
||||
title_task = asyncio.create_task(
|
||||
finalize_thread_title(
|
||||
checkpointer=checkpointer,
|
||||
thread_store=thread_store,
|
||||
thread_id=thread_id,
|
||||
source_state=copy.deepcopy(channel_values),
|
||||
app_config=ctx.app_config,
|
||||
),
|
||||
name=f"title-finalize:{thread_id}",
|
||||
)
|
||||
title_task.add_done_callback(_log_background_title_finalize_failure(thread_id))
|
||||
elif record.status == RunStatus.success:
|
||||
log_title_debug(
|
||||
"Run %s: background title generation not scheduled for thread %s because title_provisional=%r title=%r",
|
||||
run_id,
|
||||
thread_id,
|
||||
channel_values.get("title_provisional"),
|
||||
channel_values.get("title"),
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to sync title for thread %s (non-fatal)", thread_id)
|
||||
|
||||
# Update threads_meta status based on run outcome
|
||||
if thread_store is not None:
|
||||
try:
|
||||
final_status = "idle" if record.status == RunStatus.success else record.status.value
|
||||
await thread_store.update_status(thread_id, final_status)
|
||||
except Exception:
|
||||
logger.debug("Failed to update thread_meta status for %s (non-fatal)", thread_id)
|
||||
|
||||
await bridge.publish_end(run_id)
|
||||
asyncio.create_task(bridge.cleanup(run_id, delay=60))
|
||||
try:
|
||||
await _await_completion_io(
|
||||
_finalize_run(
|
||||
journal=journal,
|
||||
run_manager=run_manager,
|
||||
record=record,
|
||||
ctx=ctx,
|
||||
),
|
||||
run_id=run_id,
|
||||
phase="metadata_cleanup",
|
||||
)
|
||||
except TimeoutError:
|
||||
logger.warning("Run %s completion metadata exceeded its time budget; ending SSE", run_id)
|
||||
except asyncio.CancelledError:
|
||||
# A second cancellation can arrive while handling the original
|
||||
# cancellation or during normal cleanup. Never lose the terminator.
|
||||
logger.info("Run %s completion metadata was cancelled; ending SSE", run_id)
|
||||
except Exception:
|
||||
logger.warning("Failed to finalize run %s (non-fatal)", run_id, exc_info=True)
|
||||
finally:
|
||||
await bridge.publish_end(run_id)
|
||||
asyncio.create_task(bridge.cleanup(run_id, delay=60))
|
||||
|
||||
|
||||
async def _await_completion_io[T](
|
||||
operation: Awaitable[T],
|
||||
*,
|
||||
run_id: str,
|
||||
phase: str,
|
||||
) -> T:
|
||||
"""Bound completion-only I/O even if a backend delays cancellation.
|
||||
|
||||
Unlike wait_for, wait does not wait indefinitely for the cancelled
|
||||
operation to acknowledge cancellation. Do not use this for executing the
|
||||
graph: detaching a live tool would release run concurrency prematurely.
|
||||
"""
|
||||
task = asyncio.ensure_future(operation)
|
||||
_PENDING_COMPLETION_TASKS.add(task)
|
||||
|
||||
def consume_result(completed: asyncio.Future) -> None:
|
||||
_PENDING_COMPLETION_TASKS.discard(completed)
|
||||
if not completed.cancelled():
|
||||
completed.exception()
|
||||
|
||||
task.add_done_callback(consume_result)
|
||||
try:
|
||||
done, _ = await asyncio.wait({task}, timeout=_RUN_FINALIZATION_TIMEOUT_SECONDS)
|
||||
if not done:
|
||||
logger.warning("RUN_COMPLETION_TIMEOUT run_id=%s phase=%s", run_id, phase)
|
||||
raise TimeoutError(f"Run completion timed out during {phase}")
|
||||
return task.result()
|
||||
finally:
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
|
||||
|
||||
async def _set_terminal_status(run_manager: RunManager, record: RunRecord, status: RunStatus, *, error: str | None = None) -> None:
|
||||
try:
|
||||
await _await_completion_io(
|
||||
run_manager.set_status(record.run_id, status, error=error),
|
||||
run_id=record.run_id,
|
||||
phase="terminal_status",
|
||||
)
|
||||
except TimeoutError:
|
||||
# set_status updates the registry before awaiting its best-effort store.
|
||||
# Keep that local outcome authoritative if persistence never returns.
|
||||
record.status = status
|
||||
if error is not None:
|
||||
record.error = error
|
||||
logger.warning("Run %s terminal status persistence timed out", record.run_id)
|
||||
|
||||
|
||||
async def _finalize_run(*, journal: Any, run_manager: RunManager, record: RunRecord, ctx: RunContext) -> None:
|
||||
"""Best-effort metadata writes after graph execution, under one budget."""
|
||||
run_id = record.run_id
|
||||
thread_id = record.thread_id
|
||||
checkpointer = ctx.checkpointer
|
||||
thread_store = ctx.thread_store
|
||||
# Flush any buffered journal events and persist completion data
|
||||
if journal is not None:
|
||||
try:
|
||||
await journal.flush()
|
||||
_check_completion_cancellation()
|
||||
except Exception:
|
||||
logger.warning("Failed to flush journal for run %s", run_id, exc_info=True)
|
||||
|
||||
try:
|
||||
# Persist token usage + convenience fields to RunStore
|
||||
completion = journal.get_completion_data()
|
||||
await run_manager.update_run_completion(run_id, status=record.status.value, **completion)
|
||||
_check_completion_cancellation()
|
||||
except Exception:
|
||||
logger.warning("Failed to persist run completion for %s (non-fatal)", run_id, exc_info=True)
|
||||
|
||||
# Sync title from checkpoint to threads_meta.display_name
|
||||
if checkpointer is not None and thread_store is not None:
|
||||
try:
|
||||
ckpt_config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
ckpt_tuple = await checkpointer.aget_tuple(ckpt_config)
|
||||
if not await _completion_is_current(run_manager, record):
|
||||
return
|
||||
if ckpt_tuple is not None:
|
||||
ckpt = getattr(ckpt_tuple, "checkpoint", {}) or {}
|
||||
channel_values = ckpt.get("channel_values", {}) or {}
|
||||
title = channel_values.get("title")
|
||||
if title:
|
||||
await thread_store.update_display_name(thread_id, title)
|
||||
if not await _completion_is_current(run_manager, record):
|
||||
return
|
||||
if (
|
||||
record.status == RunStatus.success
|
||||
and channel_values.get("title_provisional") is True
|
||||
):
|
||||
log_title_debug(
|
||||
"Run %s: scheduling background title generation for thread %s",
|
||||
run_id,
|
||||
thread_id,
|
||||
)
|
||||
title_task = asyncio.create_task(
|
||||
finalize_thread_title(
|
||||
checkpointer=checkpointer,
|
||||
thread_store=thread_store,
|
||||
thread_id=thread_id,
|
||||
source_state=copy.deepcopy(channel_values),
|
||||
app_config=ctx.app_config,
|
||||
),
|
||||
name=f"title-finalize:{thread_id}",
|
||||
)
|
||||
title_task.add_done_callback(_log_background_title_finalize_failure(thread_id))
|
||||
elif record.status == RunStatus.success:
|
||||
log_title_debug(
|
||||
"Run %s: background title generation not scheduled for thread %s because title_provisional=%r title=%r",
|
||||
run_id,
|
||||
thread_id,
|
||||
channel_values.get("title_provisional"),
|
||||
channel_values.get("title"),
|
||||
)
|
||||
except Exception:
|
||||
logger.debug("Failed to sync title for thread %s (non-fatal)", thread_id)
|
||||
|
||||
# Update threads_meta status based on run outcome
|
||||
if thread_store is not None:
|
||||
try:
|
||||
if not await _completion_is_current(run_manager, record):
|
||||
return
|
||||
final_status = "idle" if record.status == RunStatus.success else record.status.value
|
||||
await thread_store.update_status(thread_id, final_status)
|
||||
except Exception:
|
||||
logger.debug("Failed to update thread_meta status for %s (non-fatal)", thread_id)
|
||||
|
||||
|
||||
def _check_completion_cancellation() -> None:
|
||||
task = asyncio.current_task()
|
||||
if task is not None and task.cancelling():
|
||||
# Some database adapters delay or suppress cancellation. They must not
|
||||
# resume subsequent title/status writes after the cleanup budget ends.
|
||||
raise asyncio.CancelledError
|
||||
|
||||
|
||||
async def _completion_is_current(run_manager: RunManager, record: RunRecord) -> bool:
|
||||
_check_completion_cancellation()
|
||||
records = await run_manager.list_by_thread(record.thread_id)
|
||||
_check_completion_cancellation()
|
||||
return not records or records[-1].run_id == record.run_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in new issue
Block a user