sync update from LAN repo (source dhk-1 @ 3b1ec5b4)

This commit is contained in:
gozeros committed 2026-11-05 10:56:34 +08:00
1 parent da6521b6cb
commit 264e2200d6
995 files changed
+148105 -13399

No files matched your search

@@ -18,12 +18,9 @@ import asyncio
import logging
import os
import uuid
from collections.abc import Callable
from datetime import UTC, datetime
from types import SimpleNamespace
from typing import Any
from langgraph.checkpoint.base import empty_checkpoint
from weakref import WeakKeyDictionary
from app.gateway.roundtable_model_fallback import (
fallback_model_chain,
@@ -39,6 +36,7 @@ from app.gateway.routers._roundtable_seed import (
SUMMARY_AGENT_ID,
ensure_roundtable_functional_agents,
)
from app.gateway.routers.multi_agent import _roundtable_file_stream_context
from app.gateway.services import RUN_SCOPE_BACKGROUND_CHILD
from deerflow.agents.roundtable_orchestrator.delivery import classify_seat_delivery, visible_seat_text
from deerflow.agents.roundtable_orchestrator.report_json import extract_report_json
@@ -51,8 +49,11 @@ from deerflow.agents.roundtable_orchestrator.types import (
SummaryResult,
)
from deerflow.config.paths import get_paths
from deerflow.config.runtime_contention_config import get_runtime_contention_config
from deerflow.persistence.roundtable_diagnostics import DiagnosticsRecorder, get_default_store
from deerflow.runtime.fair_queue import FairAsyncGate
from deerflow.runtime.serialization import serialize_channel_values
from deerflow.utils.file_io import file_io
from deerflow.utils.tool_visibility import is_internal_tool_name
logger = logging.getLogger(__name__)
@@ -86,49 +87,30 @@ _SUMMARY_FILE = "方案总结报告.md"
# 这里漏禁了 web_search,导致后台席位 / 报告能联网搜索而前端不能,现由 policy 统一。
# ── 后台作业「按用户限流」(2026-06-16)─────────────────────────────────────────
# 痛点:进 2+ 个后台挂起后,整个网关变慢——建立智能体连接、等待模型响应都慢。
# 根因:① checkpointer 用单连接 aiosqlite(AsyncSqliteSaver,内部一把 asyncio.Lock 串
# 行全进程的 checkpoint 读写);② 后台作业无并发上限——每个作业进程内驱动真实 leader/
# seat/report run,DAG/recommend 的 stage 还用 asyncio.gather 把席位真并行铺开。多个后台
# 作业同时把几十个 agent run 压到同一把 SQLite 锁 + 同一个离线模型端点上,前台实时会商
# 的 checkpoint 读写 / 模型调用只能排队 → 「干啥都慢」。
#
# 解法:给**每个用户**一个并发额度(默认 2),后台每个 agent 轮次(leader/seat/report/
# summary/dashboard)真正开跑前先取一个额度,跑完释放。同一用户的所有后台作业共享这 2 个
# 额度——随便挂多少个作业都能立刻建好,但同一瞬间真正在打模型/写 checkpoint 的后台轮次
# 至多 2 个,不会把共享资源打满。**前台实时路径(multi_agent._leader_run/_special_run、
# 普通聊天)不经过本文件的 `_run_agent_turn`,永不取这个额度,故前台不受限。**
_BACKGROUND_SLOTS_PER_USER = max(1, int(os.getenv("ROUNDTABLE_BACKGROUND_SLOTS_PER_USER", "2") or "2"))
# 按 user_id 缓存 Semaphore。网关全程跑在同一个事件循环上,普通 dict 取/建即可(无跨线程)。
_user_background_sems: dict[str, asyncio.Semaphore] = {}
# Global and per-user budgets share round-robin admission. A busy user's
# backlog cannot fill the model/checkpoint path before another user is served.
def _user_background_semaphore(user_id: str) -> asyncio.Semaphore:
"""取该用户的后台并发额度信号量(懒建,每用户一把)。"""
sem = _user_background_sems.get(user_id)
if sem is None:
sem = asyncio.Semaphore(_BACKGROUND_SLOTS_PER_USER)
_user_background_sems[user_id] = sem
return sem
_background_gates: WeakKeyDictionary = WeakKeyDictionary()
def _background_gate() -> FairAsyncGate:
loop = asyncio.get_running_loop()
if loop not in _background_gates:
limits = get_runtime_contention_config()
per_user = max(1, int(os.getenv("ROUNDTABLE_BACKGROUND_SLOTS_PER_USER", str(limits.background_per_user))))
_background_gates[loop] = FairAsyncGate(limits.background_concurrency, per_user, limits.queue_limit)
return _background_gates[loop]
# ── 进程内底层操作 ───────────────────────────────────────────────────────────
async def _create_thread(app) -> str:
"""同进程建一个带圆桌 metadata 的 thread + 空 checkpoint(镜像 _create_thread_direct)。"""
checkpointer = app.state.checkpointer
"""Create metadata only; the first graph run initializes its checkpoint."""
thread_store = app.state.thread_store
thread_id = str(uuid.uuid4())
await thread_store.create(thread_id, assistant_id=None, metadata=dict(_ROUNDTABLE_THREAD_METADATA))
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
await checkpointer.aput(
config,
empty_checkpoint(),
{"step": -1, "source": "input", "writes": None, "parents": {}, "created_at": datetime.now(UTC).isoformat()},
{},
)
return thread_id
@@ -345,6 +327,8 @@ async def _run_agent_turn(
skill_stop_names: list[str],
thinking_enabled: bool,
reasoning_effort: str,
allowed_tools: list[str] | None = None,
recursion_limit: int = 1000,
subagent_enabled: bool = False,
diag: DiagnosticsRecorder | None = None,
stage: str = "step2_seat",
@@ -356,9 +340,8 @@ async def _run_agent_turn(
镜像 multi_agent ``_run_payload`` 的 body 形状:agent 通过 ``context.agent_name`` 选,
assistant_id 固定 lead_agent;关闭 memory recall / subagent / plan mode(临时研讨任务)。
并发受**该用户的后台额度**约束(``_user_background_semaphore``):真正开跑(start_run +
await run task)前先取额度,跑完释放。同一用户的所有后台作业共享这几个额度,避免 2+
后台挂起把单连接 SQLite checkpointer / 离线模型打满、拖垮前台实时会商。
全局与每用户后台额度由 ``_background_gate`` 统一按用户轮转分配。排队不占运行名额,
真正开跑前取额度;取消时等待实际 run 停止后才释放,避免拖垮前台实时会商。
"""
from app.gateway.routers.thread_runs import RunCreateRequest
from app.gateway.services import start_run
@@ -367,7 +350,7 @@ async def _run_agent_turn(
assistant_id="lead_agent",
input={"messages": [{"type": "human", "content": [{"type": "text", "text": message}], "additional_kwargs": {}}]},
metadata={"run_scope": RUN_SCOPE_BACKGROUND_CHILD},
config={"recursion_limit": 1000},
config={"recursion_limit": recursion_limit},
context={
"agent_name": agent_name,
"model_name": model or "",
@@ -382,6 +365,8 @@ async def _run_agent_turn(
# 与 multi_agent._run_payload 对齐:关 builtin 记忆注入,避免单例总控/席位
# 被上轮会话 memory.json 带偏(臆造席位 / 直接干活)。详见该处注释。
"memory_injection_enabled": False,
"allowed_tools": allowed_tools,
**_roundtable_file_stream_context(),
},
excluded_tools=excluded_tools,
skill_stop_names=skill_stop_names,
@@ -393,7 +378,7 @@ async def _run_agent_turn(
request = SimpleNamespace(app=app, state=SimpleNamespace(user=None, auth=None))
# 取该用户的后台额度后再开跑:start_run + await task 是最重的部分(建 agent、调模型、
# 落 checkpoint)。额度满时本轮在此 await 排队,等同用户某个在跑轮次释放额度后再开。
async with _user_background_semaphore(user_id):
async with _background_gate().slot(user_id):
try:
record = await start_run( # type: ignore[arg-type]
body,
@@ -410,16 +395,23 @@ async def _run_agent_turn(
agent_name=agent_name, cycle=cycle,
)
raise
if on_started is not None:
await on_started(record)
# idle 超时守护:慢但持续产出的生成一直等;只有连续 _SEAT_IDLE_TIMEOUT_S 秒**毫无输出**
# (真卡死)才取消并抛错。绝不因「这轮总耗时长」而打断(除非显式配了 _SEAT_MAX_TOTAL_S)。
await _await_run_with_idle_guard(
app, record,
idle_timeout=_SEAT_IDLE_TIMEOUT_S,
max_total=_SEAT_MAX_TOTAL_S,
diag=diag, stage=stage, agent_name=agent_name, cycle=cycle,
)
try:
if on_started is not None:
await on_started(record)
# 持续产出的慢生成继续等,只有连续无输出或显式总时限才中断。
await _await_run_with_idle_guard(
app, record,
idle_timeout=_SEAT_IDLE_TIMEOUT_S,
max_total=_SEAT_MAX_TOTAL_S,
diag=diag, stage=stage, agent_name=agent_name, cycle=cycle,
)
finally:
# Cancelling the parent or its idle watcher is not proof that the
# child has stopped writing checkpoints. Keep admission until ack.
task = getattr(record, "task", None)
if task is not None and not task.done():
task.cancel()
await asyncio.gather(task, return_exceptions=True)
return await _read_thread_messages(app, thread_id)
@@ -568,36 +560,6 @@ def _exception_reason(exc: BaseException) -> str:
return "exception"
def _seat_empty_reason(messages: list[dict[str, Any]]) -> str | None:
"""席位/取数轮校验:本轮无可见正文 → ``empty`` / ``thinking_only``(触发换模型)。"""
saw_thinking = False
for m in reversed(messages):
if not isinstance(m, dict):
continue
role = m.get("type") or m.get("role")
if role in {"human", "user"}:
if _is_broadcast_human_text(_message_text(m)):
continue
break
if role not in {"ai", "assistant"}:
continue
kind = classify_seat_delivery(_message_text(m))
if kind == "valid":
return None
if kind == "thinking_only":
saw_thinking = True
break
return "thinking_only" if saw_thinking else "empty"
def _leader_failed_reason(messages: list[dict[str, Any]]) -> str | None:
"""总控轮校验:既无正文、也无任何工具调用(派活/澄清)→ ``empty``(视为模型没干活)。"""
ai = _latest_ai_message(messages)
has_text = bool(_latest_ai_text(messages).strip())
has_tools = bool(_normalize_tool_calls((ai or {}).get("tool_calls")))
return None if (has_text or has_tools) else "empty"
# ── 网关实现 ─────────────────────────────────────────────────────────────────
@@ -670,20 +632,24 @@ class InProcessRoundtableGateway:
message: str,
requested_model: str | None,
excluded_tools: list[str],
allowed_tools: list[str] | None,
skill_stop_names: list[str],
thinking_enabled: bool,
reasoning_effort: str,
recursion_limit: int,
subagent_enabled: bool = False,
require_visible_text: bool = False,
stage: str,
cycle: int | None = None,
validate: Callable[[list[dict[str, Any]]], str | None] | None = None,
) -> tuple[list[dict[str, Any]], list[ModelSwitch], str | None]:
"""跑一轮 agent,并在「模型出问题」时自动切到下一个模型重试。
触发换模型的三种情形(与用户选择一致):
触发换模型的两种情形:
① ``_run_agent_turn`` 抛异常(start_run 失败 / idle 卡死超时);
② 返回了中间件兜底错误文案(``is_llm_error_message`` 命中);
③ 调用方 ``validate`` 判定无效(如席位交付为空 / 总控既无正文又无派活)。
仅有思考、暂无可见正文或空交付不是模型故障,不能触发自动换模;只有调用方明确
``require_visible_text=True`` 的最终收口轮,在请求完整结束后无正文才视为不可交付。
候选模型链按 ``config.yaml models[]`` 顺序(``fallback_model_chain``),首个为
``requested_model``(空则作业级 ``self._model``,再空则配置默认)。返回
@@ -721,9 +687,11 @@ class InProcessRoundtableGateway:
message=message,
model=candidate,
excluded_tools=excluded_tools,
allowed_tools=allowed_tools,
skill_stop_names=skill_stop_names,
thinking_enabled=thinking_enabled,
reasoning_effort=reasoning_effort,
recursion_limit=recursion_limit,
subagent_enabled=subagent_enabled,
diag=self._diag,
stage=stage,
@@ -739,10 +707,10 @@ class InProcessRoundtableGateway:
detail={"reason": prev_reason, "tried": chain}, agent_name=agent_name, cycle=cycle,
)
return messages, switches, used_model
# 跑通了:再看是否「兜底错误文案」或 validate 判定无效。
# 普通轮只识别带明确错误标记的兜底文案;最终收口轮完整结束后必须有正文。
reason = is_llm_error_message(_latest_ai_message(messages))
if reason is None and validate is not None:
reason = validate(messages)
if reason is None and require_visible_text and not _latest_ai_text(messages).strip():
reason = "empty"
if reason is None:
return messages, switches, used_model
prev_model, prev_reason = candidate, reason
@@ -755,27 +723,46 @@ class InProcessRoundtableGateway:
return messages, switches, used_model
return messages, switches, used_model
async def run_leader(self, *, thread_ids, coordinator_name, message, model) -> LeaderTurn:
async def run_leader(
self,
*,
thread_ids,
coordinator_name,
message,
model,
finalize_only: bool = False,
) -> LeaderTurn:
msg = message
if self._first_leader and self._seat_intro:
msg = self._seat_intro + "\n\n" + message
self._first_leader = False
policy = roundtable_run_policy("leader")
if finalize_only:
msg += (
"\n\n【最终收口·纯正文】本轮所有工具均已关闭。请直接输出一份合并后的最终结论正文;"
"不要按席位逐份复述原文,也不要反复改写同一份总结。"
"不要调用或尝试 write_file、present_files、agent_orchestration、skill_view 或任何其它工具,"
"不要再派活,也不要只描述你准备做什么。"
)
messages, switches, _used = await self._run_turn_with_fallback(
thread_id=thread_ids[coordinator_name],
agent_name=coordinator_name,
message=msg,
requested_model=model,
excluded_tools=policy.excluded_tools,
skill_stop_names=policy.skill_stop_names,
allowed_tools=[] if finalize_only else getattr(policy, "allowed_tools", None),
skill_stop_names=[] if finalize_only else policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
require_visible_text=finalize_only,
stage="step2_leader",
validate=_leader_failed_reason,
)
ai = _latest_ai_message(messages)
# tool_calls 取最后一条 AI 消息(派活就在这条);正文取最后一条有文字的 AI 消息。
final_text = _latest_ai_text(messages)
if finalize_only and not final_text.strip():
raise RuntimeError("总控最终收口未产出正文")
tool_calls = (ai.get("tool_calls") if ai else None) or []
# 本轮工具调用(派活 / 澄清),带进 LeaderTurn 供草稿 run 重建步骤卡。
turn_tool_calls = _normalize_tool_calls(tool_calls)
@@ -830,6 +817,7 @@ class InProcessRoundtableGateway:
message=seat_task,
requested_model=model,
excluded_tools=policy.excluded_tools,
allowed_tools=getattr(policy, "allowed_tools", None),
skill_stop_names=policy.skill_stop_names,
thinking_enabled=(
policy.thinking_enabled if seat_thinking is None else seat_thinking
@@ -837,9 +825,9 @@ class InProcessRoundtableGateway:
reasoning_effort=(
policy.reasoning_effort if seat_reasoning is None else seat_reasoning
),
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
subagent_enabled=seat_subagent,
stage="step2_seat",
validate=_seat_empty_reason,
)
return SeatTurn(
final_text=_latest_ai_text(messages),
@@ -867,15 +855,17 @@ class InProcessRoundtableGateway:
message=message,
requested_model=current_model,
excluded_tools=policy.excluded_tools,
allowed_tools=getattr(policy, "allowed_tools", None),
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
stage="step3_report",
)
switches.extend(sw)
current_model = used or current_model
summary = _latest_ai_text(messages) or summary
html = _read_report_html(self._app, report_thread_id, self._user_id)
html = await file_io(_read_report_html, self._app, report_thread_id, self._user_id)
if _html_complete(html):
break
if attempt >= _ARTIFACT_RETRY_LIMIT:
@@ -925,15 +915,17 @@ class InProcessRoundtableGateway:
message=message,
requested_model=current_model,
excluded_tools=policy.excluded_tools,
allowed_tools=getattr(policy, "allowed_tools", None),
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
stage="step3_summary",
)
switches.extend(sw)
current_model = used or current_model
closing = _latest_ai_text(messages) or closing
md = _read_summary_md(self._app, summary_thread_id, self._user_id)
md = await file_io(_read_summary_md, self._app, summary_thread_id, self._user_id)
if md and md.strip():
break
if attempt >= _ARTIFACT_RETRY_LIMIT:
@@ -981,9 +973,11 @@ class InProcessRoundtableGateway:
message=message,
requested_model=current_model,
excluded_tools=policy.excluded_tools,
allowed_tools=getattr(policy, "allowed_tools", None),
skill_stop_names=policy.skill_stop_names,
thinking_enabled=policy.thinking_enabled,
reasoning_effort=policy.reasoning_effort,
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
stage="step3_dashboard",
)
current_model = used or current_model