sync update from LAN repo (source dhk-1 @ 3b1ec5b4)
This commit is contained in:
1 parent
da6521b6cb
commit
264e2200d6
995 files changed
+148105
-13399
No files matched your search
@@ -18,12 +18,9 @@ import asyncio
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
from langgraph.checkpoint.base import empty_checkpoint
|
||||
from weakref import WeakKeyDictionary
|
||||
|
||||
from app.gateway.roundtable_model_fallback import (
|
||||
fallback_model_chain,
|
||||
@@ -39,6 +36,7 @@ from app.gateway.routers._roundtable_seed import (
|
||||
SUMMARY_AGENT_ID,
|
||||
ensure_roundtable_functional_agents,
|
||||
)
|
||||
from app.gateway.routers.multi_agent import _roundtable_file_stream_context
|
||||
from app.gateway.services import RUN_SCOPE_BACKGROUND_CHILD
|
||||
from deerflow.agents.roundtable_orchestrator.delivery import classify_seat_delivery, visible_seat_text
|
||||
from deerflow.agents.roundtable_orchestrator.report_json import extract_report_json
|
||||
@@ -51,8 +49,11 @@ from deerflow.agents.roundtable_orchestrator.types import (
|
||||
SummaryResult,
|
||||
)
|
||||
from deerflow.config.paths import get_paths
|
||||
from deerflow.config.runtime_contention_config import get_runtime_contention_config
|
||||
from deerflow.persistence.roundtable_diagnostics import DiagnosticsRecorder, get_default_store
|
||||
from deerflow.runtime.fair_queue import FairAsyncGate
|
||||
from deerflow.runtime.serialization import serialize_channel_values
|
||||
from deerflow.utils.file_io import file_io
|
||||
from deerflow.utils.tool_visibility import is_internal_tool_name
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -86,49 +87,30 @@ _SUMMARY_FILE = "方案总结报告.md"
|
||||
# 这里漏禁了 web_search,导致后台席位 / 报告能联网搜索而前端不能,现由 policy 统一。
|
||||
|
||||
|
||||
# ── 后台作业「按用户限流」(2026-06-16)─────────────────────────────────────────
|
||||
# 痛点:进 2+ 个后台挂起后,整个网关变慢——建立智能体连接、等待模型响应都慢。
|
||||
# 根因:① checkpointer 用单连接 aiosqlite(AsyncSqliteSaver,内部一把 asyncio.Lock 串
|
||||
# 行全进程的 checkpoint 读写);② 后台作业无并发上限——每个作业进程内驱动真实 leader/
|
||||
# seat/report run,DAG/recommend 的 stage 还用 asyncio.gather 把席位真并行铺开。多个后台
|
||||
# 作业同时把几十个 agent run 压到同一把 SQLite 锁 + 同一个离线模型端点上,前台实时会商
|
||||
# 的 checkpoint 读写 / 模型调用只能排队 → 「干啥都慢」。
|
||||
#
|
||||
# 解法:给**每个用户**一个并发额度(默认 2),后台每个 agent 轮次(leader/seat/report/
|
||||
# summary/dashboard)真正开跑前先取一个额度,跑完释放。同一用户的所有后台作业共享这 2 个
|
||||
# 额度——随便挂多少个作业都能立刻建好,但同一瞬间真正在打模型/写 checkpoint 的后台轮次
|
||||
# 至多 2 个,不会把共享资源打满。**前台实时路径(multi_agent._leader_run/_special_run、
|
||||
# 普通聊天)不经过本文件的 `_run_agent_turn`,永不取这个额度,故前台不受限。**
|
||||
_BACKGROUND_SLOTS_PER_USER = max(1, int(os.getenv("ROUNDTABLE_BACKGROUND_SLOTS_PER_USER", "2") or "2"))
|
||||
# 按 user_id 缓存 Semaphore。网关全程跑在同一个事件循环上,普通 dict 取/建即可(无跨线程)。
|
||||
_user_background_sems: dict[str, asyncio.Semaphore] = {}
|
||||
# Global and per-user budgets share round-robin admission. A busy user's
|
||||
# backlog cannot fill the model/checkpoint path before another user is served.
|
||||
|
||||
|
||||
def _user_background_semaphore(user_id: str) -> asyncio.Semaphore:
|
||||
"""取该用户的后台并发额度信号量(懒建,每用户一把)。"""
|
||||
sem = _user_background_sems.get(user_id)
|
||||
if sem is None:
|
||||
sem = asyncio.Semaphore(_BACKGROUND_SLOTS_PER_USER)
|
||||
_user_background_sems[user_id] = sem
|
||||
return sem
|
||||
_background_gates: WeakKeyDictionary = WeakKeyDictionary()
|
||||
|
||||
|
||||
def _background_gate() -> FairAsyncGate:
|
||||
loop = asyncio.get_running_loop()
|
||||
if loop not in _background_gates:
|
||||
limits = get_runtime_contention_config()
|
||||
per_user = max(1, int(os.getenv("ROUNDTABLE_BACKGROUND_SLOTS_PER_USER", str(limits.background_per_user))))
|
||||
_background_gates[loop] = FairAsyncGate(limits.background_concurrency, per_user, limits.queue_limit)
|
||||
return _background_gates[loop]
|
||||
|
||||
|
||||
# ── 进程内底层操作 ───────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _create_thread(app) -> str:
|
||||
"""同进程建一个带圆桌 metadata 的 thread + 空 checkpoint(镜像 _create_thread_direct)。"""
|
||||
checkpointer = app.state.checkpointer
|
||||
"""Create metadata only; the first graph run initializes its checkpoint."""
|
||||
thread_store = app.state.thread_store
|
||||
thread_id = str(uuid.uuid4())
|
||||
await thread_store.create(thread_id, assistant_id=None, metadata=dict(_ROUNDTABLE_THREAD_METADATA))
|
||||
config = {"configurable": {"thread_id": thread_id, "checkpoint_ns": ""}}
|
||||
await checkpointer.aput(
|
||||
config,
|
||||
empty_checkpoint(),
|
||||
{"step": -1, "source": "input", "writes": None, "parents": {}, "created_at": datetime.now(UTC).isoformat()},
|
||||
{},
|
||||
)
|
||||
return thread_id
|
||||
|
||||
|
||||
@@ -345,6 +327,8 @@ async def _run_agent_turn(
|
||||
skill_stop_names: list[str],
|
||||
thinking_enabled: bool,
|
||||
reasoning_effort: str,
|
||||
allowed_tools: list[str] | None = None,
|
||||
recursion_limit: int = 1000,
|
||||
subagent_enabled: bool = False,
|
||||
diag: DiagnosticsRecorder | None = None,
|
||||
stage: str = "step2_seat",
|
||||
@@ -356,9 +340,8 @@ async def _run_agent_turn(
|
||||
镜像 multi_agent ``_run_payload`` 的 body 形状:agent 通过 ``context.agent_name`` 选,
|
||||
assistant_id 固定 lead_agent;关闭 memory recall / subagent / plan mode(临时研讨任务)。
|
||||
|
||||
并发受**该用户的后台额度**约束(``_user_background_semaphore``):真正开跑(start_run +
|
||||
await run task)前先取额度,跑完释放。同一用户的所有后台作业共享这几个额度,避免 2+
|
||||
后台挂起把单连接 SQLite checkpointer / 离线模型打满、拖垮前台实时会商。
|
||||
全局与每用户后台额度由 ``_background_gate`` 统一按用户轮转分配。排队不占运行名额,
|
||||
真正开跑前取额度;取消时等待实际 run 停止后才释放,避免拖垮前台实时会商。
|
||||
"""
|
||||
from app.gateway.routers.thread_runs import RunCreateRequest
|
||||
from app.gateway.services import start_run
|
||||
@@ -367,7 +350,7 @@ async def _run_agent_turn(
|
||||
assistant_id="lead_agent",
|
||||
input={"messages": [{"type": "human", "content": [{"type": "text", "text": message}], "additional_kwargs": {}}]},
|
||||
metadata={"run_scope": RUN_SCOPE_BACKGROUND_CHILD},
|
||||
config={"recursion_limit": 1000},
|
||||
config={"recursion_limit": recursion_limit},
|
||||
context={
|
||||
"agent_name": agent_name,
|
||||
"model_name": model or "",
|
||||
@@ -382,6 +365,8 @@ async def _run_agent_turn(
|
||||
# 与 multi_agent._run_payload 对齐:关 builtin 记忆注入,避免单例总控/席位
|
||||
# 被上轮会话 memory.json 带偏(臆造席位 / 直接干活)。详见该处注释。
|
||||
"memory_injection_enabled": False,
|
||||
"allowed_tools": allowed_tools,
|
||||
**_roundtable_file_stream_context(),
|
||||
},
|
||||
excluded_tools=excluded_tools,
|
||||
skill_stop_names=skill_stop_names,
|
||||
@@ -393,7 +378,7 @@ async def _run_agent_turn(
|
||||
request = SimpleNamespace(app=app, state=SimpleNamespace(user=None, auth=None))
|
||||
# 取该用户的后台额度后再开跑:start_run + await task 是最重的部分(建 agent、调模型、
|
||||
# 落 checkpoint)。额度满时本轮在此 await 排队,等同用户某个在跑轮次释放额度后再开。
|
||||
async with _user_background_semaphore(user_id):
|
||||
async with _background_gate().slot(user_id):
|
||||
try:
|
||||
record = await start_run( # type: ignore[arg-type]
|
||||
body,
|
||||
@@ -410,16 +395,23 @@ async def _run_agent_turn(
|
||||
agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
raise
|
||||
if on_started is not None:
|
||||
await on_started(record)
|
||||
# idle 超时守护:慢但持续产出的生成一直等;只有连续 _SEAT_IDLE_TIMEOUT_S 秒**毫无输出**
|
||||
# (真卡死)才取消并抛错。绝不因「这轮总耗时长」而打断(除非显式配了 _SEAT_MAX_TOTAL_S)。
|
||||
await _await_run_with_idle_guard(
|
||||
app, record,
|
||||
idle_timeout=_SEAT_IDLE_TIMEOUT_S,
|
||||
max_total=_SEAT_MAX_TOTAL_S,
|
||||
diag=diag, stage=stage, agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
try:
|
||||
if on_started is not None:
|
||||
await on_started(record)
|
||||
# 持续产出的慢生成继续等,只有连续无输出或显式总时限才中断。
|
||||
await _await_run_with_idle_guard(
|
||||
app, record,
|
||||
idle_timeout=_SEAT_IDLE_TIMEOUT_S,
|
||||
max_total=_SEAT_MAX_TOTAL_S,
|
||||
diag=diag, stage=stage, agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
finally:
|
||||
# Cancelling the parent or its idle watcher is not proof that the
|
||||
# child has stopped writing checkpoints. Keep admission until ack.
|
||||
task = getattr(record, "task", None)
|
||||
if task is not None and not task.done():
|
||||
task.cancel()
|
||||
await asyncio.gather(task, return_exceptions=True)
|
||||
return await _read_thread_messages(app, thread_id)
|
||||
|
||||
|
||||
@@ -568,36 +560,6 @@ def _exception_reason(exc: BaseException) -> str:
|
||||
return "exception"
|
||||
|
||||
|
||||
def _seat_empty_reason(messages: list[dict[str, Any]]) -> str | None:
|
||||
"""席位/取数轮校验:本轮无可见正文 → ``empty`` / ``thinking_only``(触发换模型)。"""
|
||||
saw_thinking = False
|
||||
for m in reversed(messages):
|
||||
if not isinstance(m, dict):
|
||||
continue
|
||||
role = m.get("type") or m.get("role")
|
||||
if role in {"human", "user"}:
|
||||
if _is_broadcast_human_text(_message_text(m)):
|
||||
continue
|
||||
break
|
||||
if role not in {"ai", "assistant"}:
|
||||
continue
|
||||
kind = classify_seat_delivery(_message_text(m))
|
||||
if kind == "valid":
|
||||
return None
|
||||
if kind == "thinking_only":
|
||||
saw_thinking = True
|
||||
break
|
||||
return "thinking_only" if saw_thinking else "empty"
|
||||
|
||||
|
||||
def _leader_failed_reason(messages: list[dict[str, Any]]) -> str | None:
|
||||
"""总控轮校验:既无正文、也无任何工具调用(派活/澄清)→ ``empty``(视为模型没干活)。"""
|
||||
ai = _latest_ai_message(messages)
|
||||
has_text = bool(_latest_ai_text(messages).strip())
|
||||
has_tools = bool(_normalize_tool_calls((ai or {}).get("tool_calls")))
|
||||
return None if (has_text or has_tools) else "empty"
|
||||
|
||||
|
||||
# ── 网关实现 ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -670,20 +632,24 @@ class InProcessRoundtableGateway:
|
||||
message: str,
|
||||
requested_model: str | None,
|
||||
excluded_tools: list[str],
|
||||
allowed_tools: list[str] | None,
|
||||
skill_stop_names: list[str],
|
||||
thinking_enabled: bool,
|
||||
reasoning_effort: str,
|
||||
recursion_limit: int,
|
||||
subagent_enabled: bool = False,
|
||||
require_visible_text: bool = False,
|
||||
stage: str,
|
||||
cycle: int | None = None,
|
||||
validate: Callable[[list[dict[str, Any]]], str | None] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], list[ModelSwitch], str | None]:
|
||||
"""跑一轮 agent,并在「模型出问题」时自动切到下一个模型重试。
|
||||
|
||||
触发换模型的三种情形(与用户选择一致):
|
||||
触发换模型的两种情形:
|
||||
① ``_run_agent_turn`` 抛异常(start_run 失败 / idle 卡死超时);
|
||||
② 返回了中间件兜底错误文案(``is_llm_error_message`` 命中);
|
||||
③ 调用方 ``validate`` 判定无效(如席位交付为空 / 总控既无正文又无派活)。
|
||||
|
||||
仅有思考、暂无可见正文或空交付不是模型故障,不能触发自动换模;只有调用方明确
|
||||
``require_visible_text=True`` 的最终收口轮,在请求完整结束后无正文才视为不可交付。
|
||||
|
||||
候选模型链按 ``config.yaml models[]`` 顺序(``fallback_model_chain``),首个为
|
||||
``requested_model``(空则作业级 ``self._model``,再空则配置默认)。返回
|
||||
@@ -721,9 +687,11 @@ class InProcessRoundtableGateway:
|
||||
message=message,
|
||||
model=candidate,
|
||||
excluded_tools=excluded_tools,
|
||||
allowed_tools=allowed_tools,
|
||||
skill_stop_names=skill_stop_names,
|
||||
thinking_enabled=thinking_enabled,
|
||||
reasoning_effort=reasoning_effort,
|
||||
recursion_limit=recursion_limit,
|
||||
subagent_enabled=subagent_enabled,
|
||||
diag=self._diag,
|
||||
stage=stage,
|
||||
@@ -739,10 +707,10 @@ class InProcessRoundtableGateway:
|
||||
detail={"reason": prev_reason, "tried": chain}, agent_name=agent_name, cycle=cycle,
|
||||
)
|
||||
return messages, switches, used_model
|
||||
# 跑通了:再看是否「兜底错误文案」或 validate 判定无效。
|
||||
# 普通轮只识别带明确错误标记的兜底文案;最终收口轮完整结束后必须有正文。
|
||||
reason = is_llm_error_message(_latest_ai_message(messages))
|
||||
if reason is None and validate is not None:
|
||||
reason = validate(messages)
|
||||
if reason is None and require_visible_text and not _latest_ai_text(messages).strip():
|
||||
reason = "empty"
|
||||
if reason is None:
|
||||
return messages, switches, used_model
|
||||
prev_model, prev_reason = candidate, reason
|
||||
@@ -755,27 +723,46 @@ class InProcessRoundtableGateway:
|
||||
return messages, switches, used_model
|
||||
return messages, switches, used_model
|
||||
|
||||
async def run_leader(self, *, thread_ids, coordinator_name, message, model) -> LeaderTurn:
|
||||
async def run_leader(
|
||||
self,
|
||||
*,
|
||||
thread_ids,
|
||||
coordinator_name,
|
||||
message,
|
||||
model,
|
||||
finalize_only: bool = False,
|
||||
) -> LeaderTurn:
|
||||
msg = message
|
||||
if self._first_leader and self._seat_intro:
|
||||
msg = self._seat_intro + "\n\n" + message
|
||||
self._first_leader = False
|
||||
policy = roundtable_run_policy("leader")
|
||||
if finalize_only:
|
||||
msg += (
|
||||
"\n\n【最终收口·纯正文】本轮所有工具均已关闭。请直接输出一份合并后的最终结论正文;"
|
||||
"不要按席位逐份复述原文,也不要反复改写同一份总结。"
|
||||
"不要调用或尝试 write_file、present_files、agent_orchestration、skill_view 或任何其它工具,"
|
||||
"不要再派活,也不要只描述你准备做什么。"
|
||||
)
|
||||
messages, switches, _used = await self._run_turn_with_fallback(
|
||||
thread_id=thread_ids[coordinator_name],
|
||||
agent_name=coordinator_name,
|
||||
message=msg,
|
||||
requested_model=model,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
allowed_tools=[] if finalize_only else getattr(policy, "allowed_tools", None),
|
||||
skill_stop_names=[] if finalize_only else policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
|
||||
require_visible_text=finalize_only,
|
||||
stage="step2_leader",
|
||||
validate=_leader_failed_reason,
|
||||
)
|
||||
ai = _latest_ai_message(messages)
|
||||
# tool_calls 取最后一条 AI 消息(派活就在这条);正文取最后一条有文字的 AI 消息。
|
||||
final_text = _latest_ai_text(messages)
|
||||
if finalize_only and not final_text.strip():
|
||||
raise RuntimeError("总控最终收口未产出正文")
|
||||
tool_calls = (ai.get("tool_calls") if ai else None) or []
|
||||
# 本轮工具调用(派活 / 澄清),带进 LeaderTurn 供草稿 run 重建步骤卡。
|
||||
turn_tool_calls = _normalize_tool_calls(tool_calls)
|
||||
@@ -830,6 +817,7 @@ class InProcessRoundtableGateway:
|
||||
message=seat_task,
|
||||
requested_model=model,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
allowed_tools=getattr(policy, "allowed_tools", None),
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=(
|
||||
policy.thinking_enabled if seat_thinking is None else seat_thinking
|
||||
@@ -837,9 +825,9 @@ class InProcessRoundtableGateway:
|
||||
reasoning_effort=(
|
||||
policy.reasoning_effort if seat_reasoning is None else seat_reasoning
|
||||
),
|
||||
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
|
||||
subagent_enabled=seat_subagent,
|
||||
stage="step2_seat",
|
||||
validate=_seat_empty_reason,
|
||||
)
|
||||
return SeatTurn(
|
||||
final_text=_latest_ai_text(messages),
|
||||
@@ -867,15 +855,17 @@ class InProcessRoundtableGateway:
|
||||
message=message,
|
||||
requested_model=current_model,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
allowed_tools=getattr(policy, "allowed_tools", None),
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
|
||||
stage="step3_report",
|
||||
)
|
||||
switches.extend(sw)
|
||||
current_model = used or current_model
|
||||
summary = _latest_ai_text(messages) or summary
|
||||
html = _read_report_html(self._app, report_thread_id, self._user_id)
|
||||
html = await file_io(_read_report_html, self._app, report_thread_id, self._user_id)
|
||||
if _html_complete(html):
|
||||
break
|
||||
if attempt >= _ARTIFACT_RETRY_LIMIT:
|
||||
@@ -925,15 +915,17 @@ class InProcessRoundtableGateway:
|
||||
message=message,
|
||||
requested_model=current_model,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
allowed_tools=getattr(policy, "allowed_tools", None),
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
|
||||
stage="step3_summary",
|
||||
)
|
||||
switches.extend(sw)
|
||||
current_model = used or current_model
|
||||
closing = _latest_ai_text(messages) or closing
|
||||
md = _read_summary_md(self._app, summary_thread_id, self._user_id)
|
||||
md = await file_io(_read_summary_md, self._app, summary_thread_id, self._user_id)
|
||||
if md and md.strip():
|
||||
break
|
||||
if attempt >= _ARTIFACT_RETRY_LIMIT:
|
||||
@@ -981,9 +973,11 @@ class InProcessRoundtableGateway:
|
||||
message=message,
|
||||
requested_model=current_model,
|
||||
excluded_tools=policy.excluded_tools,
|
||||
allowed_tools=getattr(policy, "allowed_tools", None),
|
||||
skill_stop_names=policy.skill_stop_names,
|
||||
thinking_enabled=policy.thinking_enabled,
|
||||
reasoning_effort=policy.reasoning_effort,
|
||||
recursion_limit=int(getattr(policy, "recursion_limit", 1000)),
|
||||
stage="step3_dashboard",
|
||||
)
|
||||
current_model = used or current_model
|
||||
|
||||
Reference in new issue
Block a user