Merge remote-tracking branch 'origin/master' into ji

This commit is contained in:
吉鑫升 committed 2026-06-11 16:39:30 +08:00
commit 5daa4b3e20
62 files changed
+4230 -1025

No files matched your search

@@ -109,20 +109,42 @@ async def _search_notebook(
"""从笔记 API 按关键词检索内容,返回 source_type='notebook' 的 Material 列表。"""
import httpx # 懒导入:仅在笔记模式下才需要
# 入参全 dump(token 只打长度,不打明文,防止泄漏到日志)
logger.info(
"[notebook-search] called keywords=%s search_space_id=%s source_ids=%s token_len=%d",
keywords,
search_space_id,
source_ids,
len(token) if token else 0,
)
base = (os.getenv("DEER_FLOW_NOTEBOOK_API_BASE") or "").rstrip("/")
if not base:
logger.warning("DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search")
# 改成 error 级别,warning 在很多部署里默认会被过滤掉看不到
logger.error(
"[notebook-search] DEER_FLOW_NOTEBOOK_API_BASE not set, skipping notebook search. "
"请在后端启动环境里设这个环境变量(指向笔记后端根地址,例如 http://localhost:8003)。"
)
return []
full_url = f"{base}{_NOTEBOOK_SEARCH_PATH}"
logger.info("[notebook-search] base=%s full_url=%s", base, full_url)
results: list[Material] = []
mat_idx = 0
try:
async with httpx.AsyncClient(timeout=20) as client:
for kw in keywords[:_NOTEBOOK_MAX_KEYWORDS]:
logger.info("[notebook-search] POST kw=%r body=%s", kw, {
"query": kw,
"source_ids": source_ids,
"search_space_id": search_space_id,
"top_k": _NOTEBOOK_TOP_K,
})
try:
# /mentioned-documents/search 接口无需鉴权,不传 Authorization header
resp = await client.post(
f"{base}{_NOTEBOOK_SEARCH_PATH}",
headers={"Authorization": f"Bearer {token}"},
full_url,
json={
"query": kw,
"source_ids": source_ids,
@@ -130,10 +152,29 @@ async def _search_notebook(
"top_k": _NOTEBOOK_TOP_K,
},
)
logger.info(
"[notebook-search] kw=%r http_status=%d body_len=%d",
kw,
resp.status_code,
len(resp.content or b""),
)
if resp.status_code >= 400:
# 把错误响应体打出来,方便定位 401 / 422 等
logger.error(
"[notebook-search] kw=%r HTTP %d response: %s",
kw,
resp.status_code,
(resp.text or "")[:1000],
)
resp.raise_for_status()
data = resp.json()
# 解析 documents 列表,每个 chunk 作为独立素材
documents: list[dict] = data.get("documents") if isinstance(data, dict) else []
logger.info(
"[notebook-search] kw=%r parsed documents=%d",
kw,
len(documents or []),
)
for doc in documents or []:
doc_title: str = doc.get("title") or "笔记文档"
doc_url: str = doc.get("url") or ""
@@ -154,10 +195,11 @@ async def _search_notebook(
url=doc_url,
))
except Exception as e:
logger.warning("notebook search kw=%r failed: %s", kw, e)
logger.warning("[notebook-search] kw=%r failed: %s", kw, e, exc_info=True)
except Exception as e:
logger.warning("notebook search client error: %s", e)
logger.warning("[notebook-search] client error: %s", e, exc_info=True)
logger.info("[notebook-search] done — returning %d materials", len(results))
return results
@@ -250,6 +292,7 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# Step 1.5: 笔记检索(如果传入了 notebook_search_space_id)
notebook_search_space_id: int | None = state.get("notebook_search_space_id")
notebook_materials: list[Material] = []
if notebook_search_space_id:
notebook_source_ids: list[int] = state.get("notebook_source_ids") or []
notebook_token: str = cfg.get("notebook_token") or ""
@@ -369,6 +412,13 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
all_results_nested: list = []
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
elif material_source == "notebook" and notebook_search_space_id:
# ── 纯笔记模式 ──
# 用户在表单选了「我的空间」作为素材来源 → 跳过 web/intranet 检索,
# 仅使用 Step 1.5 已拉取的 notebook_materials。
# 与 UI 文案"从空间中检索素材"对齐。
_emit(f"仅使用笔记素材,跳过 web/intranet 检索({len(notebook_materials)} 条)")
all_results_nested = []
else:
# ── 常规 web/intranet 模式 ──
provider = get_search_provider(ai_writing_config)
@@ -331,6 +331,13 @@ def _build_middlewares(
max_concurrent_subagents = cfg.get("max_concurrent_subagents", 3)
middlewares.append(SubagentLimitMiddleware(max_concurrent=max_concurrent_subagents))
# SkillReviewMiddleware — trigger background skill review after each turn.
# Gated by skill_evolution.enabled + skill_evolution.review_worker.enabled.
if resolved_app_config.skill_evolution.enabled and resolved_app_config.skill_evolution.review_worker.enabled:
from deerflow.agents.middlewares.skill_review_middleware import SkillReviewMiddleware
middlewares.append(SkillReviewMiddleware())
# LoopDetectionMiddleware — detect and break repetitive tool call loops
middlewares.append(LoopDetectionMiddleware())
@@ -1194,7 +1194,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
<notebook_mode>
## 角色
你是一个笔记本 AI 助手,**只能**基于笔记本文档来源回答用户问题,禁止使用任何其他知识。
你是一个空间 AI 助手,**只能**基于空间文档来源回答用户问题,禁止使用任何其他知识。
## 可用知识来源
{source_line}
@@ -1220,7 +1220,7 @@ When a search, knowledge, skill, or MCP tool returns a result list:
- 未调用 `search_notebook_sources` 就直接回答
- 引用不在检索结果中的 chunk_id
- 省略引用标记
- 使用笔记本来源之外的知识
- 使用空间来源之外的知识
</notebook_mode>"""
return prompt
@@ -0,0 +1,54 @@
"""SkillReviewMiddleware — triggers background skill review after each agent turn.
Hooks into ``after_agent``: once the lead agent graph has finished a full turn,
this middleware counts the tool calls made and, if the threshold is met, hands
the conversation messages to the SkillReviewWorker as a background asyncio task.
No blocking I/O or LLM calls happen in-band — the review runs fully in the
background and never delays the HTTP response.
"""
from __future__ import annotations
import logging
from typing import Any, override
from langchain.agents import AgentState
from langchain.agents.middleware import AgentMiddleware
from langgraph.runtime import Runtime
from deerflow.config.app_config import get_app_config
from deerflow.runtime.user_context import get_effective_user_id
from deerflow.skills.review_worker import maybe_start_review
logger = logging.getLogger(__name__)
class SkillReviewMiddlewareState(AgentState):
"""Minimal state schema — only needs ``messages``."""
pass
class SkillReviewMiddleware(AgentMiddleware[SkillReviewMiddlewareState]):
"""Dispatch a background skill review after every substantial agent turn.
'Substantial' is defined by ``skill_evolution.review_worker.min_tool_calls``
(default 5). Turns below the threshold are silently skipped to avoid
wasting LLM tokens on trivial exchanges.
"""
state_schema = SkillReviewMiddlewareState
@override
def after_agent(self, state: Any, runtime: Runtime) -> None:
"""Fire background review; never raises — failures are logged only."""
try:
app_config = get_app_config()
messages = state.get("messages", []) if hasattr(state, "get") else []
thread_id: str = runtime.context.get("thread_id", "") if runtime.context else ""
if not thread_id:
return
user_id = get_effective_user_id()
maybe_start_review(thread_id, messages, app_config, user_id)
except Exception:
logger.exception("SkillReviewMiddleware.after_agent: unexpected error (ignored)")
@@ -107,7 +107,7 @@ class ThreadDataMiddleware(AgentMiddleware[ThreadDataMiddlewareState]):
content=last_message.content,
id=last_message.id,
name=last_message.name,
additional_kwargs={**last_message.additional_kwargs, "run_id": runtime.context.get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
additional_kwargs={**last_message.additional_kwargs, "run_id": (runtime.context or {}).get("run_id"), "timestamp": datetime.now(UTC).isoformat()},
)
return {