Merge branch 'master' of http://192.168.0.200/knowledge-base-bj/deerflow-server into xuhd
This commit is contained in:
commit
22b93e32eb
493 files changed
+67159
-3160
No files matched your search
+16
-3
@@ -55,6 +55,9 @@ async def pause_material_confirm(state: AIWritingState) -> dict:
|
||||
|
||||
action = user_input.get("action", "confirm")
|
||||
|
||||
# 单输入框文本:可能是技能名 / 来源 / 检索意图,re_search 和 enable_skill_search 都用得到
|
||||
user_query = (user_input.get("userQuery") or user_input.get("user_query") or "").strip()
|
||||
|
||||
if action == "confirm":
|
||||
approved_ids = set(user_input.get("approvedMaterialIds") or [m["id"] for m in pkg["materials"]])
|
||||
filtered = [m for m in pkg["materials"] if m["id"] in approved_ids]
|
||||
@@ -66,12 +69,22 @@ async def pause_material_confirm(state: AIWritingState) -> dict:
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "用户已确认素材,开始规划大纲"}],
|
||||
}
|
||||
|
||||
# re_search:重新搜索
|
||||
# enable_skill_search:用户选择"启用技能检索",回到 researcher 走 skill 模式
|
||||
if action == "enable_skill_search":
|
||||
msg = f"已启用技能检索(输入:{user_query[:30]})" if user_query else "已启用技能检索,正在筛选可用技能…"
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "action": "enable_skill_search", "user_query": user_query},
|
||||
"status": "researching",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": msg}],
|
||||
}
|
||||
|
||||
# re_search:重新搜索(支持新版 user_query 单输入框 + 旧版 extraKeywords 数组)
|
||||
extra_kws = user_input.get("extraKeywords") or []
|
||||
msg = f"按输入「{user_query[:30]}」重新搜索…" if user_query else "重新搜索中..."
|
||||
return {
|
||||
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws},
|
||||
"last_user_intervention": {**user_input, "_extra_keywords": extra_kws, "user_query": user_query},
|
||||
"status": "researching",
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": "重新搜索中..."}],
|
||||
"progress_events": [{"type": "resumed", "agent_name": "系统", "message": msg}],
|
||||
}
|
||||
|
||||
|
||||
|
||||
+197
-73
@@ -10,6 +10,7 @@ from typing import List
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..search import get_search_provider, SearchResult
|
||||
from ..search.skill_search import SkillPicker
|
||||
from ..state import AIWritingState, Material, MaterialPackage
|
||||
from ..prompts.researcher_prompts import (
|
||||
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
|
||||
@@ -96,15 +97,24 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
# 检查干预模式(PAUSE-1 素材确认 / 素材不足求助)
|
||||
intervention = state.get("last_user_intervention") or {}
|
||||
extra_keywords: list[str] = intervention.get("_extra_keywords") or intervention.get("extra_keywords") or []
|
||||
# PAUSE-1 单输入框:用户写的自然语言(技能名 / 来源 / 检索意图),优先级最高
|
||||
user_query: str = (intervention.get("user_query") or "").strip()
|
||||
action = intervention.get("action")
|
||||
|
||||
# 素材不足求助点触发的「补充检索」:保留原大纲,检索完直接回写作、不走素材确认/大纲
|
||||
is_supplement = action == "supplement_search"
|
||||
# PAUSE-1 素材确认卡用户点「追加检索」并填了新关键词:
|
||||
# PAUSE-1 素材确认卡用户点「按输入重新搜索」并填了 user_query:
|
||||
# - 跳过 LLM 重新生成检索词(节省一次调用 + 尊重用户意图)
|
||||
# - 只检索 extra_keywords 这几个新词,不重检索之前 LLM 出过的词
|
||||
# - 直接用 user_query 作为唯一检索词
|
||||
# - 保留 existing material_package,按 URL 去重合并
|
||||
is_append_only = action == "re_search" and bool(extra_keywords)
|
||||
is_user_query_search = action == "re_search" and bool(user_query)
|
||||
# 兼容旧版"追加关键词"调用方(如果还有 client 传 extra_keywords 数组而非 user_query)
|
||||
is_append_only = action == "re_search" and not user_query and bool(extra_keywords)
|
||||
# PAUSE-1 素材确认卡用户点「启用技能检索」:
|
||||
# - 不再走 web/intranet 检索,改用 SkillSearchProvider
|
||||
# - 顺序调用候选技能 + 早停(第一个技能够就停,不够再加下一个)
|
||||
# - 旧 material_package 完全丢弃(覆盖式替换,让用户对比 web vs skill 结果)
|
||||
is_skill_search = action == "enable_skill_search"
|
||||
|
||||
# 用户指定的检索关键词数量(1~10),决定生成多少个检索词、检索多少个方向
|
||||
keyword_count = _resolve_keyword_count(state.get("keyword_count"))
|
||||
@@ -126,10 +136,17 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
writer({"type": "research_progress", "message": message})
|
||||
|
||||
# Step 1: 提取关键词
|
||||
kw_result: dict = {} # 走追加分支时跳过 LLM 调用,但下方 notice 拼接需要兜底值
|
||||
if is_append_only:
|
||||
# 追加模式:直接用用户填的关键词,不再调 LLM
|
||||
keywords: list[str] = _clean_keywords(extra_keywords, intent)
|
||||
kw_result: dict = {} # 走"用户输入 / 追加 / 技能"分支时跳过 LLM 调用,notice 拼接需要兜底值
|
||||
if is_skill_search:
|
||||
# 技能模式:每个 skill 会自己拼 query(user_query + skill.description),这里不用 LLM 生成关键词
|
||||
keywords: list[str] = [user_query or intent]
|
||||
elif is_user_query_search:
|
||||
# 用户输入分支:把整段自然语言作为唯一检索词,尊重原意,不再做关键词拆分
|
||||
keywords = [user_query]
|
||||
_emit(f"按你的输入「{user_query[:30]}」检索…")
|
||||
elif is_append_only:
|
||||
# 兼容旧版追加模式:直接用用户填的关键词,不再调 LLM
|
||||
keywords = _clean_keywords(extra_keywords, intent)
|
||||
_emit(f"追加检索 {len(keywords)} 个关键词…")
|
||||
else:
|
||||
_emit("正在生成检索词…")
|
||||
@@ -150,78 +167,172 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
if extra_keywords:
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
|
||||
# Step 2: 并发搜索
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
# Step 2: 搜索
|
||||
# 技能检索:技能不是素材,是"检索策略 / 视角"。流程:
|
||||
# a. SkillPicker 选出按相关度排序的 skills(推 list_skills + select_skills 事件)
|
||||
# b. 按顺序遍历 skills:每个 skill 用其 description 让 LLM 生成针对性关键词
|
||||
# c. 调真实 web/intranet provider 执行检索,推 invoke_skill + skill_result 事件
|
||||
# d. 累积素材到 max_materials 即 break,推 stop_early
|
||||
skill_steps: list[dict] = []
|
||||
search_errors: list[str] = []
|
||||
|
||||
# 按用户指定的关键词数量截断。
|
||||
# 追加模式:keywords 本身就是用户填的,全部检索不截断(用户填 N 个就搜 N 个)。
|
||||
# 其他模式:受 keyword_count 限制,但 extra_keywords 在内时上限放宽到 keywords 总数。
|
||||
if is_append_only:
|
||||
search_keywords = keywords
|
||||
def _emit_skill_step(step: dict) -> None:
|
||||
skill_steps.append(step)
|
||||
writer({"type": "skill_search_step", "agent_name": "素材收集专家", **step})
|
||||
|
||||
if is_skill_search:
|
||||
# ── 技能编排检索 ──
|
||||
# 技能选择走流式 LLM(推 skill_select_chunk)+ <think> 分离,前端打字机展示
|
||||
picker = SkillPicker(
|
||||
config=config,
|
||||
model_name=keyword_model,
|
||||
emit_step=_emit_skill_step,
|
||||
max_select=5,
|
||||
user_query=user_query,
|
||||
)
|
||||
selected = await picker.pick(user_intent=intent)
|
||||
search_errors.extend(picker.errors)
|
||||
|
||||
# 真实 web/intranet provider(按 config.yaml 配置)
|
||||
web_provider = get_search_provider(ai_writing_config)
|
||||
|
||||
# 每个 skill 尽量多取(不分摊),第一个就达到 max_materials 即可早停;
|
||||
# 这才贴近"第一个技能够了就停,否则下一个补充"的语义。
|
||||
per_skill = max(1, max_materials)
|
||||
raw_materials_pre: list[Material] = []
|
||||
seen_urls_pre: set[str] = set()
|
||||
stopped_early = False
|
||||
|
||||
for entry in selected:
|
||||
sk = entry["skill"]
|
||||
# 用 LLM 把 user_query + skill.description 转成针对性关键词
|
||||
_emit_skill_step({
|
||||
"type": "invoke_skill",
|
||||
"message": f"使用技能「{sk.name}」开始检索…",
|
||||
"skill_name": sk.name,
|
||||
})
|
||||
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
|
||||
try:
|
||||
res = await web_provider.search(skill_query, max_results=per_skill)
|
||||
except Exception as e:
|
||||
logger.warning("skill[%s] search failed: %s", sk.name, e)
|
||||
search_errors.append(f"skill[{sk.name}]: {e}")
|
||||
res = []
|
||||
|
||||
# 去重累积
|
||||
new_count = 0
|
||||
for r in res or []:
|
||||
if r.url and r.url in seen_urls_pre:
|
||||
continue
|
||||
if r.url:
|
||||
seen_urls_pre.add(r.url)
|
||||
raw_materials_pre.append(Material(
|
||||
id=f"m{len(raw_materials_pre)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=f"{r.source}(via 技能 {sk.name})",
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
new_count += 1
|
||||
|
||||
_emit_skill_step({
|
||||
"type": "skill_result",
|
||||
"message": f"技能「{sk.name}」检索到 {new_count} 条新素材",
|
||||
"skill_name": sk.name,
|
||||
"matched_count": new_count,
|
||||
})
|
||||
|
||||
# 早停:累积素材已达上限
|
||||
if len(raw_materials_pre) >= max_materials:
|
||||
stopped_early = True
|
||||
_emit_skill_step({
|
||||
"type": "stop_early",
|
||||
"message": f"已获取 {len(raw_materials_pre)} 条素材,达到上限,停止调用后续技能",
|
||||
"matched_count": len(raw_materials_pre),
|
||||
})
|
||||
break
|
||||
|
||||
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
|
||||
all_results_nested: list = []
|
||||
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
|
||||
else:
|
||||
search_limit = keyword_count
|
||||
if extra_keywords:
|
||||
search_limit = max(search_limit, len(keywords))
|
||||
search_keywords = keywords[:search_limit]
|
||||
total = len(search_keywords)
|
||||
# 每个检索词分到的检索条数 = 总量上限 / 检索词数(向上取整)
|
||||
per_keyword = _per_keyword_limit(max_materials, total)
|
||||
_emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
# ── 常规 web/intranet 模式 ──
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
# 并发搜索,但每完成一个就推送进度(计数在单线程 asyncio 下无需加锁)
|
||||
completed = 0
|
||||
# 按用户指定的关键词数量截断。
|
||||
# 用户输入 / 追加模式:keywords 本身就是用户填的,全部检索不截断。
|
||||
if is_user_query_search or is_append_only:
|
||||
search_keywords = keywords
|
||||
else:
|
||||
search_limit = keyword_count
|
||||
if extra_keywords:
|
||||
search_limit = max(search_limit, len(keywords))
|
||||
search_keywords = keywords[:search_limit]
|
||||
total = len(search_keywords)
|
||||
per_keyword = _per_keyword_limit(max_materials, total)
|
||||
_emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
|
||||
async def _search_one(kw: str) -> list[SearchResult]:
|
||||
nonlocal completed
|
||||
try:
|
||||
return await provider.search(kw, max_results=per_keyword)
|
||||
finally:
|
||||
completed += 1
|
||||
if completed < total:
|
||||
_emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
|
||||
else:
|
||||
_emit(f"已完成全部 {total} 次检索")
|
||||
completed = 0
|
||||
|
||||
search_tasks = [_search_one(kw) for kw in search_keywords]
|
||||
all_results_nested: list[list[SearchResult]] = await asyncio.gather(*search_tasks, return_exceptions=True)
|
||||
async def _search_one(kw: str) -> list[SearchResult]:
|
||||
nonlocal completed
|
||||
try:
|
||||
return await provider.search(kw, max_results=per_keyword)
|
||||
finally:
|
||||
completed += 1
|
||||
if completed < total:
|
||||
_emit(f"已完成 {completed}/{total} 次检索,正在继续检索相关内容…")
|
||||
else:
|
||||
_emit(f"已完成全部 {total} 次检索")
|
||||
|
||||
search_tasks = [_search_one(kw) for kw in search_keywords]
|
||||
all_results_nested = await asyncio.gather(*search_tasks, return_exceptions=True)
|
||||
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
|
||||
# 合并去重
|
||||
# - 首次检索 / 普通 re_search(不带 extra_keywords,用户只想 LLM 重出):旧素材丢弃
|
||||
# - 素材不足求助 supplement_search:保留旧素材 + 追加新检索结果
|
||||
# - PAUSE-1 追加检索 is_append_only:保留旧素材 + 仅检索新关键词
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials: list[Material] = []
|
||||
# - 技能检索 is_skill_search:覆盖式替换,旧素材完全丢弃(编排循环已经把素材累积到 raw_materials_pre)
|
||||
existing_pkg = state.get("material_package")
|
||||
preserve_existing = existing_pkg and (is_supplement or is_append_only)
|
||||
if preserve_existing:
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
if is_skill_search:
|
||||
# 技能编排循环已经完成素材累积,直接用 raw_materials_pre 当 raw_materials
|
||||
raw_materials: list[Material] = raw_materials_pre # type: ignore[name-defined]
|
||||
else:
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials = []
|
||||
preserve_existing = existing_pkg and (is_supplement or is_append_only)
|
||||
if preserve_existing:
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
|
||||
search_errors: list[str] = []
|
||||
for results in all_results_nested:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
continue
|
||||
for r in results:
|
||||
if r.url and r.url in seen_urls:
|
||||
for results in all_results_nested:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
continue
|
||||
seen_urls.add(r.url)
|
||||
raw_materials.append(Material(
|
||||
id=f"m{len(raw_materials)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=r.source,
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
|
||||
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
for r in results:
|
||||
if r.url and r.url in seen_urls:
|
||||
continue
|
||||
if r.url:
|
||||
seen_urls.add(r.url)
|
||||
raw_materials.append(Material(
|
||||
id=f"m{len(raw_materials)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=r.source,
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
|
||||
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
|
||||
_emit("检索完成,正在筛选排序素材…")
|
||||
@@ -264,6 +375,14 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
old_kws = list(existing_pkg.get("keywords") or [])
|
||||
pkg_keywords = list(dict.fromkeys(old_kws + keywords))
|
||||
pkg = MaterialPackage(keywords=pkg_keywords, materials=materials, summary=summary)
|
||||
# 技能检索时把过程步骤一并放进 pkg,前端在素材确认卡里能展示"使用了哪些技能"
|
||||
if is_skill_search:
|
||||
pkg["skill_search_used"] = True
|
||||
pkg["skill_search_steps"] = skill_steps
|
||||
elif existing_pkg and existing_pkg.get("skill_search_used") and (is_supplement or is_append_only):
|
||||
# 保留之前一次技能检索的痕迹(不污染纯 web 检索)
|
||||
pkg["skill_search_used"] = existing_pkg.get("skill_search_used", False)
|
||||
pkg["skill_search_steps"] = existing_pkg.get("skill_search_steps", [])
|
||||
|
||||
# 容错告警:检索接口 / 模型调用出错时以非致命 notice 提示用户,不打断写作流程
|
||||
notices: list[dict] = []
|
||||
@@ -274,19 +393,24 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
if search_errors:
|
||||
uniq = list(dict.fromkeys(search_errors))
|
||||
# 解析失败时收集到的接口原始响应体,前端折叠展示便于定位
|
||||
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
|
||||
# (技能模式下 provider 局部变量没暴露,所以 raw_dumps 兜底为空列表)
|
||||
provider_obj = locals().get("provider") or locals().get("web_provider")
|
||||
raw_dumps = list(dict.fromkeys(getattr(provider_obj, "raw_responses", []) or []))
|
||||
notices.append(_notice(
|
||||
"ES 波动导致检索异常",
|
||||
";".join(uniq[:3]),
|
||||
raw="\n\n----------\n\n".join(raw_dumps[:3]),
|
||||
))
|
||||
|
||||
# 进度事件文案:追加模式下让用户看到只搜了哪几个新词
|
||||
step_start_msg = (
|
||||
f"追加检索关键词:{', '.join(keywords[:3])}"
|
||||
if is_append_only
|
||||
else f"搜索关键词:{', '.join(keywords[:3])}"
|
||||
)
|
||||
# 进度事件文案:用户输入 / 追加模式 / 技能模式下让用户看到差异化提示
|
||||
if is_skill_search:
|
||||
step_start_msg = f"启用技能检索({len(skill_steps)} 步)"
|
||||
elif is_user_query_search:
|
||||
step_start_msg = f"按你的输入检索:{user_query[:40]}"
|
||||
elif is_append_only:
|
||||
step_start_msg = f"追加检索关键词:{', '.join(keywords[:3])}"
|
||||
else:
|
||||
step_start_msg = f"搜索关键词:{', '.join(keywords[:3])}"
|
||||
return {
|
||||
"material_package": pkg,
|
||||
# 补充检索:直接回写作(沿用原大纲);常规检索:进入素材确认暂停点
|
||||
|
||||
+2
@@ -3,6 +3,7 @@ from .internet_search import InternetSearchProvider
|
||||
from .intranet_stub import IntranetSearchProvider
|
||||
from .demo_search import DemoSearchProvider
|
||||
from .duckduckgo_search import DuckDuckGoSearchProvider
|
||||
from .skill_search import SkillSearchProvider
|
||||
|
||||
|
||||
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
|
||||
@@ -60,5 +61,6 @@ __all__ = [
|
||||
"BaseSearchProvider", "SearchResult",
|
||||
"InternetSearchProvider", "IntranetSearchProvider",
|
||||
"DemoSearchProvider", "DuckDuckGoSearchProvider",
|
||||
"SkillSearchProvider",
|
||||
"get_search_provider",
|
||||
]
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
"""技能选择器:根据用户意图 + 输入,从启用的 Skills 里挑出按相关度排序的若干个。
|
||||
|
||||
注意:这个模块**只负责选择技能,不直接产出素材**。
|
||||
真正的素材产出由 researcher 节点用每个 skill 的描述去编排真实 web/intranet 检索。
|
||||
|
||||
流程(对应素材确认卡上的"技能检索"按钮):
|
||||
1) list_skills 事件 —— 列出当前启用的全部 skill
|
||||
2) skill_select_chunk 事件 —— LLM 流式打字 + <think></think> 分离(前端打字机展示)
|
||||
3) select_skills 事件 —— LLM 给出的排序结果(结构化字段,写到 SkillStepsCard 里)
|
||||
|
||||
下游 researcher 节点拿到 selected_skills 后,**按顺序**:
|
||||
- 用 skill.description + user_query 生成针对性 query
|
||||
- 调真实 web search provider 执行
|
||||
- 累积素材达到阈值就停(推 stop_early)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
from langchain_core.messages import HumanMessage, SystemMessage
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..nodes._utils import (
|
||||
ThinkTagParser,
|
||||
_content_to_text,
|
||||
get_model,
|
||||
parse_json,
|
||||
safe_get_stream_writer,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
_SELECT_SKILLS_SYSTEM = """你是一名情报研究员,需要从可用「技能」中挑出对当前检索任务最有帮助的若干个,并按相关度从高到低排序。
|
||||
技能是一段封装好的指令 / 知识 / 检索策略,每个技能有 name 和 description。
|
||||
|
||||
【输出要求】
|
||||
- 只输出一个 JSON 对象,不要 markdown 代码块。
|
||||
- 必须使用英文双引号。
|
||||
- selected_skills 中的 name 必须严格来自下方"可用技能"列表,不要编造,按相关度从高到低排序。
|
||||
- 如果用户输入里明确点名了某个技能(按 name 完全匹配),把它排在第一位。
|
||||
- 至少选 1 个,最多选 {max_select} 个。
|
||||
|
||||
【完整示例】
|
||||
{{"selected_skills": ["platform-sales-data", "industry-report"], "reason": "用户想查各平台销量,平台销量技能最相关"}}"""
|
||||
|
||||
_SELECT_SKILLS_USER = """写作意图:{user_intent}
|
||||
用户补充输入:{user_query}
|
||||
|
||||
可用技能(共 {skill_count} 个):
|
||||
{skill_list}
|
||||
|
||||
请从中按相关度从高到低选出最多 {max_select} 个技能。
|
||||
只输出 JSON:
|
||||
{{"selected_skills": ["name1", "name2"], "reason": "为什么选这些 + 排序依据"}}"""
|
||||
|
||||
|
||||
EmitStepFn = Callable[[dict], None]
|
||||
|
||||
|
||||
class SkillPicker:
|
||||
"""从启用的 Skills 中流式选择按相关度排序的若干个供下游编排使用。
|
||||
|
||||
构造参数:
|
||||
config: LangGraph runnable config(用于创建 model)
|
||||
model_name: 指定使用的模型,None 则用 configurable.model_name
|
||||
emit_step: 推送时间线步骤事件(list_skills / select_skills)
|
||||
max_select: LLM 单次最多挑几个技能
|
||||
user_query: 用户在素材确认卡输入框写的自然语言
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: RunnableConfig,
|
||||
*,
|
||||
model_name: Optional[str] = None,
|
||||
emit_step: Optional[EmitStepFn] = None,
|
||||
max_select: int = 5,
|
||||
user_query: str = "",
|
||||
) -> None:
|
||||
self._config = config
|
||||
self._model_name = model_name
|
||||
self._emit_step = emit_step or (lambda _step: None)
|
||||
self._max_select = max_select
|
||||
self._user_query = (user_query or "").strip()
|
||||
self._errors: list[str] = []
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
return list(self._errors)
|
||||
|
||||
def _load_enabled_skills(self) -> list:
|
||||
try:
|
||||
from deerflow.skills import get_or_new_skill_storage
|
||||
storage = get_or_new_skill_storage()
|
||||
skills = storage.load_skills(enabled_only=True)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing.skill_search: 加载启用技能失败")
|
||||
self._errors.append(f"skill list error: {e}")
|
||||
skills = []
|
||||
self._emit_step({
|
||||
"type": "list_skills",
|
||||
"message": f"发现 {len(skills)} 个可用技能",
|
||||
"skill_names": [s.name for s in skills],
|
||||
})
|
||||
return skills
|
||||
|
||||
async def pick(self, user_intent: str) -> List[dict]:
|
||||
"""流式选择并排序候选技能,返回 [{skill, reason}, ...],按相关度降序。
|
||||
|
||||
过程中通过 custom 通道推送 ``skill_select_chunk`` 事件给前端打字机;
|
||||
结束后再推一条 ``select_skills`` 结构化事件。
|
||||
"""
|
||||
skills = self._load_enabled_skills()
|
||||
if not skills:
|
||||
self._emit_step({
|
||||
"type": "select_skills",
|
||||
"message": "当前没有启用的技能可用",
|
||||
"skill_names": [],
|
||||
})
|
||||
return []
|
||||
|
||||
skill_list_text = "\n".join(
|
||||
f"- {s.name}: {s.description or '(无描述)'}" for s in skills
|
||||
)
|
||||
|
||||
fallback = {"selected_skills": [s.name for s in skills[: self._max_select]], "reason": ""}
|
||||
|
||||
# 流式 LLM 调用:与 intent_parser / writer_outline 同套机制
|
||||
writer = safe_get_stream_writer()
|
||||
parser = ThinkTagParser()
|
||||
full_text = ""
|
||||
|
||||
def _emit_chunk(text: str, is_thinking: bool) -> None:
|
||||
writer({
|
||||
"type": "skill_select_chunk",
|
||||
"text": text,
|
||||
"is_thinking": is_thinking,
|
||||
"agent_name": "素材收集专家",
|
||||
})
|
||||
|
||||
try:
|
||||
model = get_model(self._config, model_name=self._model_name)
|
||||
async for chunk in model.astream([
|
||||
SystemMessage(content=_SELECT_SKILLS_SYSTEM.format(max_select=self._max_select)),
|
||||
HumanMessage(content=_SELECT_SKILLS_USER.format(
|
||||
user_intent=user_intent or "(无明确写作意图)",
|
||||
user_query=self._user_query or "(无)",
|
||||
skill_count=len(skills),
|
||||
skill_list=skill_list_text,
|
||||
max_select=self._max_select,
|
||||
)),
|
||||
]):
|
||||
delta = _content_to_text(chunk.content)
|
||||
if not delta:
|
||||
continue
|
||||
for is_thinking, text in parser.feed(delta):
|
||||
if not is_thinking:
|
||||
full_text += text
|
||||
_emit_chunk(text, is_thinking)
|
||||
except Exception as e:
|
||||
logger.exception("ai_writing.skill_search: 流式选择技能失败")
|
||||
self._errors.append(f"skill select error: {e}")
|
||||
|
||||
result = parse_json(full_text, fallback) if full_text else fallback
|
||||
|
||||
wanted_names = result.get("selected_skills") or []
|
||||
if not isinstance(wanted_names, list):
|
||||
wanted_names = []
|
||||
by_name = {s.name: s for s in skills}
|
||||
reason = result.get("reason", "") or ""
|
||||
chosen: list[dict] = []
|
||||
for name in wanted_names:
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
sk = by_name.get(name.strip())
|
||||
if sk and sk.name not in {c["skill"].name for c in chosen}:
|
||||
chosen.append({"skill": sk, "reason": reason})
|
||||
if len(chosen) >= self._max_select:
|
||||
break
|
||||
|
||||
self._emit_step({
|
||||
"type": "select_skills",
|
||||
"message": f"按相关度排序,候选 {len(chosen)} 个技能(将按顺序使用)",
|
||||
"skill_names": [c["skill"].name for c in chosen],
|
||||
"reason": reason,
|
||||
})
|
||||
return chosen
|
||||
|
||||
|
||||
# 保留旧名字让 search/__init__.py 的导出不报错;研究节点不再用它
|
||||
SkillSearchProvider = SkillPicker
|
||||
+29
-4
@@ -3,21 +3,43 @@ import operator
|
||||
|
||||
# ── 素材 ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
class Material(TypedDict):
|
||||
class Material(TypedDict, total=False):
|
||||
id: str
|
||||
title: str
|
||||
content: str # 截断到 500 字
|
||||
source: str
|
||||
published_at: str
|
||||
relevance_score: float
|
||||
source_type: str # "internet" | "intranet"
|
||||
source_type: str # "internet" | "intranet" | "skill"
|
||||
url: str
|
||||
recUuid: str
|
||||
# skill 类素材附带的元信息(skill_name / container_path / category),其他类型为空
|
||||
extra_metadata: dict
|
||||
|
||||
class MaterialPackage(TypedDict):
|
||||
class SkillStep(TypedDict, total=False):
|
||||
"""技能检索过程的单步记录,用于前端时间线展示。
|
||||
|
||||
type=match_skills:根据用户输入流式分析有没有匹配技能(无 skill_names)
|
||||
type=list_skills:列出可用技能
|
||||
type=select_skills:LLM 从中选出的相关技能
|
||||
type=invoke_skill:实际调用某一个技能检索
|
||||
type=stop_early:达到素材阈值,提前停止后续技能调用
|
||||
"""
|
||||
type: str
|
||||
message: str
|
||||
skill_names: List[str]
|
||||
skill_name: str
|
||||
matched_count: int
|
||||
reason: str
|
||||
|
||||
class MaterialPackage(TypedDict, total=False):
|
||||
keywords: List[str]
|
||||
materials: List[Material]
|
||||
summary: str
|
||||
# 是否本次素材来自技能检索(覆盖式替换 web 检索)
|
||||
skill_search_used: bool
|
||||
# 技能检索的步骤记录(前端展示"使用了 X 技能"卡片)
|
||||
skill_search_steps: List[SkillStep]
|
||||
|
||||
# ── 大纲 ──────────────────────────────────────────────────────────────────────
|
||||
# 三级结构:章节(##) → 小节(###) → 子要点(bullet -)。
|
||||
@@ -72,10 +94,13 @@ class ReviewResult(TypedDict):
|
||||
|
||||
# ── 用户干预 ──────────────────────────────────────────────────────────────────
|
||||
|
||||
class UserInterventionPayload(TypedDict):
|
||||
class UserInterventionPayload(TypedDict, total=False):
|
||||
action: str
|
||||
approved_material_ids: List[str]
|
||||
extra_keywords: List[str]
|
||||
# PAUSE-1 素材确认 / enable_skill_search 共用:用户在单输入框里填的自然语言
|
||||
# (可能是要使用的技能名、想检索的来源、或者一段补充检索意图)。
|
||||
user_query: str
|
||||
edited_outline: Optional[Outline]
|
||||
outline_feedback: str
|
||||
user_revision_notes: str
|
||||
|
||||
+18
-16
@@ -703,12 +703,14 @@ def _get_cached_skills_prompt_section(
|
||||
|
||||
display_pattern = (
|
||||
"""
|
||||
**Skill Result Display Pattern:**
|
||||
- If a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[[1]]`, the second as `[[2]]`, and so on. Use only numbers that correspond to returned results.
|
||||
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
|
||||
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
|
||||
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
|
||||
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
|
||||
**Skill Result Display Pattern:**
|
||||
- If RAG/reference display mode is enabled, do not number each skill result independently. Treat all retrieval results from the same user turn as one combined `referenceBatch` and cite with `[1]`, `[2]`, etc. from the system-provided reference prompt.
|
||||
- In RAG/reference display mode, do not output `<skill-display .../>`; the frontend renders cards, tables, lists, and citation details in the right reference panel.
|
||||
- If RAG/reference display mode is not enabled and a skill has `<display mode="citation">`, use the skill's returned result order as reference numbers. Cite facts from the first result as `[1]`, the second as `[2]`, and so on. Use only numbers that correspond to returned results.
|
||||
- If a skill has `<display mode="list">`, summarize the returned items as a concise list and keep item order stable.
|
||||
- If a skill has `<display mode="table">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the frontend should render the real table. Do not hand-write the table rows yourself.
|
||||
- If a skill has `<display mode="card">`, summarize the result and output `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the frontend should render the cards.
|
||||
- If a skill has `<display mode="custom">`, follow the custom display instruction in the `<display>` tag when presenting the skill result.
|
||||
"""
|
||||
if display_signature
|
||||
else ""
|
||||
@@ -993,16 +995,16 @@ Writing mode is enabled for this run.
|
||||
network_policy_section = _build_network_policy_section()
|
||||
rag_mode_section = (
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- If the skill has a display adapter, follow that adapter first.
|
||||
- For `citation`, cite returned items with `[[1]]`, `[[2]]`, etc. Only cite real returned items.
|
||||
- For `table`, do not hand-write the table. Summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="table" />` where the table should appear. The frontend will render the real tool result as a stable table.
|
||||
- For `list` or `card`, summarize and place `<skill-display tool="SKILL_OR_TOOL_NAME" mode="list" />` or `<skill-display tool="SKILL_OR_TOOL_NAME" mode="card" />` where the structured result should appear.
|
||||
- If no display adapter exists, default to citation mode with numbered citations.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
<rag_citation_mode>
|
||||
RAG/reference display mode is enabled.
|
||||
When a search, knowledge, skill, or MCP tool returns a result list:
|
||||
- If multiple search terms are needed, prefer calling the retrieval tools for all known terms in the same step, then answer after the tool results are returned.
|
||||
- Treat every retrieval result from the same user turn as one combined `referenceBatch`; do not restart numbering for each search term, tool call, skill, or display adapter.
|
||||
- Use only the system-provided merged reference numbers in the final answer, formatted as `[1]`, `[2]`, `[1][3][5]`.
|
||||
- Do not output `<skill-display .../>` in RAG/reference mode. The frontend renders configured cards, tables, lists, and citation details in the right reference panel.
|
||||
- If no display adapter exists, the frontend defaults the right reference panel to numbered citations.
|
||||
</rag_citation_mode>
|
||||
"""
|
||||
if rag_mode_enabled
|
||||
else ""
|
||||
)
|
||||
|
||||
+4
-2
@@ -80,13 +80,15 @@ class RagCitationMiddleware(AgentMiddleware[AgentState]):
|
||||
return None
|
||||
summary = build_reference_debug_summary(messages, [batch] if batch else None)
|
||||
_log_rag_debug(
|
||||
"middleware reference-prompt enabled=true prompt=true messages=%s tool_messages=%s "
|
||||
"raw_items=%s normalized_items=%s kept_items=%s latest_batch_sources=%s",
|
||||
"middleware reference-prompt enabled=true prompt=true single_batch=true messages=%s "
|
||||
"tool_messages=%s raw_items=%s normalized_items=%s kept_items=%s latest_batch_id=%s "
|
||||
"latest_batch_sources=%s",
|
||||
summary["messages"],
|
||||
summary["tool_messages"],
|
||||
summary["raw_items"],
|
||||
summary["normalized_items"],
|
||||
summary["kept_items"],
|
||||
(batch or {}).get("id"),
|
||||
len((batch or {}).get("sources") or []),
|
||||
)
|
||||
_log_rag_debug(
|
||||
|
||||
@@ -37,24 +37,45 @@ def build_reference_batches(messages: list[Any]) -> list[dict[str, Any]]:
|
||||
return batches
|
||||
|
||||
|
||||
def build_latest_reference_prompt(messages: list[Any]) -> tuple[str, dict[str, Any] | None]:
|
||||
def build_latest_reference_batch(messages: list[Any]) -> dict[str, Any] | None:
|
||||
"""Return the single combined citation batch for the latest human turn."""
|
||||
human_index = _find_latest_human_index(messages)
|
||||
if human_index is None:
|
||||
return "", None
|
||||
return None
|
||||
sources = _collect_sources(messages[human_index + 1 :])
|
||||
if not sources:
|
||||
return "", None
|
||||
return None
|
||||
assistant = _find_reference_batch_anchor(messages, human_index + 1, len(messages))
|
||||
message_id = _message_id(assistant) or _message_id(messages[human_index]) or str(human_index)
|
||||
batch = {
|
||||
return {
|
||||
"id": message_id,
|
||||
"assistant_message_id": message_id,
|
||||
"source_count": len(sources),
|
||||
"sources": sources,
|
||||
}
|
||||
|
||||
|
||||
def build_latest_reference_prompt(messages: list[Any]) -> tuple[str, dict[str, Any] | None]:
|
||||
batch = build_latest_reference_batch(messages)
|
||||
if batch is None:
|
||||
return "", None
|
||||
return build_reference_prompt_for_batch(batch), batch
|
||||
|
||||
|
||||
def build_reference_prompt_for_batch(batch: dict[str, Any]) -> str:
|
||||
"""Build the authoritative model prompt from one already-combined batch."""
|
||||
sources = [
|
||||
source
|
||||
for source in batch.get("sources", [])
|
||||
if isinstance(source, dict)
|
||||
]
|
||||
source_count = len(sources)
|
||||
lines = [
|
||||
"本轮检索结果已由系统统一合并并重新编号。最终回答必须使用下面这套编号。",
|
||||
"如果你仍需要补充检索,请先继续调用工具;最终回答只能引用最新合并后的编号。",
|
||||
f"本轮所有检索结果已合并为同一个 referenceBatch,共 {source_count} 条,编号从 [1] 到 [{source_count}] 唯一连续。",
|
||||
"下面这一批就是当前回答唯一可用的参考文献批次;不要按检索词、工具调用次数或工具原始编号重新分批编号。",
|
||||
"如果你仍需要补充检索,请只继续调用工具,不要输出正文;最终回答只能引用补充检索后系统重新合并出的最新 referenceBatch。",
|
||||
"",
|
||||
f"<reference_batch source_count=\"{source_count}\">",
|
||||
]
|
||||
for source in sources:
|
||||
title = source.get("title") or f"参考文献 {source.get('index')}"
|
||||
@@ -73,18 +94,20 @@ def build_latest_reference_prompt(messages: list[Any]) -> tuple[str, dict[str, A
|
||||
if content:
|
||||
lines.append(content)
|
||||
lines.append("")
|
||||
lines.append("</reference_batch>")
|
||||
lines.append("")
|
||||
lines.extend(
|
||||
[
|
||||
"引用规则:",
|
||||
"1. 回答正文中的关键结论必须标注对应编号,例如 [1]、[2]。",
|
||||
"2. 多个来源可写为 [1][3][5]。",
|
||||
"3. 只能引用上方存在的编号,不要编造编号。",
|
||||
"4. 不要使用工具原始编号,也不要重新排序编号。",
|
||||
"4. 不要使用工具原始编号,也不要重新排序、重新分批或重新从 [1] 开始编号。",
|
||||
"5. 不要在正文末尾堆完整参考文献列表,前端会展示右侧参考文献。",
|
||||
"6. 参考文献模式下不要输出 <skill-display .../> 占位符,也不要在正文里额外渲染卡片、表格或列表;前端会在右侧参考文献区域展示。",
|
||||
]
|
||||
)
|
||||
return "\n".join(lines).strip(), batch
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def _find_latest_human_index(messages: list[Any]) -> int | None:
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Tests for RAG citation batch construction."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from deerflow.runtime.references import (
|
||||
build_latest_reference_batch,
|
||||
build_latest_reference_prompt,
|
||||
build_reference_batches,
|
||||
)
|
||||
|
||||
|
||||
def _tool_message(call_id: str, query: str, start: int) -> dict:
|
||||
return {
|
||||
"type": "tool",
|
||||
"name": "web_search",
|
||||
"tool_call_id": call_id,
|
||||
"content": json.dumps(
|
||||
{
|
||||
"query": query,
|
||||
"results": [
|
||||
{
|
||||
"title": f"{query} result {start}",
|
||||
"content": f"{query} content {start}",
|
||||
"url": f"https://example.test/{start}",
|
||||
},
|
||||
{
|
||||
"title": f"{query} result {start + 1}",
|
||||
"content": f"{query} content {start + 1}",
|
||||
"url": f"https://example.test/{start + 1}",
|
||||
},
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def test_latest_reference_prompt_uses_one_combined_batch_for_multi_search_turn() -> None:
|
||||
messages = [
|
||||
{"type": "human", "id": "human-1", "content": "search two terms"},
|
||||
{
|
||||
"type": "ai",
|
||||
"id": "ai-search-1",
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "call-1", "name": "web_search", "args": {"query": "Trump"}}],
|
||||
},
|
||||
_tool_message("call-1", "Trump", 1),
|
||||
{
|
||||
"type": "ai",
|
||||
"id": "ai-search-2",
|
||||
"content": "",
|
||||
"tool_calls": [{"id": "call-2", "name": "web_search", "args": {"query": "White House"}}],
|
||||
},
|
||||
_tool_message("call-2", "White House", 3),
|
||||
]
|
||||
|
||||
batch = build_latest_reference_batch(messages)
|
||||
assert batch is not None
|
||||
assert batch["source_count"] == 4
|
||||
assert [source["index"] for source in batch["sources"]] == [1, 2, 3, 4]
|
||||
assert [source["title"] for source in batch["sources"]] == [
|
||||
"Trump result 1",
|
||||
"Trump result 2",
|
||||
"White House result 3",
|
||||
"White House result 4",
|
||||
]
|
||||
|
||||
prompt, prompt_batch = build_latest_reference_prompt(messages)
|
||||
assert prompt_batch == batch
|
||||
assert '<reference_batch source_count="4">' in prompt
|
||||
assert "[1] Trump result 1" in prompt
|
||||
assert "[4] White House result 4" in prompt
|
||||
assert "重新分批" in prompt
|
||||
|
||||
|
||||
def test_reference_batches_keep_separate_user_turns_but_merge_searches_inside_turn() -> None:
|
||||
messages = [
|
||||
{"type": "human", "id": "human-1", "content": "first"},
|
||||
{"type": "ai", "id": "ai-1", "content": "", "tool_calls": [{"id": "c1", "name": "web_search"}]},
|
||||
_tool_message("c1", "first", 1),
|
||||
{"type": "ai", "id": "answer-1", "content": "answer [1]"},
|
||||
{"type": "human", "id": "human-2", "content": "second"},
|
||||
{"type": "ai", "id": "ai-2", "content": "", "tool_calls": [{"id": "c2", "name": "web_search"}]},
|
||||
_tool_message("c2", "second-a", 3),
|
||||
{"type": "ai", "id": "ai-3", "content": "", "tool_calls": [{"id": "c3", "name": "web_search"}]},
|
||||
_tool_message("c3", "second-b", 5),
|
||||
{"type": "ai", "id": "answer-2", "content": "answer [1][4]"},
|
||||
]
|
||||
|
||||
batches = build_reference_batches(messages)
|
||||
assert [len(batch["sources"]) for batch in batches] == [2, 4]
|
||||
assert [source["index"] for source in batches[1]["sources"]] == [1, 2, 3, 4]
|
||||
assert batches[0]["assistant_message_id"] == "answer-1"
|
||||
assert batches[1]["assistant_message_id"] == "answer-2"
|
||||
Reference in new issue
Block a user