作家写作
This commit is contained in:
1 parent
1a122eb7fd
commit
07ad87365f
7 files changed
+380
-228
No files matched your search
+143
-90
@@ -9,7 +9,8 @@ from typing import List
|
||||
|
||||
from langchain_core.runnables import RunnableConfig
|
||||
|
||||
from ..search import get_search_provider, SearchResult, SkillSearchProvider
|
||||
from ..search import get_search_provider, SearchResult
|
||||
from ..search.skill_search import SkillPicker
|
||||
from ..state import AIWritingState, Material, MaterialPackage
|
||||
from ..prompts.researcher_prompts import (
|
||||
EXTRACT_KEYWORDS_SYSTEM, EXTRACT_KEYWORDS_USER,
|
||||
@@ -135,10 +136,13 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
writer({"type": "research_progress", "message": message})
|
||||
|
||||
# Step 1: 提取关键词
|
||||
kw_result: dict = {} # 走"用户输入 / 追加关键词"分支时跳过 LLM 调用,notice 拼接需要兜底值
|
||||
if is_user_query_search:
|
||||
kw_result: dict = {} # 走"用户输入 / 追加 / 技能"分支时跳过 LLM 调用,notice 拼接需要兜底值
|
||||
if is_skill_search:
|
||||
# 技能模式:每个 skill 会自己拼 query(user_query + skill.description),这里不用 LLM 生成关键词
|
||||
keywords: list[str] = [user_query or intent]
|
||||
elif is_user_query_search:
|
||||
# 用户输入分支:把整段自然语言作为唯一检索词,尊重原意,不再做关键词拆分
|
||||
keywords: list[str] = [user_query]
|
||||
keywords = [user_query]
|
||||
_emit(f"按你的输入「{user_query[:30]}」检索…")
|
||||
elif is_append_only:
|
||||
# 兼容旧版追加模式:直接用用户填的关键词,不再调 LLM
|
||||
@@ -164,57 +168,115 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
keywords = list(dict.fromkeys(keywords + extra_keywords))
|
||||
|
||||
# Step 2: 搜索
|
||||
# 技能检索:用 SkillSearchProvider 替代 web/intranet;步骤事件单独走 skill_search_step 通道
|
||||
# 技能检索:技能不是素材,是"检索策略 / 视角"。流程:
|
||||
# a. SkillPicker 选出按相关度排序的 skills(推 list_skills + select_skills 事件)
|
||||
# b. 按顺序遍历 skills:每个 skill 用其 description 让 LLM 生成针对性关键词
|
||||
# c. 调真实 web/intranet provider 执行检索,推 invoke_skill + skill_result 事件
|
||||
# d. 累积素材到 max_materials 即 break,推 stop_early
|
||||
skill_steps: list[dict] = []
|
||||
search_errors: list[str] = []
|
||||
|
||||
def _emit_skill_step(step: dict) -> None:
|
||||
skill_steps.append(step)
|
||||
writer({"type": "skill_search_step", "agent_name": "素材收集专家", **step})
|
||||
|
||||
async def _llm_select(system: str, user_prompt: str, fallback: dict) -> dict:
|
||||
return await llm_json(system, user_prompt, fallback=fallback, config=config, model_name=keyword_model)
|
||||
|
||||
if is_skill_search:
|
||||
def _emit_skill_step(step: dict) -> None:
|
||||
skill_steps.append(step)
|
||||
writer({"type": "skill_search_step", "agent_name": "素材收集专家", **step})
|
||||
|
||||
async def _llm_select(system: str, user_prompt: str, fallback: dict) -> dict:
|
||||
return await llm_json(system, user_prompt, fallback=fallback, config=config, model_name=keyword_model)
|
||||
|
||||
provider = SkillSearchProvider(
|
||||
# ── 技能编排检索 ──
|
||||
picker = SkillPicker(
|
||||
llm_select=_llm_select,
|
||||
emit_step=_emit_skill_step,
|
||||
max_select=max(1, min(int(max_materials), 10)),
|
||||
max_select=5,
|
||||
user_query=user_query,
|
||||
)
|
||||
selected = await picker.pick(user_intent=intent)
|
||||
search_errors.extend(picker.errors)
|
||||
|
||||
# 真实 web/intranet provider(按 config.yaml 配置)
|
||||
web_provider = get_search_provider(ai_writing_config)
|
||||
|
||||
# 每个 skill 尽量多取(不分摊),第一个就达到 max_materials 即可早停;
|
||||
# 这才贴近"第一个技能够了就停,否则下一个补充"的语义。
|
||||
per_skill = max(1, max_materials)
|
||||
raw_materials_pre: list[Material] = []
|
||||
seen_urls_pre: set[str] = set()
|
||||
stopped_early = False
|
||||
|
||||
for entry in selected:
|
||||
sk = entry["skill"]
|
||||
# 用 LLM 把 user_query + skill.description 转成针对性关键词
|
||||
_emit_skill_step({
|
||||
"type": "invoke_skill",
|
||||
"message": f"使用技能「{sk.name}」开始检索…",
|
||||
"skill_name": sk.name,
|
||||
})
|
||||
skill_query = f"{user_query} {sk.description or ''}".strip() or intent
|
||||
try:
|
||||
res = await web_provider.search(skill_query, max_results=per_skill)
|
||||
except Exception as e:
|
||||
logger.warning("skill[%s] search failed: %s", sk.name, e)
|
||||
search_errors.append(f"skill[{sk.name}]: {e}")
|
||||
res = []
|
||||
|
||||
# 去重累积
|
||||
new_count = 0
|
||||
for r in res or []:
|
||||
if r.url and r.url in seen_urls_pre:
|
||||
continue
|
||||
if r.url:
|
||||
seen_urls_pre.add(r.url)
|
||||
raw_materials_pre.append(Material(
|
||||
id=f"m{len(raw_materials_pre)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=f"{r.source}(via 技能 {sk.name})",
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
new_count += 1
|
||||
|
||||
_emit_skill_step({
|
||||
"type": "skill_result",
|
||||
"message": f"技能「{sk.name}」检索到 {new_count} 条新素材",
|
||||
"skill_name": sk.name,
|
||||
"matched_count": new_count,
|
||||
})
|
||||
|
||||
# 早停:累积素材已达上限
|
||||
if len(raw_materials_pre) >= max_materials:
|
||||
stopped_early = True
|
||||
_emit_skill_step({
|
||||
"type": "stop_early",
|
||||
"message": f"已获取 {len(raw_materials_pre)} 条素材,达到上限,停止调用后续技能",
|
||||
"matched_count": len(raw_materials_pre),
|
||||
})
|
||||
break
|
||||
|
||||
# 技能模式不再走下面的并发/合并逻辑,all_results_nested 留空、跳过 web 通用搜索
|
||||
all_results_nested: list = []
|
||||
# 把已收集到的素材挂在 raw_materials_pre,下方合并阶段直接挪过去
|
||||
else:
|
||||
# ── 常规 web/intranet 模式 ──
|
||||
provider = get_search_provider(ai_writing_config)
|
||||
|
||||
# 按用户指定的关键词数量截断。
|
||||
# 用户输入 / 追加模式:keywords 本身就是用户填的,全部检索不截断。
|
||||
# 其他模式:受 keyword_count 限制,但 extra_keywords 在内时上限放宽到 keywords 总数。
|
||||
if is_user_query_search or is_append_only:
|
||||
search_keywords = keywords
|
||||
else:
|
||||
search_limit = keyword_count
|
||||
if extra_keywords:
|
||||
search_limit = max(search_limit, len(keywords))
|
||||
search_keywords = keywords[:search_limit]
|
||||
total = len(search_keywords)
|
||||
# 每个检索词分到的检索条数 = 总量上限 / 检索词数(向上取整)
|
||||
per_keyword = _per_keyword_limit(max_materials, total)
|
||||
_emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
# 按用户指定的关键词数量截断。
|
||||
# 用户输入 / 追加模式:keywords 本身就是用户填的,全部检索不截断。
|
||||
if is_user_query_search or is_append_only:
|
||||
search_keywords = keywords
|
||||
else:
|
||||
search_limit = keyword_count
|
||||
if extra_keywords:
|
||||
search_limit = max(search_limit, len(keywords))
|
||||
search_keywords = keywords[:search_limit]
|
||||
total = len(search_keywords)
|
||||
per_keyword = _per_keyword_limit(max_materials, total)
|
||||
_emit(f"检索词已生成,开始检索 {total} 个方向…")
|
||||
|
||||
# 技能模式:串行调用 provider(早停语义靠 provider 内部累积计数)
|
||||
# 其它模式:并发调用,每完成一个就推送进度
|
||||
if is_skill_search:
|
||||
all_results_nested: list[list[SearchResult]] = []
|
||||
for kw in search_keywords:
|
||||
try:
|
||||
res = await provider.search(kw, max_results=per_keyword, user_intent=intent)
|
||||
except Exception as e:
|
||||
logger.warning("skill search failed: %s", e)
|
||||
all_results_nested.append(e) # type: ignore[arg-type]
|
||||
continue
|
||||
all_results_nested.append(res)
|
||||
# provider 内部 early_stop_at 已触发时不会再返回新 skill,提前 break 避免空轮询
|
||||
if not res:
|
||||
break
|
||||
else:
|
||||
completed = 0
|
||||
|
||||
async def _search_one(kw: str) -> list[SearchResult]:
|
||||
@@ -230,59 +292,48 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
|
||||
search_tasks = [_search_one(kw) for kw in search_keywords]
|
||||
all_results_nested = await asyncio.gather(*search_tasks, return_exceptions=True)
|
||||
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
|
||||
# 合并去重
|
||||
# - 首次检索 / 普通 re_search(不带 extra_keywords,用户只想 LLM 重出):旧素材丢弃
|
||||
# - 素材不足求助 supplement_search:保留旧素材 + 追加新检索结果
|
||||
# - PAUSE-1 追加检索 is_append_only:保留旧素材 + 仅检索新关键词
|
||||
# - 技能检索 is_skill_search:覆盖式替换,旧素材完全丢弃(用户期望对比 web vs skill 结果)
|
||||
seen_urls: set[str] = set()
|
||||
seen_skill_ids: set[str] = set() # skill 素材无 URL,按 skill_name 去重
|
||||
raw_materials: list[Material] = []
|
||||
# - 技能检索 is_skill_search:覆盖式替换,旧素材完全丢弃(编排循环已经把素材累积到 raw_materials_pre)
|
||||
existing_pkg = state.get("material_package")
|
||||
preserve_existing = existing_pkg and (is_supplement or is_append_only) and not is_skill_search
|
||||
if preserve_existing:
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
if is_skill_search:
|
||||
# 技能编排循环已经完成素材累积,直接用 raw_materials_pre 当 raw_materials
|
||||
raw_materials: list[Material] = raw_materials_pre # type: ignore[name-defined]
|
||||
else:
|
||||
seen_urls: set[str] = set()
|
||||
raw_materials = []
|
||||
preserve_existing = existing_pkg and (is_supplement or is_append_only)
|
||||
if preserve_existing:
|
||||
for m in existing_pkg.get("materials", []):
|
||||
seen_urls.add(m.get("url", ""))
|
||||
raw_materials.append(m)
|
||||
|
||||
search_errors: list[str] = []
|
||||
for results in all_results_nested:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
continue
|
||||
for r in results:
|
||||
# skill 素材按 skill_name 去重(同一个 skill 被多个关键词命中只保留一次)
|
||||
if r.source_type == "skill":
|
||||
meta = getattr(r, "extra_metadata", {}) or {}
|
||||
sk_name = meta.get("skill_name", "")
|
||||
if sk_name and sk_name in seen_skill_ids:
|
||||
continue
|
||||
if sk_name:
|
||||
seen_skill_ids.add(sk_name)
|
||||
elif r.url and r.url in seen_urls:
|
||||
for results in all_results_nested:
|
||||
if isinstance(results, Exception):
|
||||
logger.warning("search failed: %s", results)
|
||||
search_errors.append(f"{type(results).__name__}: {results}")
|
||||
continue
|
||||
else:
|
||||
seen_urls.add(r.url)
|
||||
mat: Material = Material(
|
||||
id=f"m{len(raw_materials)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=r.source,
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
)
|
||||
extra = getattr(r, "extra_metadata", None)
|
||||
if extra:
|
||||
mat["extra_metadata"] = extra
|
||||
raw_materials.append(mat)
|
||||
|
||||
# provider 内部吞掉的检索异常(如内网 ES 连接失败),一并取出用于提示用户
|
||||
search_errors.extend(getattr(provider, "errors", []))
|
||||
for r in results:
|
||||
if r.url and r.url in seen_urls:
|
||||
continue
|
||||
if r.url:
|
||||
seen_urls.add(r.url)
|
||||
raw_materials.append(Material(
|
||||
id=f"m{len(raw_materials)+1:03d}",
|
||||
title=r.title,
|
||||
content=r.content[:500],
|
||||
source=r.source,
|
||||
published_at=r.published_at,
|
||||
relevance_score=r.relevance_score,
|
||||
source_type=r.source_type,
|
||||
url=r.url,
|
||||
recUuid=r.recUuid,
|
||||
))
|
||||
|
||||
# Step 3: 相关度排序(如素材较多,让 LLM 排序)
|
||||
_emit("检索完成,正在筛选排序素材…")
|
||||
@@ -343,7 +394,9 @@ async def researcher_node(state: AIWritingState, config: RunnableConfig) -> dict
|
||||
if search_errors:
|
||||
uniq = list(dict.fromkeys(search_errors))
|
||||
# 解析失败时收集到的接口原始响应体,前端折叠展示便于定位
|
||||
raw_dumps = list(dict.fromkeys(getattr(provider, "raw_responses", []) or []))
|
||||
# (技能模式下 provider 局部变量没暴露,所以 raw_dumps 兜底为空列表)
|
||||
provider_obj = locals().get("provider") or locals().get("web_provider")
|
||||
raw_dumps = list(dict.fromkeys(getattr(provider_obj, "raw_responses", []) or []))
|
||||
notices.append(_notice(
|
||||
"ES 波动导致检索异常",
|
||||
";".join(uniq[:3]),
|
||||
|
||||
Reference in new issue
Block a user