初始化

This commit is contained in:
gozeros committed 2026-09-07 18:24:55 +08:00
1 parent 34474bd5fc
commit b638b541d0
3008 files changed
+837699

No files matched your search

@@ -0,0 +1,70 @@
from .base import BaseSearchProvider, SearchResult
from .demo_search import DemoSearchProvider
from .duckduckgo_search import DuckDuckGoSearchProvider
from .internet_search import InternetSearchProvider
from .intranet_stub import IntranetSearchProvider, normalize_intranet_timeout, normalize_intranet_verify_ssl
from .skill_search import SkillSearchProvider
def get_search_provider(ai_writing_config: dict) -> BaseSearchProvider:
"""
config.yaml 中配置:
ai_writing:
search_provider: internet # internet(默认,DDG) / intranet / tavily / demo
intranet_search_url: http://10.106.57.180:7863/knowledge_base/search_docs_es
intranet_knowledge_base: samples
intranet_search_timeout: 300 # 内网检索超时秒数,最少 300 秒
intranet_verify_ssl: false # 内网 HTTPS 自签名证书场景默认关闭证书校验
ddg_region: "cn-zh" # DuckDuckGo 区域,默认 cn-zh
internet 模式:DuckDuckGo 为主搜索(免 API Key),失败时 fallback 到 demo 演示素材。
"""
cfg = ai_writing_config or {}
provider = cfg.get("search_provider", "internet")
if provider == "intranet":
return IntranetSearchProvider(
base_url=cfg.get("intranet_search_url", ""),
knowledge_base_name=cfg.get("intranet_knowledge_base", "samples"),
timeout=normalize_intranet_timeout(cfg.get("intranet_search_timeout")),
verify_ssl=normalize_intranet_verify_ssl(cfg.get("intranet_verify_ssl")),
)
if provider == "tavily":
return _FallbackSearchProvider(InternetSearchProvider(), DemoSearchProvider())
if provider == "demo":
return DemoSearchProvider()
# 默认 internet 模式:DuckDuckGo + demo fallback
region = cfg.get("ddg_region", "cn-zh")
return _FallbackSearchProvider(DuckDuckGoSearchProvider(region=region), DemoSearchProvider())
class _FallbackSearchProvider(BaseSearchProvider):
"""当主 provider 返回空结果时自动切换到 fallback provider。"""
def __init__(self, primary: BaseSearchProvider, fallback: BaseSearchProvider):
self._primary = primary
self._fallback = fallback
@property
def errors(self) -> list[str]:
"""聚合主/兜底 provider 的检索异常,供 researcher 节点取用。"""
return list(getattr(self._primary, "errors", [])) + list(getattr(self._fallback, "errors", []))
async def search(self, query: str, max_results: int = 5, **kwargs):
results = await self._primary.search(query, max_results=max_results, **kwargs)
if not results:
results = await self._fallback.search(query, max_results=max_results, **kwargs)
return results
async def health_check(self) -> bool:
return await self._primary.health_check() or await self._fallback.health_check()
__all__ = [
"BaseSearchProvider", "SearchResult",
"InternetSearchProvider", "IntranetSearchProvider",
"DemoSearchProvider", "DuckDuckGoSearchProvider",
"SkillSearchProvider",
"get_search_provider",
]
@@ -0,0 +1,84 @@
import re
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import List
# 复合检索词里常见的「话题/类别后缀」——核心实体后面跟的修饰词。内网 ES 对
# 「实体+话题」这类长词常检不出(如「特朗普竞选」0 条),但「实体」本身能检出
# (「特朗普」30 条)。剥掉这些后缀回退到核心实体重试,能救回大量检索词。
_TOPIC_SUFFIXES = (
"竞选", "诉讼", "法律", "政策", "主张", "纲领", "动向", "动态", "影响", "分析",
"进展", "现状", "情况", "问题", "事件", "活动", "关系", "布局", "态势", "格局",
"趋势", "背景", "历史", "观点", "评价", "报道", "新闻", "研究", "报告", "概况",
"走势", "前景", "措施", "方案", "计划", "战略", "改革", "发展",
)
def simplify_query(query: str) -> str:
"""把一个「检不出」的复合检索词简化成更核心、更短的检索词,供 0 结果时重试。
规则(无需大模型,纯启发式):
1. 多词(含空格)→ 取第一个词(通常是核心实体),如「美国政局 特朗普」→「美国政局」;
2. 单词复合 → 反复剥掉结尾的话题后缀,保留核心实体,如「特朗普法律诉讼」→「特朗普」、
「特朗普竞选」→「特朗普」。
返回空串 = 无法进一步简化(调用方据此不再重试,避免空转)。
"""
s = (query or "").strip()
if not s:
return ""
# 1) 多词:取第一个词
if re.search(r"\s", s):
first = re.split(r"\s+", s)[0].strip()
if first and first != s and len(first) >= 2:
return first
# 2) 单词复合:循环剥离结尾话题后缀
changed = s
while True:
stripped = changed
for suf in sorted(_TOPIC_SUFFIXES, key=len, reverse=True):
if stripped.endswith(suf) and len(stripped) - len(suf) >= 2:
stripped = stripped[: -len(suf)]
break
if stripped == changed:
break
changed = stripped
if changed != s and len(changed) >= 2:
return changed
return ""
@dataclass
class SearchResult:
"""统一搜索结果格式 —— 内外网共用"""
title: str
content: str
source: str
published_at: str
url: str = ""
relevance_score: float = 0.0
source_type: str = "internet"
recUuid: str = ""
extra_metadata: dict = field(default_factory=dict)
class BaseSearchProvider(ABC):
"""搜索提供者抽象基类。
对接内网步骤:
1. 继承此类,创建 IntranetSearchProvider
2. 实现 search() —— 调用内网 API,将响应映射到 SearchResult
3. 在 config.yaml 中设置 ai_writing.search_provider: intranet
4. 无需修改任何 Agent 代码
"""
@abstractmethod
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]: ...
@abstractmethod
async def health_check(self) -> bool: ...
@@ -0,0 +1,183 @@
"""离线演示搜索提供者 —— 当无法访问互联网/内网时返回模拟素材,供测试完整写作流程。"""
from __future__ import annotations
import hashlib
from typing import List
from .base import BaseSearchProvider, SearchResult
# 演示素材库:按话题分类,用于关键词模糊匹配
_DEMO_CORPUS = [
{
"keywords": ["新能源", "电动车", "汽车", "ev", "electric"],
"items": [
{
"title": "2025年新能源汽车市场规模突破万亿,渗透率达45%",
"content": "据中国汽车工业协会数据,2025年前三季度新能源汽车累计销量超过850万辆,同比增长32%,市场渗透率达到45.3%。比亚迪、吉利、长城等头部企业份额持续集中,纯电与插混比例趋近5:5。",
"source": "中国汽车工业协会",
"published_at": "2025-10-15",
"relevance_score": 0.95,
},
{
"title": "固态电池量产进入倒计时:多家车企宣布2026年装车计划",
"content": "宁德时代、比亚迪、丰田同步推进固态电池量产。固态电池能量密度可达400Wh/kg,较现有液态锂电池提升60%,续航里程有望突破1000公里。成本仍是主要瓶颈,预计量产初期每度电成本高出30%。",
"source": "电池中国网",
"published_at": "2025-09-20",
"relevance_score": 0.88,
},
{
"title": "充电基础设施缺口分析:农村地区桩车比仅1:12",
"content": "截至2025年8月,全国充电桩保有量达1120万根,城市地区桩车比约1:2.8,但农村及县域地区桩车比仅1:12,制约新能源汽车下沉市场。国家电网计划2026年前新增农村快充站3万座。",
"source": "中国充电联盟",
"published_at": "2025-08-30",
"relevance_score": 0.82,
},
],
},
{
"keywords": ["人工智能", "ai", "大模型", "llm", "机器学习", "深度学习"],
"items": [
{
"title": "国内大模型进入「百亿参数普惠化」阶段,推理成本下降90%",
"content": "2025年第三季度,国内主要大模型服务商将百亿参数模型推理价格降至每百万token不足1元,较2024年同期下降90%。开源生态加速,Qwen3、DeepSeek-V3等模型在多项基准测试中超越GPT-4o。",
"source": "中国信通院",
"published_at": "2025-10-08",
"relevance_score": 0.93,
},
{
"title": "企业AI应用落地白皮书:制造业ROI最高达300%",
"content": "调研覆盖500家企业的白皮书显示,AI在质检、排产、预测性维护三个场景ROI最高,部分制造企业ROI超300%。阻碍因素:数据孤岛(67%)、人才不足(54%)、安全合规(48%)。",
"source": "麦肯锡中国",
"published_at": "2025-09-12",
"relevance_score": 0.87,
},
{
"title": "AI写作助手用户规模达2.3亿,内容质量争议持续",
"content": "国内AI写作工具月活用户突破2.3亿,覆盖学术论文、营销文案、新闻报道等场景。争议焦点:AI生成内容的版权归属、事实准确性(幻觉率约15%)、对写作从业者就业影响。",
"source": "艾瑞咨询",
"published_at": "2025-10-01",
"relevance_score": 0.79,
},
],
},
{
"keywords": ["碳中和", "双碳", "绿色", "低碳", "碳排放", "可再生能源", "光伏", "风电"],
"items": [
{
"title": "光伏装机量突破900GW,度电成本降至0.18元",
"content": "国家能源局数据显示,截至2025年9月,全国光伏累计装机量达912GW,占总装机的28%。光伏度电成本已降至0.18元,首次低于煤电基准价,推动更多省份出现「光伏平价」区域。",
"source": "国家能源局",
"published_at": "2025-10-12",
"relevance_score": 0.92,
},
{
"title": "碳市场价格突破200元/吨,倒逼高耗能行业转型",
"content": "全国碳排放权交易市场碳价已突破200元/吨,较2024年底上涨65%。水泥、电解铝、造纸等高耗能行业面临显著成本压力,碳减排投入年均增速达40%。",
"source": "上海环境能源交易所",
"published_at": "2025-09-28",
"relevance_score": 0.85,
},
],
},
{
"keywords": ["医疗", "健康", "药物", "生物", "医药", "病"],
"items": [
{
"title": "AI辅助诊断落地三甲医院,影像科漏诊率降低23%",
"content": "北京协和、上海瑞金等20家三甲医院数据显示,引入AI辅助诊断系统后,肺结节、眼底病变早期漏诊率降低23%,放射科医生平均阅片时间减少40%。监管层面,NMPA已批准38款AI医疗器械。",
"source": "医学界",
"published_at": "2025-09-18",
"relevance_score": 0.91,
},
],
},
{
"keywords": ["经济", "GDP", "贸易", "出口", "产业", "制造"],
"items": [
{
"title": "2025年前三季度GDP同比增长5.1%,消费贡献率提升至58%",
"content": "国家统计局公布,2025年前三季度GDP同比增长5.1%,最终消费对经济增长贡献率达58%,较去年同期提升8个百分点。工业增加值增长5.8%,其中装备制造业增长9.2%,成重要支撑。",
"source": "国家统计局",
"published_at": "2025-10-18",
"relevance_score": 0.90,
},
{
"title": "跨境电商出口额突破2.4万亿元,东南亚成最大增量市场",
"content": "商务部数据显示,2025年前三季度跨境电商出口额达2.4万亿元,同比增长28%。东南亚市场增速最快(+45%),中东、南美紧随其后。Temu、速卖通、Shopee三平台合计占出口份额约62%。",
"source": "商务部",
"published_at": "2025-10-05",
"relevance_score": 0.84,
},
],
},
]
# 通用兜底素材(与任意查询都能凑合搭配)
_FALLBACK_ITEMS = [
{
"title": "行业研究方法论:如何构建有效的一手与二手数据体系",
"content": "高质量行业报告需整合政府统计、上市公司财报、行业协会数据与专家访谈。一手调研样本量建议≥200份,二手数据优先选用权威机构发布的年度报告与季度白皮书,并注明数据时效性与口径差异。",
"source": "产业研究院",
"published_at": "2025-08-15",
"relevance_score": 0.55,
},
{
"title": "政策解读:2025年产业发展专项支持资金管理办法",
"content": "工信部发布2025年产业发展专项资金管理办法,重点支持「专精特新」企业数字化改造、绿色制造和关键技术攻关。单项目最高补贴1000万元,要求申报企业研发投入不低于营收3%。",
"source": "工信部",
"published_at": "2025-07-20",
"relevance_score": 0.50,
},
{
"title": "市场竞争格局分析:头部集中与长尾分散并存",
"content": "多个新兴行业呈现「哑铃型」竞争格局:头部3-5家企业合计市占率超60%,长尾中小企业依靠细分场景差异化生存,中间层企业面临最大竞争压力。并购整合预期升温,2025年战略性并购交易同比增长35%。",
"source": "普华永道",
"published_at": "2025-09-01",
"relevance_score": 0.48,
},
]
def _score_topic(query: str, keywords: list[str]) -> int:
"""简单关键词匹配打分"""
q = query.lower()
return sum(1 for kw in keywords if kw in q)
class DemoSearchProvider(BaseSearchProvider):
"""离线演示搜索提供者:根据查询关键词从内置语料库返回模拟素材。
仅用于开发测试;生产环境应配置 intranet 提供者。
"""
async def search(self, query: str, max_results: int = 5, **kwargs) -> List[SearchResult]:
best_topic = max(_DEMO_CORPUS, key=lambda t: _score_topic(query, t["keywords"]), default=None)
best_score = _score_topic(query, best_topic["keywords"]) if best_topic else 0
items = []
if best_score > 0 and best_topic:
items.extend(best_topic["items"])
# 用通用素材补齐
for fb in _FALLBACK_ITEMS:
if len(items) >= max_results:
break
items.append(fb)
results = []
for i, it in enumerate(items[:max_results]):
url_hash = hashlib.md5(it["title"].encode()).hexdigest()[:8]
results.append(SearchResult(
title=it["title"],
content=it["content"],
source=it["source"],
published_at=it["published_at"],
url=f"https://demo.internal/{url_hash}",
relevance_score=it["relevance_score"],
source_type="demo",
recUuid=f"demo-{i:03d}",
))
return results
async def health_check(self) -> bool:
return True
@@ -0,0 +1,81 @@
"""DuckDuckGo 搜索提供者 —— 免 API Key 互联网搜索。"""
from __future__ import annotations
import asyncio
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
class DuckDuckGoSearchProvider(BaseSearchProvider):
"""使用 DuckDuckGo 进行互联网搜索,无需 API Key。"""
def __init__(self, region: str = "cn-zh", max_timeout: int = 120):
self.region = region
self.max_timeout = max_timeout
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户
self.errors: list[str] = []
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]:
try:
results = await asyncio.to_thread(
self._sync_search, query, max_results
)
return results
except Exception as e:
logger.warning("DuckDuckGo search failed for query=%r: %s", query, e)
self.errors.append(f"{type(e).__name__}: {e}")
return []
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
try:
from ddgs import DDGS
except ImportError:
logger.error("ddgs not installed. Run: uv add ddgs")
self.errors.append("ImportError: ddgs 未安装")
return []
ddgs = DDGS(timeout=self.max_timeout)
try:
raw = ddgs.text(
query,
region=self.region,
safesearch="moderate",
max_results=max_results,
)
items = list(raw) if raw else []
except Exception as e:
logger.warning("DDGS.text failed: %s", e)
self.errors.append(f"{type(e).__name__}: {e}")
return []
results = []
for item in items:
url = item.get("href", item.get("link", ""))
source = url.split("/")[2] if url and "/" in url else "web"
results.append(SearchResult(
title=item.get("title", ""),
content=item.get("body", item.get("snippet", ""))[:500],
source=source,
published_at=item.get("published", ""),
url=url,
relevance_score=0.75,
source_type="internet",
))
return results
async def health_check(self) -> bool:
try:
results = await self.search("test", max_results=1)
return len(results) > 0
except Exception:
return False
@@ -0,0 +1,66 @@
import asyncio
import logging
from typing import List
from .base import BaseSearchProvider, SearchResult
logger = logging.getLogger(__name__)
class InternetSearchProvider(BaseSearchProvider):
"""互联网搜索,使用项目已有的 Tavily 搜索工具。"""
async def search(
self,
query: str,
max_results: int = 5,
date_from: str = "",
date_to: str = "",
) -> List[SearchResult]:
try:
results = await asyncio.to_thread(self._sync_search, query, max_results)
return results
except Exception as e:
logger.warning("internet search failed for query=%r: %s", query, e)
return []
def _sync_search(self, query: str, max_results: int) -> List[SearchResult]:
from deerflow.config import get_app_config
try:
from tavily import TavilyClient
except ImportError:
logger.warning("tavily not installed, falling back to demo search")
return []
config = get_app_config().get_tool_config("web_search")
api_key = None
if config is not None and "api_key" in config.model_extra:
api_key = config.model_extra.get("api_key")
if not api_key:
logger.warning("Tavily API key not configured, falling back to demo search")
return []
client = TavilyClient(api_key=api_key)
res = client.search(query, max_results=max_results)
results = []
for item in res.get("results", []):
content = item.get("content", "")
results.append(SearchResult(
title=item.get("title", ""),
content=content[:500],
source=item.get("url", "").split("/")[2] if "/" in item.get("url", "") else "web",
published_at=item.get("published_date", ""),
url=item.get("url", ""),
relevance_score=item.get("score", 0.0),
source_type="internet",
))
return results
async def health_check(self) -> bool:
try:
results = await self.search("test", max_results=1)
return True
except Exception:
return False
@@ -0,0 +1,282 @@
"""内网知识库搜索适配器 —— 对接 ES 检索接口 search_docs_es。"""
from __future__ import annotations
import logging
import time
from .base import BaseSearchProvider, SearchResult, simplify_query
logger = logging.getLogger(__name__)
# 内网知识库默认检索接口与知识库名(可经 config.yaml 的 ai_writing 段覆盖)
_DEFAULT_ENDPOINT = "http://10.106.57.180:7863/knowledge_base/search_docs_es"
_DEFAULT_KB = "samples"
# 内网检索最少等待 5 分钟;该接口经常慢返回,低于这个值会造成素材收集误判为空。
_MIN_TIMEOUT_SECONDS = 300
_DEFAULT_TIMEOUT_SECONDS = 600
# 内网 HTTPS 检索服务常使用自签名/内部 CA 证书,默认跳过证书校验避免 SSL 证书链错误。
_DEFAULT_VERIFY_SSL = False
# 单条原始响应体最大保留长度,避免把超大响应整段灌进前端时间线
_RAW_SNIPPET_LIMIT = 2000
def normalize_intranet_timeout(timeout: object, default: int = _DEFAULT_TIMEOUT_SECONDS) -> int:
"""返回内网检索超时秒数,并强制不低于 5 分钟。"""
try:
value = int(timeout)
except (TypeError, ValueError):
value = default
return max(value, _MIN_TIMEOUT_SECONDS)
def normalize_intranet_verify_ssl(value: object, default: bool = _DEFAULT_VERIFY_SSL) -> bool:
"""返回内网检索是否校验 SSL 证书,兼容 config.yaml 里的 bool / 字符串写法。"""
if isinstance(value, bool):
return value
if value is None:
return default
if isinstance(value, (int, float)):
return bool(value)
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"1", "true", "yes", "on"}:
return True
if normalized in {"0", "false", "no", "off"}:
return False
return default
def _pick(d: dict, keys: tuple) -> str:
"""从结果项里按候选字段名取第一个非空值(两套接口字段名不同,做并集兼容)。"""
for k in keys:
v = d.get(k)
if isinstance(v, str) and v.strip():
return v.strip()
if v not in (None, "") and not isinstance(v, (dict, list)):
return str(v).strip()
return ""
def _raw_snippet(body: str) -> str:
"""Return a bounded raw response snippet for diagnostics."""
body = (body or "").strip()
if not body:
return ""
if len(body) > _RAW_SNIPPET_LIMIT:
return body[:_RAW_SNIPPET_LIMIT] + "…(响应体过长,已截断)"
return body
class IntranetSearchProvider(BaseSearchProvider):
"""内网知识库搜索接口适配器。
接口:POST {endpoint}
请求体:{"query": "...", "knowledge_base_name": "samples", "ts": "yes"}
响应体:[{"page_content": "...", "metadata": {m_title, content, recUuid, m_publish, source1}}]
"""
def __init__(
self,
base_url: str = "",
knowledge_base_name: str = _DEFAULT_KB,
timeout: int = _DEFAULT_TIMEOUT_SECONDS,
verify_ssl: bool = _DEFAULT_VERIFY_SSL,
**_ignored,
):
self.endpoint = base_url or _DEFAULT_ENDPOINT
self.knowledge_base_name = knowledge_base_name or _DEFAULT_KB
# 内网 ES 接口很慢,兜底检索超时默认 10 分钟,且无论配置如何至少 5 分钟。
self.timeout = normalize_intranet_timeout(timeout)
self.verify_ssl = normalize_intranet_verify_ssl(verify_ssl)
# 收集本次运行的检索异常,供 researcher 节点取出并提示用户(ES 波动等)
self.errors: list[str] = []
# 解析失败时收集的原始响应体,供前端「错误返回结果」折叠展示
self.raw_responses: list[str] = []
# 每一次真实 HTTP 调用的诊断明细。search() 可能会因 0 结果触发简化词重试,
# 因此一个业务关键词可能对应多条 calls。
self.calls: list[dict] = []
def _record_raw(self, body: str) -> None:
"""收集解析失败 / 结构异常时的原始响应体,供前端折叠查看。"""
snippet = _raw_snippet(body)
if snippet:
self.raw_responses.append(snippet)
async def search(self, query: str, max_results: int = 5, **kwargs) -> list[SearchResult]:
results = await self._search_once(query, max_results)
# 0 结果兜底:复合检索词常检不出(如「特朗普竞选」),换成更核心的检索词重试一次
# (「特朗普竞选」→「特朗普」)。仅在首检为空、且能简化出不同的词时才重试。
if not results:
simpler = simplify_query(query)
if simpler and simpler != query:
logger.info("intranet search empty for %r, retry with simplified %r", query, simpler)
results = await self._search_once(simpler, max_results)
return results
async def _search_once(self, query: str, max_results: int = 5) -> list[SearchResult]:
import httpx
payload = {
"query": query,
"knowledge_base_name": self.knowledge_base_name,
"ts": "yes",
}
# 先拿到原始响应文本,JSON 解析失败时把它原样回传前端便于定位接口返回了什么
raw_text = ""
started = time.monotonic()
call: dict = {
"query": query,
"endpoint": self.endpoint,
"knowledge_base_name": self.knowledge_base_name,
"max_results": max_results,
"status": "pending",
"ok": False,
"count": 0,
"verify_ssl": self.verify_ssl,
}
# 排查用:每次真实调用都打一行请求明细(端点 / 知识库 / 检索词 / 超时),
# 内网复现时直接看后端日志即可知道"到底请求了哪个地址、用什么参数"。
logger.info(
"[intranet-search] POST endpoint=%s kb=%s timeout=%ss verify_ssl=%s query=%r",
self.endpoint, self.knowledge_base_name, self.timeout, self.verify_ssl, query,
)
try:
# trust_env=False:**关键**。内网 ES 是内网地址,但本部署的模型走公网,
# 服务器通常设了 HTTP_PROXY/HTTPS_PROXY 出公网;httpx 默认 trust_env=True
# 会把这个内网请求也套用公网代理 → 代理到不了内网 IP → 连接被重置/空回复,
# 表现为「接口异常」。强制忽略代理 env、直连内网(与 multi_agent.py 的回环
# 调用同款修法)。
# verify=False:内网 HTTPS 常见自签名/内部 CA 证书;这里仅跳过本内网检索端点的证书校验。
async with httpx.AsyncClient(timeout=self.timeout, trust_env=False, verify=self.verify_ssl) as client:
resp = await client.post(self.endpoint, json=payload)
call["http_status"] = getattr(resp, "status_code", None)
resp.raise_for_status()
raw_text = resp.text
data = resp.json()
except httpx.HTTPStatusError as e:
# 非 2xx:接口有响应但状态码异常,把响应体一并暴露给前端
body_for_log = e.response.text if e.response is not None else ""
logger.warning(
"[intranet-search] HTTP error endpoint=%s query=%r status=%s body=%s",
self.endpoint, query, getattr(e.response, "status_code", None), _raw_snippet(body_for_log),
)
message = f"{type(e).__name__}: {e}"
self.errors.append(message)
call.update({
"status": "http_error",
"error": message,
"http_status": getattr(e.response, "status_code", None),
})
snippet = _raw_snippet(body_for_log)
if snippet:
call["raw_response"] = snippet
self._record_raw(body_for_log)
return []
except Exception as e:
# 连接失败 / 超时 / JSON 解析失败等;JSON 解析失败时 raw_text 已是接口响应体。
# 打出异常类型,便于区分 ConnectError / ConnectTimeout / ReadTimeout /
# RemoteProtocolError(空回复) / JSONDecodeError(返回非 JSON) 等具体根因。
logger.warning(
"[intranet-search] request failed endpoint=%s query=%r error=%s: %s",
self.endpoint, query, type(e).__name__, e,
)
message = f"{type(e).__name__}: {e}"
self.errors.append(message)
call.update({"status": "error", "error": message})
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self._record_raw(raw_text)
return []
finally:
call["elapsed_ms"] = int((time.monotonic() - started) * 1000)
# 非 pending 表示已经在异常分支写好;成功分支会在 return 前覆盖并追加。
if call.get("status") != "pending":
self.calls.append(call)
# 正常情况下接口直接返回数组;兼容包了一层 data/results 的结构(含工作正常的
# knowledge-search-v2:{"success":true,"results":[...]})。
if isinstance(data, list):
items = data
elif isinstance(data, dict):
if "success" in data:
call["response_success"] = bool(data.get("success"))
if "count" in data:
call["response_count"] = data.get("count")
# 接口显式标失败时把整体回传前端便于定位
if data.get("success") is False:
message = f"接口返回 success=false:{str(data.get('error') or '')[:200]}"
self.errors.append(message)
call.update({"status": "success_false", "error": message})
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self.calls.append(call)
self._record_raw(raw_text)
return []
items = data.get("data") or data.get("results") or data.get("documents") or []
else:
# 返回结构既不是数组也不是已知 dict 包装:视为异常,把原始响应回传前端
logger.warning("intranet search unexpected response type for query=%r: %s", query, type(data).__name__)
message = f"返回结构异常:期望数组,实际为 {type(data).__name__}"
self.errors.append(message)
call.update({"status": "bad_shape", "error": message})
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self.calls.append(call)
self._record_raw(raw_text)
return []
logger.info(
"[intranet-search] OK endpoint=%s query=%r http_status=%s raw_items=%s",
self.endpoint, query, call.get("http_status"), len(items) if isinstance(items, list) else 0,
)
results: list[SearchResult] = []
for i, item in enumerate(items[:max_results]):
if not isinstance(item, dict):
continue
# 字段名做并集兼容两套接口:
# 老 ES: {page_content, metadata:{m_title, content, recUuid, m_publish, source1}}
# knowledge-search-v2:{title, content_preview, url, recUuid, source_category, publish_time, relevance_score}
md = item.get("metadata") if isinstance(item.get("metadata"), dict) else {}
merged = {**md, **item} if md else item
content = _pick(merged, ("content", "content_preview", "page_content", "text", "summary"))
title = _pick(merged, ("title", "m_title", "name", "doc_title"))
if not content and not title:
continue
try:
score = float(merged.get("relevance_score"))
except (TypeError, ValueError):
# 接口未返回相关度分值:按返回顺序赋递减分以保留排序
score = max(0.5, 0.95 - i * 0.03)
results.append(SearchResult(
title=title or "(无标题)",
content=str(content)[:500],
source=_pick(merged, ("source_category", "source1", "source", "source_name", "kb_name")) or "内网知识库",
published_at=_pick(merged, ("publish_time", "m_publish", "published_at", "date")),
# 工作接口直接给了 url;老 ES 没有 url(由前端依据 recUuid 拼接),保持空串
url=_pick(merged, ("url",)),
relevance_score=score,
source_type="intranet",
recUuid=_pick(merged, ("recUuid", "rec_uuid", "id")),
))
call.update({
"status": "ok" if results else "empty",
"ok": True,
"count": len(results),
"item_count": len(items) if isinstance(items, list) else 0,
})
if not results:
snippet = _raw_snippet(raw_text)
if snippet:
call["raw_response"] = snippet
self.calls.append(call)
return results
async def health_check(self) -> bool:
try:
await self.search("测试", max_results=1)
return True
except Exception:
return False
@@ -0,0 +1,229 @@
"""技能选择器:根据用户意图 + 输入,从启用的 Skills 里挑出按相关度排序的若干个。
注意:这个模块**只负责选择技能,不直接产出素材**。
真正的素材产出由 researcher 节点用每个 skill 的描述去编排真实 web/intranet 检索。
流程(对应素材确认卡上的"技能检索"按钮):
1) list_skills 事件 —— 列出当前启用的全部 skill
2) skill_select_chunk 事件 —— LLM 流式打字 + <think></think> 分离(前端打字机展示)
3) select_skills 事件 —— LLM 给出的排序结果(结构化字段,写到 SkillStepsCard 里)
下游 researcher 节点拿到 selected_skills 后,**按顺序**:
- 用 skill.description + user_query 生成针对性 query
- 调真实 web search provider 执行
- 累积素材达到阈值就停(推 stop_early)
"""
from __future__ import annotations
import logging
import time
from typing import Callable, List, Optional
from langchain_core.messages import HumanMessage, SystemMessage
from langchain_core.runnables import RunnableConfig
from ..nodes._utils import (
ThinkTagParser,
_content_to_text,
get_model,
parse_json,
reasoning_delta,
safe_get_stream_writer,
)
logger = logging.getLogger(__name__)
_SELECT_SKILLS_SYSTEM = """你是一名情报研究员,需要从可用「技能」中挑出对当前检索任务最有帮助的若干个,并按相关度从高到低排序。
技能是一段封装好的指令 / 知识 / 检索策略,每个技能有 name 和 description。
【输出要求】
- 只输出一个 JSON 对象,不要 markdown 代码块。
- 必须使用英文双引号。
- selected_skills 中的 name 必须严格来自下方"可用技能"列表,不要编造,按相关度从高到低排序。
- 如果用户输入里明确点名了某个技能(按 name 完全匹配),把它排在第一位。
- 至少选 1 个,最多选 {max_select} 个。
【完整示例】
{{"selected_skills": ["platform-sales-data", "industry-report"], "reason": "用户想查各平台销量,平台销量技能最相关"}}"""
_SELECT_SKILLS_USER = """写作意图:{user_intent}
用户补充输入:{user_query}
可用技能(共 {skill_count} 个):
{skill_list}
请从中按相关度从高到低选出最多 {max_select} 个技能。
只输出 JSON:
{{"selected_skills": ["name1", "name2"], "reason": "为什么选这些 + 排序依据"}}"""
EmitStepFn = Callable[[dict], None]
class SkillPicker:
"""从启用的 Skills 中流式选择按相关度排序的若干个供下游编排使用。
构造参数:
config: LangGraph runnable config(用于创建 model)
model_name: 指定使用的模型,None 则用 configurable.model_name
emit_step: 推送时间线步骤事件(list_skills / select_skills)
max_select: LLM 单次最多挑几个技能
user_query: 用户在素材确认卡输入框写的自然语言
candidates: 候选技能池。None = 全部启用的技能(旧行为);
传入列表(如素材收集专家 agent 配置的技能)则只在其中匹配排序
"""
def __init__(
self,
config: RunnableConfig,
*,
model_name: Optional[str] = None,
emit_step: Optional[EmitStepFn] = None,
max_select: int = 5,
user_query: str = "",
candidates: Optional[list] = None,
) -> None:
self._config = config
self._model_name = model_name
self._emit_step = emit_step or (lambda _step: None)
self._max_select = max_select
self._user_query = (user_query or "").strip()
self._candidates = candidates
self._errors: list[str] = []
@property
def errors(self) -> list[str]:
return list(self._errors)
def _load_enabled_skills(self) -> list:
if self._candidates is not None:
skills = list(self._candidates)
self._emit_step({
"type": "list_skills",
"message": f"素材收集专家已配置 {len(skills)} 个检索技能",
"skill_names": [s.name for s in skills],
})
return skills
try:
from deerflow.skills import get_or_new_skill_storage
storage = get_or_new_skill_storage()
skills = storage.load_skills(enabled_only=True)
except Exception as e:
logger.exception("ai_writing.skill_search: 加载启用技能失败")
self._errors.append(f"skill list error: {e}")
skills = []
self._emit_step({
"type": "list_skills",
"message": f"发现 {len(skills)} 个可用技能",
"skill_names": [s.name for s in skills],
})
return skills
async def pick(self, user_intent: str) -> List[dict]:
"""流式选择并排序候选技能,返回 [{skill, reason}, ...],按相关度降序。
过程中通过 custom 通道推送 ``skill_select_chunk`` 事件给前端打字机;
结束后再推一条 ``select_skills`` 结构化事件。
"""
skills = self._load_enabled_skills()
if not skills:
self._emit_step({
"type": "select_skills",
"message": "当前没有启用的技能可用",
"skill_names": [],
})
return []
skill_list_text = "\n".join(
f"- {s.name}: {s.description or '(无描述)'}" for s in skills
)
fallback = {"selected_skills": [s.name for s in skills[: self._max_select]], "reason": ""}
# list_skills 之后立刻发一个"分析中"提示步骤,避免 LLM 第一个 chunk 到来前
# 用户看到空窗期(某些模型 astream 在头部 buffer 时间较长)
self._emit_step({
"type": "match_skills",
"message": "正在让模型从候选技能中筛选最合适的…",
})
# 流式 LLM 调用:与 intent_parser / writer_outline 同套机制
writer = safe_get_stream_writer()
parser = ThinkTagParser()
full_text = ""
_chunk_count = {"n": 0}
def _emit_chunk(text: str, is_thinking: bool) -> None:
# 只在首个 chunk 打一条 timing,避免逐 token 刷屏;用来和 skill_search_step
# 的 [skill-timing] 对比"选择打字"与"步骤事件"谁先到。
if _chunk_count["n"] == 0:
logger.info("[skill-timing] t=%.3f first skill_select_chunk", time.monotonic())
_chunk_count["n"] += 1
writer({
"type": "skill_select_chunk",
"text": text,
"is_thinking": is_thinking,
"agent_name": "素材收集专家",
})
# 立刻推一个空 chunk,让前端 streaming view 立即挂载(占位 + 闪烁光标),
# 即使后续 LLM 首个 chunk 到达较慢,UI 也不再静默
_emit_chunk("", False)
try:
model = get_model(self._config, model_name=self._model_name)
async for chunk in model.astream([
SystemMessage(content=_SELECT_SKILLS_SYSTEM.format(max_select=self._max_select)),
HumanMessage(content=_SELECT_SKILLS_USER.format(
user_intent=user_intent or "(无明确写作意图)",
user_query=self._user_query or "(无)",
skill_count=len(skills),
skill_list=skill_list_text,
max_select=self._max_select,
)),
]):
# 推理模型的思考走 reasoning_content(content 为空):作为思考实时推送
reasoning = reasoning_delta(chunk)
if reasoning:
_emit_chunk(reasoning, True)
delta = _content_to_text(chunk.content)
if not delta:
continue
for is_thinking, text in parser.feed(delta):
if not is_thinking:
full_text += text
_emit_chunk(text, is_thinking)
except Exception as e:
logger.exception("ai_writing.skill_search: 流式选择技能失败")
self._errors.append(f"skill select error: {e}")
result = parse_json(full_text, fallback) if full_text else fallback
wanted_names = result.get("selected_skills") or []
if not isinstance(wanted_names, list):
wanted_names = []
by_name = {s.name: s for s in skills}
reason = result.get("reason", "") or ""
chosen: list[dict] = []
for name in wanted_names:
if not isinstance(name, str):
continue
sk = by_name.get(name.strip())
if sk and sk.name not in {c["skill"].name for c in chosen}:
chosen.append({"skill": sk, "reason": reason})
if len(chosen) >= self._max_select:
break
self._emit_step({
"type": "select_skills",
"message": f"按相关度排序,候选 {len(chosen)} 个技能(将按顺序使用)",
"skill_names": [c["skill"].name for c in chosen],
"reason": reason,
})
return chosen
# 保留旧名字让 search/__init__.py 的导出不报错;研究节点不再用它
SkillSearchProvider = SkillPicker